Finn_Dane/spices2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

make output more generic

Browse Source
This commit is contained in:
Finn Dane 2023-04-16 13:05:13 +02:00
parent fe2af2978b
commit b039834d83
3 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/RdaReader/include/RdaReader.hpp
View File

@ -15,25 +15,25 @@ class RdaReader {
const std::vector<char> &datasetName, const std::vector<char> &datasetName,
const std::vector<char> &sharedIndex, const std::vector<char> &sharedIndex,
const std::vector<std::istream *> &rdas, const std::vector<std::istream *> &rdas,
std::ostream &output const std::function<void(const char *s, size_t n)> output
); );
size_t readDataset( size_t readDataset(
const std::vector<char> &datasetName, const std::vector<char> &datasetName,
const std::vector<char> &sharedIndex, const std::vector<char> &sharedIndex,
const std::vector<std::istream *> &rdas, const std::vector<std::istream *> &rdas,
std::ostream &output, const std::function<void(const char *s, size_t n)> output,
std::mutex &outputMutex std::mutex &outputMutex
); );
size_t readRda( size_t readRda(
std::istream &rda, std::istream &rda,
uint64_t id, uint64_t id,
std::ostream &output const std::function<void(const char *s, size_t n)> output
); );
size_t readRda( size_t readRda(
std::istream &rda, std::istream &rda,
uint64_t id, uint64_t id,
std::ostream &output, const std::function<void(const char *s, size_t n)> output,
std::mutex &outputMutex std::mutex &outputMutex
); );
private: private:

11
src/RdaReader/src/RdaReader.cpp
View File

@ -17,7 +17,7 @@ RdaReader::RdaReader() : logger(nullptr) {}
RdaReader::RdaReader(std::function<void(const std::string &)> logger) : logger(logger) {} RdaReader::RdaReader(std::function<void(const std::string &)> logger) : logger(logger) {}
size_t RdaReader::readRda(std::istream &rda, uint64_t id, std::ostream &output, std::mutex &outputMutex) { size_t RdaReader::readRda(std::istream &rda, uint64_t id, const std::function<void(const char *s, size_t n)> output, std::mutex &outputMutex) {
log("Reading an rda\n"); log("Reading an rda\n");
BucketedZstdData bucket(rda); BucketedZstdData bucket(rda);
@ -25,7 +25,8 @@ size_t RdaReader::readRda(std::istream &rda, uint64_t id, std::ostream &output,
const std::lock_guard lock(outputMutex); const std::lock_guard lock(outputMutex);
log("Writing " + std::to_string(data.value().size()) + " entries to output\n"); log("Writing " + std::to_string(data.value().size()) + " entries to output\n");
for(const auto &entry : data.value()) { for(const auto &entry : data.value()) {
output.write(entry.data(), entry.size()) << '\n'; output(entry.data(), entry.size());
output("\n", 1);
} }
log("Done writing entries\n"); log("Done writing entries\n");
return data.value().size(); return data.value().size();
@ -34,7 +35,7 @@ size_t RdaReader::readRda(std::istream &rda, uint64_t id, std::ostream &output,
return 0; return 0;
} }
size_t RdaReader::readRda(std::istream &rda, uint64_t id, std::ostream &output) { size_t RdaReader::readRda(std::istream &rda, uint64_t id, const std::function<void(const char *s, size_t n)> output) {
std::mutex dummyMutex; std::mutex dummyMutex;
return readRda(rda, id, output, dummyMutex); return readRda(rda, id, output, dummyMutex);
} }
@ -49,7 +50,7 @@ std::optional<std::istream *> getVictimIfAvailable(std::vector<std::istream *> &
return {}; return {};
} }
size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::vector<char> &sharedIndex, const std::vector<std::istream *> &rdas, std::ostream &output, std::mutex &outputMutex) { size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::vector<char> &sharedIndex, const std::vector<std::istream *> &rdas, const std::function<void(const char *s, size_t n)> output, std::mutex &outputMutex) {
log("Reading shared index... "); log("Reading shared index... ");
SharedIndex sharedIndexReader(sharedIndex); SharedIndex sharedIndexReader(sharedIndex);
@ -79,7 +80,7 @@ size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::v
return 0; return 0;
} }
size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::vector<char> &sharedIndex, const std::vector<std::istream *> &rdas, std::ostream &output) { size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::vector<char> &sharedIndex, const std::vector<std::istream *> &rdas, const std::function<void(const char *s, size_t n)> output) {
std::mutex dummyMutex; std::mutex dummyMutex;
return readDataset(datasetName, sharedIndex, rdas, output, dummyMutex); return readDataset(datasetName, sharedIndex, rdas, output, dummyMutex);
} }

2
src/rdaExtractor/src/rdaExtractor.cpp
View File

@ -72,7 +72,7 @@ int main(int argc, char **argv) {
rdaRefs.push_back(&stream); rdaRefs.push_back(&stream);
} }
RdaReader rdaReader(threadedLog); RdaReader rdaReader(threadedLog);
if(size_t totalRead = rdaReader.readDataset(datasetName, sharedIndexData, rdaRefs, std::cout)) { if(size_t totalRead = rdaReader.readDataset(datasetName, sharedIndexData, rdaRefs, [](const char *s, size_t n){std::cout.write(s, n);})) {
std::cerr << "Found a total of " << totalRead << " entries" << std::endl; std::cerr << "Found a total of " << totalRead << " entries" << std::endl;
} else { } else {
std::cerr << "Cannot find '" << argv[1] << "' in the shared index" << std::endl; std::cerr << "Cannot find '" << argv[1] << "' in the shared index" << std::endl;