Finn_Dane
/
spices2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

adds functionality for giving only a rootdirectory to lib

This commit is contained in:
Finn Dane 2023-04-16 15:39:17 +02:00
parent b039834d83
commit 65460beabf
3 changed files with 61 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/RdaReader/include/RdaReader.hpp
View File

@ -6,11 +6,25 @@
#include <vector>
#include <mutex>
#include <functional>
#include <filesystem>
class RdaReader {
public:
RdaReader();
RdaReader(std::function<void(const std::string &)> logger);
size_t readDataset(
const std::vector<char> &datasetName,
const std::filesystem::path &rootDir,
const std::function<void(const char *s, size_t n)> output
);
size_t readDataset(
const std::vector<char> &datasetName,
const std::filesystem::path &rootDir,
const std::function<void(const char *s, size_t n)> output,
std::mutex &outputMutex
);
size_t readDataset(
const std::vector<char> &datasetName,
const std::vector<char> &sharedIndex,

46
src/RdaReader/src/RdaReader.cpp
View File

@ -83,4 +83,50 @@ size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::v
size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::vector<char> &sharedIndex, const std::vector<std::istream *> &rdas, const std::function<void(const char *s, size_t n)> output) {
std::mutex dummyMutex;
return readDataset(datasetName, sharedIndex, rdas, output, dummyMutex);
}
inline std::optional<std::vector<char>> readSharedIndex(const std::filesystem::path &filePath) {
std::ifstream sharedIndexFile(filePath, std::ios::binary | std::ios::ate);
if(!sharedIndexFile.good()) {
return {};
}
std::vector<char> sharedIndexData(sharedIndexFile.tellg());
sharedIndexFile.seekg(0, std::ios::beg);
sharedIndexFile.read(sharedIndexData.data(), sharedIndexData.size());
return sharedIndexData;
}
size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::filesystem::path &rootDir, const std::function<void(const char *s, size_t n)> output, std::mutex &outputMutex) {
const std::filesystem::path sharedIndexPath(rootDir / "sharedindex.shi");
std::vector<char> sharedIndexData;
if(auto data = readSharedIndex(sharedIndexPath)) {
sharedIndexData = data.value();
} else {
log("cannot find '" + sharedIndexPath.string() + "'\n");
return 0;
}
std::vector<std::ifstream> rdasIfstreams;
for(const std::filesystem::directory_entry &file : std::filesystem::directory_iterator(rootDir)) {
if(file.path().extension() == ".rda") rdasIfstreams.emplace_back(std::ifstream(file.path(), std::ios::binary));
}
std::vector<std::istream *> rdaRefs;
for(std::ifstream &stream : rdasIfstreams) {
rdaRefs.push_back(&stream);
}
if(size_t totalRead = readDataset(datasetName, sharedIndexData, rdaRefs, [](const char *s, size_t n){std::cout.write(s, n);}, outputMutex)) {
log("Found a total of " + std::to_string(totalRead) + " entries\n");
return totalRead;
} else {
log("Cannot find '" + std::string(datasetName.data()) + "' in the shared index\n");
return 0;
}
}
size_t RdaReader::readDataset(const std::vector<char> &datasetName, const std::filesystem::path &rootDir, const std::function<void(const char *s, size_t n)> output) {
std::mutex dummyMutex;
return readDataset(datasetName, rootDir, output, dummyMutex);
}

25
src/rdaExtractor/src/rdaExtractor.cpp
View File

@ -52,29 +52,6 @@ int main(int argc, char **argv) {
std::vector<char> datasetName(datesetString.begin(), datesetString.end());
const std::filesystem::path rootDirectory(argv[2]);
const std::filesystem::path sharedIndexPath(rootDirectory / "sharedindex.shi");
std::vector<char> sharedIndexData;
if(auto data = readSharedIndex(sharedIndexPath)) {
sharedIndexData = data.value();
} else {
std::cerr << "cannot find '" << sharedIndexPath << "'" << std::endl;
return 2;
}
std::vector<std::ifstream> rdasIfstreams;
for(const std::filesystem::directory_entry &file : std::filesystem::directory_iterator(rootDirectory)) {
if(file.path().extension() == ".rda") rdasIfstreams.emplace_back(std::ifstream(file.path(), std::ios::binary));
}
std::vector<std::istream *> rdaRefs;
for(std::ifstream &stream : rdasIfstreams) {
rdaRefs.push_back(&stream);
}
RdaReader rdaReader(threadedLog);
if(size_t totalRead = rdaReader.readDataset(datasetName, sharedIndexData, rdaRefs, [](const char *s, size_t n){std::cout.write(s, n);})) {
std::cerr << "Found a total of " << totalRead << " entries" << std::endl;
} else {
std::cerr << "Cannot find '" << argv[1] << "' in the shared index" << std::endl;
}
rdaReader.readDataset(datasetName, rootDirectory, [](const char *s, size_t n){std::cout.write(s, n);});
}