Reduce memory usage when loading packages
* Avoid creation of map with all archive contents; instead parse packages while walking though the archive * Avoid instantiation of std::string in come cases (using std::string_view) * Reuse libarchive's archive entry when walking though archive * Use visitor-based database parser in all places to avoid intermediate big array with all package objects
This commit is contained in:
parent
10cf6169a6
commit
26caa78956
|
@ -404,21 +404,6 @@ StorageID Database::forceUpdatePackage(const std::shared_ptr<Package> &package)
|
||||||
return res.id;
|
return res.id;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Database::replacePackages(const std::vector<std::shared_ptr<Package>> &newPackages, DateTime lastModified)
|
|
||||||
{
|
|
||||||
for (const auto &package : newPackages) {
|
|
||||||
if (const auto existingPackage = findPackage(package->name)) {
|
|
||||||
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
auto updater = PackageUpdater(*this, true);
|
|
||||||
for (const auto &package : newPackages) {
|
|
||||||
updater.update(package);
|
|
||||||
}
|
|
||||||
updater.commit();
|
|
||||||
lastUpdate = lastModified;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*!
|
/*!
|
||||||
* \brief Determines which packages are unresolvable assuming new packages are added to the database and certain provides are removed.
|
* \brief Determines which packages are unresolvable assuming new packages are added to the database and certain provides are removed.
|
||||||
* \param config The configuration supposed to contain database dependencies.
|
* \param config The configuration supposed to contain database dependencies.
|
||||||
|
@ -799,6 +784,18 @@ StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
|
||||||
return res.id;
|
return res.id;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool PackageUpdater::insertFromDatabaseFile(const std::string &databaseFilePath)
|
||||||
|
{
|
||||||
|
LibPkg::Package::fromDatabaseFile(databaseFilePath, [this](const std::shared_ptr<LibPkg::Package> &package) {
|
||||||
|
if (const auto [id, existingPackage] = findPackageWithID(package->name); existingPackage) {
|
||||||
|
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
|
||||||
|
}
|
||||||
|
update(package);
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
void PackageUpdater::commit()
|
void PackageUpdater::commit()
|
||||||
{
|
{
|
||||||
const auto &storage = m_database.m_storage;
|
const auto &storage = m_database.m_storage;
|
||||||
|
|
|
@ -106,6 +106,7 @@ struct LIBPKG_EXPORT PackageUpdater {
|
||||||
|
|
||||||
PackageSpec findPackageWithID(const std::string &packageName);
|
PackageSpec findPackageWithID(const std::string &packageName);
|
||||||
StorageID update(const std::shared_ptr<Package> &package);
|
StorageID update(const std::shared_ptr<Package> &package);
|
||||||
|
bool insertFromDatabaseFile(const std::string &databaseFilePath);
|
||||||
void commit();
|
void commit();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
@ -147,8 +148,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
|
||||||
void resetConfiguration(bool keepLocalPaths = false);
|
void resetConfiguration(bool keepLocalPaths = false);
|
||||||
void clearPackages();
|
void clearPackages();
|
||||||
void loadPackagesFromConfiguredPaths(bool withFiles = false, bool force = false);
|
void loadPackagesFromConfiguredPaths(bool withFiles = false, bool force = false);
|
||||||
void loadPackages(const std::string &databaseData, CppUtilities::DateTime lastModified);
|
void loadPackages(const std::string &databaseFilePath, CppUtilities::DateTime lastModified);
|
||||||
void loadPackages(FileMap &&databaseFiles, CppUtilities::DateTime lastModified);
|
|
||||||
static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);
|
static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);
|
||||||
std::vector<std::shared_ptr<Package>> findPackages(const std::function<bool(const Database &, const Package &)> &pred);
|
std::vector<std::shared_ptr<Package>> findPackages(const std::function<bool(const Database &, const Package &)> &pred);
|
||||||
void allPackages(const PackageVisitorMove &visitor);
|
void allPackages(const PackageVisitorMove &visitor);
|
||||||
|
@ -164,7 +164,6 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
|
||||||
void removePackage(const std::string &packageName);
|
void removePackage(const std::string &packageName);
|
||||||
StorageID updatePackage(const std::shared_ptr<Package> &package);
|
StorageID updatePackage(const std::shared_ptr<Package> &package);
|
||||||
StorageID forceUpdatePackage(const std::shared_ptr<Package> &package);
|
StorageID forceUpdatePackage(const std::shared_ptr<Package> &package);
|
||||||
void replacePackages(const std::vector<std::shared_ptr<Package>> &newPackages, CppUtilities::DateTime lastModified);
|
|
||||||
std::unordered_map<PackageSpec, UnresolvedDependencies> detectUnresolvedPackages(Config &config,
|
std::unordered_map<PackageSpec, UnresolvedDependencies> detectUnresolvedPackages(Config &config,
|
||||||
const std::vector<std::shared_ptr<Package>> &newPackages, const DependencySet &removedPackages,
|
const std::vector<std::shared_ptr<Package>> &newPackages, const DependencySet &removedPackages,
|
||||||
const std::unordered_set<std::string_view> &depsToIgnore = std::unordered_set<std::string_view>(),
|
const std::unordered_set<std::string_view> &depsToIgnore = std::unordered_set<std::string_view>(),
|
||||||
|
|
|
@ -384,9 +384,9 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
|
||||||
std::string computeRegularPackageName() const;
|
std::string computeRegularPackageName() const;
|
||||||
PackageNameData decomposeName() const;
|
PackageNameData decomposeName() const;
|
||||||
void addInfoFromPkgInfoFile(const std::string &info);
|
void addInfoFromPkgInfoFile(const std::string &info);
|
||||||
void addDepsAndProvidesFromContainedDirectory(const std::string &directoryPath);
|
void addDepsAndProvidesFromContainedDirectory(std::string_view directoryPath);
|
||||||
void addDepsAndProvidesFromContainedFile(
|
void addDepsAndProvidesFromContainedFile(
|
||||||
const std::string &directoryPath, const ArchiveFile &file, std::set<std::string> &dllsReferencedByImportLibs);
|
std::string_view directoryPath, const ArchiveFile &file, std::set<std::string> &dllsReferencedByImportLibs);
|
||||||
void addDepsAndProvidesFromContents(const FileMap &contents);
|
void addDepsAndProvidesFromContents(const FileMap &contents);
|
||||||
std::vector<std::string> processDllsReferencedByImportLibs(std::set<std::string> &&dllsReferencedByImportLibs);
|
std::vector<std::string> processDllsReferencedByImportLibs(std::set<std::string> &&dllsReferencedByImportLibs);
|
||||||
bool addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bool force = false);
|
bool addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bool force = false);
|
||||||
|
@ -397,8 +397,7 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
|
||||||
|
|
||||||
static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false);
|
static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false);
|
||||||
static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts);
|
static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts);
|
||||||
static std::vector<std::shared_ptr<Package>> fromDatabaseFile(FileMap &&databaseFile);
|
static void fromDatabaseFile(const std::string &archivePath, const std::function<bool(std::shared_ptr<Package>)> &visitor);
|
||||||
static void fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor);
|
|
||||||
static std::shared_ptr<Package> fromPkgFile(const std::string &path);
|
static std::shared_ptr<Package> fromPkgFile(const std::string &path);
|
||||||
static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName);
|
static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName);
|
||||||
static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName);
|
static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName);
|
||||||
|
|
|
@ -11,6 +11,7 @@
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <fstream>
|
#include <fstream>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
|
#include <string_view>
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
using namespace CppUtilities;
|
using namespace CppUtilities;
|
||||||
|
@ -199,7 +200,7 @@ void Binary::load(std::string_view filePath)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Binary::load(const string &fileContent, const string &fileName, const string &directoryPath, bool isRegularFile)
|
void Binary::load(std::string_view fileContent, std::string_view fileName, std::string_view directoryPath, bool isRegularFile)
|
||||||
{
|
{
|
||||||
stringstream fileStream(ios_base::in | ios_base::out | ios_base::binary);
|
stringstream fileStream(ios_base::in | ios_base::out | ios_base::binary);
|
||||||
fileStream.exceptions(ios_base::failbit | ios_base::badbit);
|
fileStream.exceptions(ios_base::failbit | ios_base::badbit);
|
||||||
|
@ -239,17 +240,17 @@ static std::string toLower(std::string str)
|
||||||
return str;
|
return str;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string Binary::addPrefix(const std::string &dependencyName) const
|
std::string Binary::addPrefix(std::string_view dependencyName) const
|
||||||
{
|
{
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case BinaryType::Elf:
|
case BinaryType::Elf:
|
||||||
return argsToString(extraPrefix, "elf-", architecture, ':', ':', dependencyName);
|
return argsToString(extraPrefix, "elf-", architecture, ':', ':', dependencyName);
|
||||||
case BinaryType::Pe:
|
case BinaryType::Pe:
|
||||||
return argsToString(extraPrefix, "pe-", architecture, ':', ':', toLower(dependencyName));
|
return argsToString(extraPrefix, "pe-", architecture, ':', ':', toLower(std::string(dependencyName)));
|
||||||
case BinaryType::Ar:
|
case BinaryType::Ar:
|
||||||
switch (subType) {
|
switch (subType) {
|
||||||
case BinarySubType::WindowsImportLibrary:
|
case BinarySubType::WindowsImportLibrary:
|
||||||
return argsToString(extraPrefix, "pe-", architecture, ':', ':', toLower(dependencyName));
|
return argsToString(extraPrefix, "pe-", architecture, ':', ':', toLower(std::string(dependencyName)));
|
||||||
default:;
|
default:;
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
|
@ -257,7 +258,7 @@ std::string Binary::addPrefix(const std::string &dependencyName) const
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Binary::parse(istream &stream, const string *fileContent)
|
void Binary::parse(std::istream &stream, const std::string_view *fileContent)
|
||||||
{
|
{
|
||||||
type = BinaryType::Invalid;
|
type = BinaryType::Invalid;
|
||||||
|
|
||||||
|
@ -285,7 +286,7 @@ void Binary::parse(istream &stream, const string *fileContent)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void Binary::parseElf(BinaryReader &reader, const string *fileContent)
|
void Binary::parseElf(BinaryReader &reader, const std::string_view *fileContent)
|
||||||
{
|
{
|
||||||
istream &stream = *reader.stream();
|
istream &stream = *reader.stream();
|
||||||
|
|
||||||
|
@ -731,8 +732,8 @@ std::uint16_t Binary::readElfInt16(BinaryReader &reader)
|
||||||
return isBigEndian ? reader.readUInt16BE() : reader.readUInt16LE();
|
return isBigEndian ? reader.readUInt16BE() : reader.readUInt16LE();
|
||||||
}
|
}
|
||||||
|
|
||||||
string Binary::readElfString(
|
std::string Binary::readElfString(std::istream &stream, const std::string_view *fileContent, std::uint64_t stringTableAddress,
|
||||||
istream &stream, const string *fileContent, std::uint64_t stringTableAddress, std::uint64_t stringTableSize, std::uint64_t relativeStringAddress)
|
std::uint64_t stringTableSize, std::uint64_t relativeStringAddress)
|
||||||
{
|
{
|
||||||
// check bounds
|
// check bounds
|
||||||
if (relativeStringAddress >= stringTableSize) {
|
if (relativeStringAddress >= stringTableSize) {
|
||||||
|
|
|
@ -83,8 +83,8 @@ inline VirtualAddressMapping::VirtualAddressMapping()
|
||||||
|
|
||||||
struct LIBPKG_EXPORT Binary {
|
struct LIBPKG_EXPORT Binary {
|
||||||
void load(std::string_view filePath);
|
void load(std::string_view filePath);
|
||||||
void load(const std::string &fileContent, const std::string &fileName, const std::string &directoryPath, bool isRegularFile = false);
|
void load(std::string_view fileContent, std::string_view fileName, std::string_view directoryPath, bool isRegularFile = false);
|
||||||
std::string addPrefix(const std::string &dependencyName) const;
|
std::string addPrefix(std::string_view dependencyName) const;
|
||||||
|
|
||||||
BinaryType type = BinaryType::Invalid;
|
BinaryType type = BinaryType::Invalid;
|
||||||
BinarySubType subType = BinarySubType::None;
|
BinarySubType subType = BinarySubType::None;
|
||||||
|
@ -99,8 +99,8 @@ struct LIBPKG_EXPORT Binary {
|
||||||
VirtualAddressMapping virtualAddressMapping;
|
VirtualAddressMapping virtualAddressMapping;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void parse(std::istream &stream, const std::string *fileContent = nullptr);
|
void parse(std::istream &stream, const std::string_view *fileContent = nullptr);
|
||||||
void parseElf(CppUtilities::BinaryReader &reader, const std::string *fileContent = nullptr);
|
void parseElf(CppUtilities::BinaryReader &reader, const std::string_view *fileContent = nullptr);
|
||||||
void parsePe(CppUtilities::BinaryReader &reader, typename std::iostream::off_type baseFileOffset = 0);
|
void parsePe(CppUtilities::BinaryReader &reader, typename std::iostream::off_type baseFileOffset = 0);
|
||||||
void parseAr(CppUtilities::BinaryReader &reader);
|
void parseAr(CppUtilities::BinaryReader &reader);
|
||||||
void parseCoff(CppUtilities::BinaryReader &reader);
|
void parseCoff(CppUtilities::BinaryReader &reader);
|
||||||
|
@ -108,8 +108,8 @@ private:
|
||||||
std::uint64_t readElfAddress(CppUtilities::BinaryReader &reader);
|
std::uint64_t readElfAddress(CppUtilities::BinaryReader &reader);
|
||||||
std::uint32_t readElfInt32(CppUtilities::BinaryReader &reader);
|
std::uint32_t readElfInt32(CppUtilities::BinaryReader &reader);
|
||||||
std::uint16_t readElfInt16(CppUtilities::BinaryReader &reader);
|
std::uint16_t readElfInt16(CppUtilities::BinaryReader &reader);
|
||||||
std::string readElfString(std::istream &stream, const std::string *fileContent, std::uint64_t stringTableOffset, std::uint64_t stringTableSize,
|
std::string readElfString(std::istream &stream, const std::string_view *fileContent, std::uint64_t stringTableOffset,
|
||||||
std::uint64_t stringOffset);
|
std::uint64_t stringTableSize, std::uint64_t stringOffset);
|
||||||
};
|
};
|
||||||
} // namespace LibPkg
|
} // namespace LibPkg
|
||||||
|
|
||||||
|
|
|
@ -29,32 +29,16 @@ void Database::loadPackagesFromConfiguredPaths(bool withFiles, bool force)
|
||||||
}
|
}
|
||||||
const auto lastFileUpdate = lastModified(dbPath);
|
const auto lastFileUpdate = lastModified(dbPath);
|
||||||
if (force || lastFileUpdate > lastUpdate) {
|
if (force || lastFileUpdate > lastUpdate) {
|
||||||
loadPackages(extractFiles(dbPath, &isFileRelevant), lastFileUpdate);
|
loadPackages(dbPath, lastFileUpdate);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void LibPkg::Database::loadPackages(const string &databaseData, DateTime lastModified)
|
void Database::loadPackages(const std::string &databaseFilePath, DateTime lastModified)
|
||||||
{
|
{
|
||||||
loadPackages(extractFilesFromBuffer(databaseData, name + " db file", &isFileRelevant), lastModified);
|
auto updater = PackageUpdater(*this, true);
|
||||||
}
|
updater.insertFromDatabaseFile(databaseFilePath);
|
||||||
|
|
||||||
void Database::loadPackages(FileMap &&databaseFiles, DateTime lastModified)
|
|
||||||
{
|
|
||||||
lastUpdate = lastModified;
|
|
||||||
auto updater = PackageUpdater(*this);
|
|
||||||
for (auto &dir : databaseFiles) {
|
|
||||||
if (dir.first.find('/') != std::string::npos) {
|
|
||||||
cerr << Phrases::WarningMessage << "Database \"" << name << "\" contains unexpected sub directory: " << dir.first << Phrases::EndFlush;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
auto descriptionParts = std::vector<std::string>();
|
|
||||||
descriptionParts.reserve(dir.second.size());
|
|
||||||
for (auto &file : dir.second) {
|
|
||||||
descriptionParts.emplace_back(std::move(file.content));
|
|
||||||
}
|
|
||||||
updater.update(Package::fromDescription(descriptionParts));
|
|
||||||
}
|
|
||||||
updater.commit();
|
updater.commit();
|
||||||
|
lastUpdate = lastModified;
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace LibPkg
|
} // namespace LibPkg
|
||||||
|
|
|
@ -716,33 +716,40 @@ std::shared_ptr<Package> Package::fromDescription(const std::vector<std::string>
|
||||||
return package;
|
return package;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<std::shared_ptr<Package>> Package::fromDatabaseFile(FileMap &&databaseFile)
|
void Package::fromDatabaseFile(const std::string &archivePath, const std::function<bool(std::shared_ptr<Package>)> &visitor)
|
||||||
{
|
{
|
||||||
std::vector<std::shared_ptr<Package>> packages;
|
// walk though archive, file-by-file; parse files as soon as desc/files available and return via visitor
|
||||||
packages.reserve(databaseFile.size());
|
auto packages = std::unordered_map<std::string, std::vector<std::string>>();
|
||||||
std::vector<std::string> descriptionParts;
|
walkThroughArchive(
|
||||||
for (auto &dir : databaseFile) {
|
archivePath, &Database::isFileRelevant,
|
||||||
descriptionParts.clear();
|
[&packages, &visitor](std::string_view path, ArchiveFile &&file) {
|
||||||
descriptionParts.reserve(dir.second.size());
|
auto i = packages.find(std::string(path)); // FIXME: this should actually accept std::string_view in C++20
|
||||||
for (auto &file : dir.second) {
|
if (i == packages.end()) {
|
||||||
descriptionParts.emplace_back(std::move(file.content));
|
i = packages.emplace(std::string(path), 2).first;
|
||||||
}
|
}
|
||||||
packages.emplace_back(Package::fromDescription(descriptionParts));
|
auto &parts = i->second;
|
||||||
}
|
if (file.name == "desc") {
|
||||||
return packages;
|
parts[0] = std::move(file.content);
|
||||||
}
|
} else if (file.name == "files") {
|
||||||
|
parts[1] = std::move(file.content);
|
||||||
|
}
|
||||||
|
if (!parts[0].empty() && !parts[1].empty()) {
|
||||||
|
if (visitor(Package::fromDescription(parts))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
// remove buffered data immediately to avoid using too much memory
|
||||||
|
packages.erase(i);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
[](std::string_view) { return false; });
|
||||||
|
|
||||||
void Package::fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor)
|
// take care of packages without "files" file
|
||||||
{
|
for (auto &[packageName, parts] : packages) {
|
||||||
std::vector<std::string> descriptionParts;
|
if (!parts[0].empty()) {
|
||||||
for (auto &dir : databaseFile) {
|
if (visitor(Package::fromDescription(parts))) {
|
||||||
descriptionParts.clear();
|
return;
|
||||||
descriptionParts.reserve(dir.second.size());
|
}
|
||||||
for (auto &file : dir.second) {
|
|
||||||
descriptionParts.emplace_back(std::move(file.content));
|
|
||||||
}
|
|
||||||
if (visitor(Package::fromDescription(descriptionParts))) {
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -768,11 +775,11 @@ void Package::addInfoFromPkgInfoFile(const string &info)
|
||||||
static const regex pythonVersionRegex("usr/lib/python(2|3)\\.([0-9]*)(\\..*)?/site-packages");
|
static const regex pythonVersionRegex("usr/lib/python(2|3)\\.([0-9]*)(\\..*)?/site-packages");
|
||||||
static const regex perlVersionRegex("usr/lib/perl5/5\\.([0-9]*)(\\..*)?/vendor_perl");
|
static const regex perlVersionRegex("usr/lib/perl5/5\\.([0-9]*)(\\..*)?/vendor_perl");
|
||||||
|
|
||||||
void Package::addDepsAndProvidesFromContainedDirectory(const std::string &directoryPath)
|
void Package::addDepsAndProvidesFromContainedDirectory(std::string_view directoryPath)
|
||||||
{
|
{
|
||||||
// check for Python modules
|
// check for Python modules
|
||||||
thread_local smatch match;
|
thread_local auto match = std::match_results<std::string_view::const_iterator>();
|
||||||
if (regex_match(directoryPath, match, pythonVersionRegex)) {
|
if (std::regex_match(directoryPath.begin(), directoryPath.end(), match, pythonVersionRegex)) {
|
||||||
const auto majorVersion = match[1].str();
|
const auto majorVersion = match[1].str();
|
||||||
const auto minorVersion = match[2].str();
|
const auto minorVersion = match[2].str();
|
||||||
const char *const pythonPackage(majorVersion == "3" ? "python" : "python2");
|
const char *const pythonPackage(majorVersion == "3" ? "python" : "python2");
|
||||||
|
@ -783,7 +790,7 @@ void Package::addDepsAndProvidesFromContainedDirectory(const std::string &direct
|
||||||
}
|
}
|
||||||
|
|
||||||
// check for Perl modules
|
// check for Perl modules
|
||||||
if (regex_match(directoryPath, match, perlVersionRegex)) {
|
if (std::regex_match(directoryPath.begin(), directoryPath.end(), match, perlVersionRegex)) {
|
||||||
const auto minorVersion = match[1].str();
|
const auto minorVersion = match[1].str();
|
||||||
auto currentVersion = "5." + minorVersion;
|
auto currentVersion = "5." + minorVersion;
|
||||||
auto nextVersion = "5." + numberToString(stringToNumber<unsigned int>(minorVersion) + 1);
|
auto nextVersion = "5." + numberToString(stringToNumber<unsigned int>(minorVersion) + 1);
|
||||||
|
@ -793,7 +800,7 @@ void Package::addDepsAndProvidesFromContainedDirectory(const std::string &direct
|
||||||
}
|
}
|
||||||
|
|
||||||
void Package::addDepsAndProvidesFromContainedFile(
|
void Package::addDepsAndProvidesFromContainedFile(
|
||||||
const std::string &directoryPath, const ArchiveFile &file, std::set<std::string> &dllsReferencedByImportLibs)
|
std::string_view directoryPath, const ArchiveFile &file, std::set<std::string> &dllsReferencedByImportLibs)
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
Binary binary;
|
Binary binary;
|
||||||
|
@ -858,24 +865,26 @@ std::shared_ptr<Package> Package::fromPkgFile(const string &path)
|
||||||
shared_ptr<Package> package;
|
shared_ptr<Package> package;
|
||||||
LibPkg::walkThroughArchive(
|
LibPkg::walkThroughArchive(
|
||||||
path, &LibPkg::Package::isPkgInfoFileOrBinary,
|
path, &LibPkg::Package::isPkgInfoFileOrBinary,
|
||||||
[&package, &tmpPackageForLibraryDeps, &dllsReferencedByImportLibs](std::string &&directoryPath, LibPkg::ArchiveFile &&file) {
|
[&package, &tmpPackageForLibraryDeps, &dllsReferencedByImportLibs](std::string_view directoryPath, LibPkg::ArchiveFile &&file) {
|
||||||
if (directoryPath.empty() && file.name == ".PKGINFO") {
|
if (directoryPath.empty() && file.name == ".PKGINFO") {
|
||||||
if (package) {
|
if (package) {
|
||||||
return; // only consider one .PKGINFO file (multiple ones are likely not possible in any supported archive formats anyways)
|
return false; // only consider one .PKGINFO file (multiple ones are likely not possible in any supported archive formats anyways)
|
||||||
}
|
}
|
||||||
auto packages = fromInfo(file.content, true);
|
auto packages = fromInfo(file.content, true);
|
||||||
if (!packages.empty()) {
|
if (!packages.empty()) {
|
||||||
package = std::move(packages.front().pkg);
|
package = std::move(packages.front().pkg);
|
||||||
}
|
}
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
tmpPackageForLibraryDeps.addDepsAndProvidesFromContainedFile(directoryPath, file, dllsReferencedByImportLibs);
|
tmpPackageForLibraryDeps.addDepsAndProvidesFromContainedFile(directoryPath, file, dllsReferencedByImportLibs);
|
||||||
|
return false;
|
||||||
},
|
},
|
||||||
[&tmpPackageForLibraryDeps](std::string &&directoryPath) {
|
[&tmpPackageForLibraryDeps](std::string_view directoryPath) {
|
||||||
if (directoryPath.empty()) {
|
if (directoryPath.empty()) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
tmpPackageForLibraryDeps.addDepsAndProvidesFromContainedDirectory(directoryPath);
|
tmpPackageForLibraryDeps.addDepsAndProvidesFromContainedDirectory(directoryPath);
|
||||||
|
return false;
|
||||||
});
|
});
|
||||||
if (!package) {
|
if (!package) {
|
||||||
throw runtime_error("Package " % path + " does not contain a valid .PKGINFO");
|
throw runtime_error("Package " % path + " does not contain a valid .PKGINFO");
|
||||||
|
|
|
@ -21,17 +21,19 @@ using namespace CppUtilities;
|
||||||
namespace LibPkg {
|
namespace LibPkg {
|
||||||
|
|
||||||
struct AddDirectoryToFileMap {
|
struct AddDirectoryToFileMap {
|
||||||
void operator()(std::string &&path)
|
bool operator()(std::string_view path)
|
||||||
{
|
{
|
||||||
fileMap[std::move(path)];
|
fileMap[std::string(path)];
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
FileMap &fileMap;
|
FileMap &fileMap;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct AddFileToFileMap {
|
struct AddFileToFileMap {
|
||||||
void operator()(std::string &&directoryPath, ArchiveFile &&file)
|
bool operator()(std::string_view directoryPath, ArchiveFile &&file)
|
||||||
{
|
{
|
||||||
fileMap[std::move(directoryPath)].emplace_back(std::move(file));
|
fileMap[std::string(directoryPath)].emplace_back(std::move(file));
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
FileMap &fileMap;
|
FileMap &fileMap;
|
||||||
};
|
};
|
||||||
|
@ -40,8 +42,8 @@ void walkThroughArchiveInternal(struct archive *ar, const string &archiveName, c
|
||||||
DirectoryHandler &&directoryHandler)
|
DirectoryHandler &&directoryHandler)
|
||||||
{
|
{
|
||||||
// iterate through all archive entries
|
// iterate through all archive entries
|
||||||
struct archive_entry *entry;
|
struct archive_entry *const entry = archive_entry_new();
|
||||||
while (archive_read_next_header(ar, &entry) == ARCHIVE_OK) {
|
while (archive_read_next_header2(ar, entry) == ARCHIVE_OK) {
|
||||||
// check entry type (only dirs, files and symlinks relevant here)
|
// check entry type (only dirs, files and symlinks relevant here)
|
||||||
const auto entryType(archive_entry_filetype(entry));
|
const auto entryType(archive_entry_filetype(entry));
|
||||||
if (entryType != AE_IFDIR && entryType != AE_IFREG && entryType != AE_IFLNK) {
|
if (entryType != AE_IFDIR && entryType != AE_IFREG && entryType != AE_IFLNK) {
|
||||||
|
@ -70,7 +72,9 @@ void walkThroughArchiveInternal(struct archive *ar, const string &archiveName, c
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
directoryHandler(string(filePath, dirEnd));
|
if (directoryHandler(std::string_view(filePath, dirEnd))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -94,21 +98,23 @@ void walkThroughArchiveInternal(struct archive *ar, const string &archiveName, c
|
||||||
|
|
||||||
// read symlink
|
// read symlink
|
||||||
if (entryType == AE_IFLNK) {
|
if (entryType == AE_IFLNK) {
|
||||||
fileHandler(string(filePath, static_cast<string::size_type>(dirEnd - filePath)),
|
if (fileHandler(std::string_view(filePath, static_cast<string::size_type>(dirEnd - filePath)),
|
||||||
ArchiveFile(fileName, string(archive_entry_symlink_utf8(entry)), ArchiveFileType::Link, creationTime, modificationTime));
|
ArchiveFile(fileName, std::string(archive_entry_symlink_utf8(entry)), ArchiveFileType::Link, creationTime, modificationTime))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// determine file size to pre-allocate buffer for file content
|
// determine file size to pre-allocate buffer for file content
|
||||||
const la_int64_t fileSize = archive_entry_size(entry);
|
const la_int64_t fileSize = archive_entry_size(entry);
|
||||||
string fileContent;
|
std::string fileContent;
|
||||||
if (fileSize > 0) {
|
if (fileSize > 0) {
|
||||||
fileContent.reserve(static_cast<string::size_type>(fileSize));
|
fileContent.reserve(static_cast<string::size_type>(fileSize));
|
||||||
}
|
}
|
||||||
|
|
||||||
// read file content
|
// read file content
|
||||||
const char *buff;
|
const char *buff;
|
||||||
size_t size;
|
std::size_t size;
|
||||||
la_int64_t offset;
|
la_int64_t offset;
|
||||||
for (;;) {
|
for (;;) {
|
||||||
int returnCode = archive_read_data_block(ar, reinterpret_cast<const void **>(&buff), &size, &offset);
|
int returnCode = archive_read_data_block(ar, reinterpret_cast<const void **>(&buff), &size, &offset);
|
||||||
|
@ -119,11 +125,14 @@ void walkThroughArchiveInternal(struct archive *ar, const string &archiveName, c
|
||||||
}
|
}
|
||||||
|
|
||||||
// move it to results
|
// move it to results
|
||||||
fileHandler(string(filePath, static_cast<string::size_type>(dirEnd - filePath)),
|
if (fileHandler(std::string_view(filePath, static_cast<string::size_type>(dirEnd - filePath)),
|
||||||
ArchiveFile(fileName, move(fileContent), ArchiveFileType::Regular, creationTime, modificationTime));
|
ArchiveFile(fileName, std::move(fileContent), ArchiveFileType::Regular, creationTime, modificationTime))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// free resources used by libarchive
|
// free resources used by libarchive
|
||||||
|
archive_entry_free(entry);
|
||||||
int returnCode = archive_read_free(ar);
|
int returnCode = archive_read_free(ar);
|
||||||
if (returnCode != ARCHIVE_OK) {
|
if (returnCode != ARCHIVE_OK) {
|
||||||
throw runtime_error("Unable to free archive: " + archiveName);
|
throw runtime_error("Unable to free archive: " + archiveName);
|
||||||
|
|
|
@ -37,8 +37,8 @@ struct LIBPKG_EXPORT ArchiveFile {
|
||||||
|
|
||||||
using FileMap = std::map<std::string, std::vector<ArchiveFile>>;
|
using FileMap = std::map<std::string, std::vector<ArchiveFile>>;
|
||||||
using FilePredicate = std::function<bool(const char *, const char *, mode_t)>;
|
using FilePredicate = std::function<bool(const char *, const char *, mode_t)>;
|
||||||
using DirectoryHandler = std::function<void(std::string &&path)>;
|
using DirectoryHandler = std::function<bool(std::string_view path)>;
|
||||||
using FileHandler = std::function<void(std::string &&path, ArchiveFile &&file)>;
|
using FileHandler = std::function<bool(std::string_view path, ArchiveFile &&file)>;
|
||||||
|
|
||||||
LIBPKG_EXPORT FileMap extractFiles(const std::string &archivePath, const FilePredicate &isFileRelevant = FilePredicate());
|
LIBPKG_EXPORT FileMap extractFiles(const std::string &archivePath, const FilePredicate &isFileRelevant = FilePredicate());
|
||||||
LIBPKG_EXPORT void walkThroughArchive(const std::string &archivePath, const FilePredicate &isFileRelevant = FilePredicate(),
|
LIBPKG_EXPORT void walkThroughArchive(const std::string &archivePath, const FilePredicate &isFileRelevant = FilePredicate(),
|
||||||
|
|
|
@ -80,6 +80,7 @@ void ReloadDatabase::run()
|
||||||
boost::asio::post(
|
boost::asio::post(
|
||||||
m_setup.building.ioContext.get_executor(), [this, force, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
|
m_setup.building.ioContext.get_executor(), [this, force, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
|
||||||
try {
|
try {
|
||||||
|
auto configLock = m_setup.config.lockToRead();
|
||||||
auto dbFileLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(dbName, dbArch));
|
auto dbFileLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(dbName, dbArch));
|
||||||
const auto lastModified = LibPkg::lastModified(dbPath);
|
const auto lastModified = LibPkg::lastModified(dbPath);
|
||||||
if (!force) {
|
if (!force) {
|
||||||
|
@ -93,13 +94,10 @@ void ReloadDatabase::run()
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
|
|
||||||
dbFileLock.lock().unlock();
|
|
||||||
|
|
||||||
m_buildAction->appendOutput(
|
m_buildAction->appendOutput(
|
||||||
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
||||||
|
|
||||||
auto configLock = m_setup.config.lockToRead();
|
|
||||||
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
||||||
if (!destinationDb) {
|
if (!destinationDb) {
|
||||||
configLock.unlock();
|
configLock.unlock();
|
||||||
|
@ -110,13 +108,8 @@ void ReloadDatabase::run()
|
||||||
}
|
}
|
||||||
|
|
||||||
auto updater = LibPkg::PackageUpdater(*destinationDb, true);
|
auto updater = LibPkg::PackageUpdater(*destinationDb, true);
|
||||||
LibPkg::Package::fromDatabaseFile(std::move(dbFile), [&updater](std::shared_ptr<LibPkg::Package> package) {
|
updater.insertFromDatabaseFile(dbPath);
|
||||||
if (const auto [id, existingPackage] = updater.findPackageWithID(package->name); existingPackage) {
|
dbFileLock.lock().unlock();
|
||||||
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
|
|
||||||
}
|
|
||||||
updater.update(package);
|
|
||||||
return false;
|
|
||||||
});
|
|
||||||
updater.commit();
|
updater.commit();
|
||||||
destinationDb->lastUpdate = lastModified;
|
destinationDb->lastUpdate = lastModified;
|
||||||
} catch (const std::runtime_error &e) {
|
} catch (const std::runtime_error &e) {
|
||||||
|
|
|
@ -320,21 +320,23 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents()
|
||||||
|
|
||||||
// extract the binary package's files
|
// extract the binary package's files
|
||||||
try {
|
try {
|
||||||
std::set<std::string> dllsReferencedByImportLibs;
|
auto dllsReferencedByImportLibs = std::set<std::string>();
|
||||||
LibPkg::walkThroughArchive(
|
LibPkg::walkThroughArchive(
|
||||||
currentPkg.path, &LibPkg::Package::isPkgInfoFileOrBinary,
|
currentPkg.path, &LibPkg::Package::isPkgInfoFileOrBinary,
|
||||||
[¤tPkg, &dllsReferencedByImportLibs](std::string &&directoryPath, LibPkg::ArchiveFile &&file) {
|
[¤tPkg, &dllsReferencedByImportLibs](std::string_view directoryPath, LibPkg::ArchiveFile &&file) {
|
||||||
if (directoryPath.empty() && file.name == ".PKGINFO") {
|
if (directoryPath.empty() && file.name == ".PKGINFO") {
|
||||||
currentPkg.info.addInfoFromPkgInfoFile(file.content);
|
currentPkg.info.addInfoFromPkgInfoFile(file.content);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
currentPkg.info.addDepsAndProvidesFromContainedFile(directoryPath, file, dllsReferencedByImportLibs);
|
currentPkg.info.addDepsAndProvidesFromContainedFile(directoryPath, file, dllsReferencedByImportLibs);
|
||||||
|
return false;
|
||||||
},
|
},
|
||||||
[¤tPkg](std::string &&directoryPath) {
|
[¤tPkg](std::string_view directoryPath) {
|
||||||
if (directoryPath.empty()) {
|
if (directoryPath.empty()) {
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
currentPkg.info.addDepsAndProvidesFromContainedDirectory(directoryPath);
|
currentPkg.info.addDepsAndProvidesFromContainedDirectory(directoryPath);
|
||||||
|
return false;
|
||||||
});
|
});
|
||||||
if (auto dllIssues = currentPkg.info.processDllsReferencedByImportLibs(std::move(dllsReferencedByImportLibs)); !dllIssues.empty()) {
|
if (auto dllIssues = currentPkg.info.processDllsReferencedByImportLibs(std::move(dllsReferencedByImportLibs)); !dllIssues.empty()) {
|
||||||
std::unique_lock<std::mutex> submitWarningLock(submitWarningMutex);
|
std::unique_lock<std::mutex> submitWarningLock(submitWarningMutex);
|
||||||
|
|
|
@ -153,10 +153,6 @@ void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQu
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// load packages
|
// load packages
|
||||||
auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant);
|
|
||||||
auto packages = Package::fromDatabaseFile(std::move(files));
|
|
||||||
|
|
||||||
// insert packages
|
|
||||||
auto lock = setup.config.lockToRead();
|
auto lock = setup.config.lockToRead();
|
||||||
auto db = setup.config.findDatabase(dbName, dbArch);
|
auto db = setup.config.findDatabase(dbName, dbArch);
|
||||||
if (!db) {
|
if (!db) {
|
||||||
|
@ -164,7 +160,10 @@ void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQu
|
||||||
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
db->replacePackages(packages, lastModified);
|
auto updater = LibPkg::PackageUpdater(*db, true);
|
||||||
|
updater.insertFromDatabaseFile(session2.destinationFilePath);
|
||||||
|
updater.commit();
|
||||||
|
db->lastUpdate = lastModified;
|
||||||
} catch (const std::runtime_error &e) {
|
} catch (const std::runtime_error &e) {
|
||||||
log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
|
log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
|
||||||
dbQuerySession->addResponse(std::move(dbName));
|
dbQuerySession->addResponse(std::move(dbName));
|
||||||
|
|
Loading…
Reference in New Issue