Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Reduce memory usage when loading packages 

* Avoid creation of map with all archive contents; instead parse packages
  while walking though the archive
* Avoid instantiation of std::string in come cases (using std::string_view)
* Reuse libarchive's archive entry when walking though archive
* Use visitor-based database parser in all places to avoid intermediate big
  array with all package objects
This commit is contained in:
Martchus 2022-04-20 22:47:10 +02:00
parent 10cf6169a6
commit 26caa78956
12 changed files with 119 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
libpkg/data/database.cpp
View File

@ -404,21 +404,6 @@ StorageID Database::forceUpdatePackage(const std::shared_ptr<Package> &package)
return res.id;
}
void Database::replacePackages(const std::vector<std::shared_ptr<Package>> &newPackages, DateTime lastModified)
{
for (const auto &package : newPackages) {
if (const auto existingPackage = findPackage(package->name)) {
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
}
}
auto updater = PackageUpdater(*this, true);
for (const auto &package : newPackages) {
updater.update(package);
}
updater.commit();
lastUpdate = lastModified;
}
/*!
* \brief Determines which packages are unresolvable assuming new packages are added to the database and certain provides are removed.
* \param config The configuration supposed to contain database dependencies.
@ -799,6 +784,18 @@ StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
return res.id;
}
bool PackageUpdater::insertFromDatabaseFile(const std::string &databaseFilePath)
{
LibPkg::Package::fromDatabaseFile(databaseFilePath, [this](const std::shared_ptr<LibPkg::Package> &package) {
if (const auto [id, existingPackage] = findPackageWithID(package->name); existingPackage) {
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
}
update(package);
return false;
});
return false;
}
void PackageUpdater::commit()
{
const auto &storage = m_database.m_storage;

5
libpkg/data/database.h
View File

@ -106,6 +106,7 @@ struct LIBPKG_EXPORT PackageUpdater {
PackageSpec findPackageWithID(const std::string &packageName);
StorageID update(const std::shared_ptr<Package> &package);
bool insertFromDatabaseFile(const std::string &databaseFilePath);
void commit();
private:
@ -147,8 +148,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
void resetConfiguration(bool keepLocalPaths = false);
void clearPackages();
void loadPackagesFromConfiguredPaths(bool withFiles = false, bool force = false);
void loadPackages(const std::string &databaseData, CppUtilities::DateTime lastModified);
void loadPackages(FileMap &&databaseFiles, CppUtilities::DateTime lastModified);
void loadPackages(const std::string &databaseFilePath, CppUtilities::DateTime lastModified);
static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);
std::vector<std::shared_ptr<Package>> findPackages(const std::function<bool(const Database &, const Package &)> &pred);
void allPackages(const PackageVisitorMove &visitor);
@ -164,7 +164,6 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
void removePackage(const std::string &packageName);
StorageID updatePackage(const std::shared_ptr<Package> &package);
StorageID forceUpdatePackage(const std::shared_ptr<Package> &package);
void replacePackages(const std::vector<std::shared_ptr<Package>> &newPackages, CppUtilities::DateTime lastModified);
std::unordered_map<PackageSpec, UnresolvedDependencies> detectUnresolvedPackages(Config &config,
const std::vector<std::shared_ptr<Package>> &newPackages, const DependencySet &removedPackages,
const std::unordered_set<std::string_view> &depsToIgnore = std::unordered_set<std::string_view>(),

7
libpkg/data/package.h
View File

@ -384,9 +384,9 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
std::string computeRegularPackageName() const;
PackageNameData decomposeName() const;
void addInfoFromPkgInfoFile(const std::string &info);
void addDepsAndProvidesFromContainedDirectory(const std::string &directoryPath);
void addDepsAndProvidesFromContainedDirectory(std::string_view directoryPath);
void addDepsAndProvidesFromContainedFile(
const std::string &directoryPath, const ArchiveFile &file, std::set<std::string> &dllsReferencedByImportLibs);
std::string_view directoryPath, const ArchiveFile &file, std::set<std::string> &dllsReferencedByImportLibs);
void addDepsAndProvidesFromContents(const FileMap &contents);
std::vector<std::string> processDllsReferencedByImportLibs(std::set<std::string> &&dllsReferencedByImportLibs);
bool addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bool force = false);
@ -397,8 +397,7 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false);
static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts);
static std::vector<std::shared_ptr<Package>> fromDatabaseFile(FileMap &&databaseFile);
static void fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor);
static void fromDatabaseFile(const std::string &archivePath, const std::function<bool(std::shared_ptr<Package>)> &visitor);
static std::shared_ptr<Package> fromPkgFile(const std::string &path);
static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName);
static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName);

17
libpkg/parser/binary.cpp
View File

@ -11,6 +11,7 @@
#include <filesystem>
#include <fstream>
#include <memory>
#include <string_view>
using namespace std;
using namespace CppUtilities;
@ -199,7 +200,7 @@ void Binary::load(std::string_view filePath)
}
}
void Binary::load(const string &fileContent, const string &fileName, const string &directoryPath, bool isRegularFile)
void Binary::load(std::string_view fileContent, std::string_view fileName, std::string_view directoryPath, bool isRegularFile)
{
stringstream fileStream(ios_base::in | ios_base::out | ios_base::binary);
fileStream.exceptions(ios_base::failbit | ios_base::badbit);
@ -239,17 +240,17 @@ static std::string toLower(std::string str)
return str;
}
std::string Binary::addPrefix(const std::string &dependencyName) const
std::string Binary::addPrefix(std::string_view dependencyName) const
{
switch (type) {
case BinaryType::Elf:
return argsToString(extraPrefix, "elf-", architecture, ':', ':', dependencyName);
case BinaryType::Pe:
return argsToString(extraPrefix, "pe-", architecture, ':', ':', toLower(dependencyName));
return argsToString(extraPrefix, "pe-", architecture, ':', ':', toLower(std::string(dependencyName)));
case BinaryType::Ar:
switch (subType) {
case BinarySubType::WindowsImportLibrary:
return argsToString(extraPrefix, "pe-", architecture, ':', ':', toLower(dependencyName));
return argsToString(extraPrefix, "pe-", architecture, ':', ':', toLower(std::string(dependencyName)));
default:;
}
default:
@ -257,7 +258,7 @@ std::string Binary::addPrefix(const std::string &dependencyName) const
}
}
void Binary::parse(istream &stream, const string *fileContent)
void Binary::parse(std::istream &stream, const std::string_view *fileContent)
{
type = BinaryType::Invalid;
@ -285,7 +286,7 @@ void Binary::parse(istream &stream, const string *fileContent)
}
}
void Binary::parseElf(BinaryReader &reader, const string *fileContent)
void Binary::parseElf(BinaryReader &reader, const std::string_view *fileContent)
{
istream &stream = *reader.stream();
@ -731,8 +732,8 @@ std::uint16_t Binary::readElfInt16(BinaryReader &reader)
return isBigEndian ? reader.readUInt16BE() : reader.readUInt16LE();
}
string Binary::readElfString(
istream &stream, const string *fileContent, std::uint64_t stringTableAddress, std::uint64_t stringTableSize, std::uint64_t relativeStringAddress)
std::string Binary::readElfString(std::istream &stream, const std::string_view *fileContent, std::uint64_t stringTableAddress,
std::uint64_t stringTableSize, std::uint64_t relativeStringAddress)
{
// check bounds
if (relativeStringAddress >= stringTableSize) {

12
libpkg/parser/binary.h
View File

@ -83,8 +83,8 @@ inline VirtualAddressMapping::VirtualAddressMapping()
struct LIBPKG_EXPORT Binary {
void load(std::string_view filePath);
void load(const std::string &fileContent, const std::string &fileName, const std::string &directoryPath, bool isRegularFile = false);
std::string addPrefix(const std::string &dependencyName) const;
void load(std::string_view fileContent, std::string_view fileName, std::string_view directoryPath, bool isRegularFile = false);
std::string addPrefix(std::string_view dependencyName) const;
BinaryType type = BinaryType::Invalid;
BinarySubType subType = BinarySubType::None;
@ -99,8 +99,8 @@ struct LIBPKG_EXPORT Binary {
VirtualAddressMapping virtualAddressMapping;
private:
void parse(std::istream &stream, const std::string *fileContent = nullptr);
void parseElf(CppUtilities::BinaryReader &reader, const std::string *fileContent = nullptr);
void parse(std::istream &stream, const std::string_view *fileContent = nullptr);
void parseElf(CppUtilities::BinaryReader &reader, const std::string_view *fileContent = nullptr);
void parsePe(CppUtilities::BinaryReader &reader, typename std::iostream::off_type baseFileOffset = 0);
void parseAr(CppUtilities::BinaryReader &reader);
void parseCoff(CppUtilities::BinaryReader &reader);
@ -108,8 +108,8 @@ private:
std::uint64_t readElfAddress(CppUtilities::BinaryReader &reader);
std::uint32_t readElfInt32(CppUtilities::BinaryReader &reader);
std::uint16_t readElfInt16(CppUtilities::BinaryReader &reader);
std::string readElfString(std::istream &stream, const std::string *fileContent, std::uint64_t stringTableOffset, std::uint64_t stringTableSize,
std::uint64_t stringOffset);
std::string readElfString(std::istream &stream, const std::string_view *fileContent, std::uint64_t stringTableOffset,
std::uint64_t stringTableSize, std::uint64_t stringOffset);
};
} // namespace LibPkg

26
libpkg/parser/database.cpp
View File

@ -29,32 +29,16 @@ void Database::loadPackagesFromConfiguredPaths(bool withFiles, bool force)
}
const auto lastFileUpdate = lastModified(dbPath);
if (force || lastFileUpdate > lastUpdate) {
loadPackages(extractFiles(dbPath, &isFileRelevant), lastFileUpdate);
loadPackages(dbPath, lastFileUpdate);
}
}
void LibPkg::Database::loadPackages(const string &databaseData, DateTime lastModified)
void Database::loadPackages(const std::string &databaseFilePath, DateTime lastModified)
{
loadPackages(extractFilesFromBuffer(databaseData, name + " db file", &isFileRelevant), lastModified);
}
void Database::loadPackages(FileMap &&databaseFiles, DateTime lastModified)
{
lastUpdate = lastModified;
auto updater = PackageUpdater(*this);
for (auto &dir : databaseFiles) {
if (dir.first.find('/') != std::string::npos) {
cerr << Phrases::WarningMessage << "Database \"" << name << "\" contains unexpected sub directory: " << dir.first << Phrases::EndFlush;
continue;
}
auto descriptionParts = std::vector<std::string>();
descriptionParts.reserve(dir.second.size());
for (auto &file : dir.second) {
descriptionParts.emplace_back(std::move(file.content));
}
updater.update(Package::fromDescription(descriptionParts));
}
auto updater = PackageUpdater(*this, true);
updater.insertFromDatabaseFile(databaseFilePath);
updater.commit();
lastUpdate = lastModified;
}
} // namespace LibPkg

79
libpkg/parser/package.cpp
View File

@ -716,33 +716,40 @@ std::shared_ptr<Package> Package::fromDescription(const std::vector<std::string>
return package;
}
std::vector<std::shared_ptr<Package>> Package::fromDatabaseFile(FileMap &&databaseFile)
void Package::fromDatabaseFile(const std::string &archivePath, const std::function<bool(std::shared_ptr<Package>)> &visitor)
{
std::vector<std::shared_ptr<Package>> packages;
packages.reserve(databaseFile.size());
std::vector<std::string> descriptionParts;
for (auto &dir : databaseFile) {
descriptionParts.clear();
descriptionParts.reserve(dir.second.size());
for (auto &file : dir.second) {
descriptionParts.emplace_back(std::move(file.content));
}
packages.emplace_back(Package::fromDescription(descriptionParts));
}
return packages;
}
// walk though archive, file-by-file; parse files as soon as desc/files available and return via visitor
auto packages = std::unordered_map<std::string, std::vector<std::string>>();
walkThroughArchive(
archivePath, &Database::isFileRelevant,
[&packages, &visitor](std::string_view path, ArchiveFile &&file) {
auto i = packages.find(std::string(path)); // FIXME: this should actually accept std::string_view in C++20
if (i == packages.end()) {
i = packages.emplace(std::string(path), 2).first;
}
auto &parts = i->second;
if (file.name == "desc") {
parts[0] = std::move(file.content);
} else if (file.name == "files") {
parts[1] = std::move(file.content);
}
if (!parts[0].empty() && !parts[1].empty()) {
if (visitor(Package::fromDescription(parts))) {
return true;
}
// remove buffered data immediately to avoid using too much memory
packages.erase(i);
}
return false;
},
[](std::string_view) { return false; });
void Package::fromDatabaseFile(FileMap &&databaseFile, const std::function<bool(std::shared_ptr<Package>)> &visitor)
{
std::vector<std::string> descriptionParts;
for (auto &dir : databaseFile) {
descriptionParts.clear();
descriptionParts.reserve(dir.second.size());
for (auto &file : dir.second) {
descriptionParts.emplace_back(std::move(file.content));
}
if (visitor(Package::fromDescription(descriptionParts))) {
return;
// take care of packages without "files" file
for (auto &[packageName, parts] : packages) {
if (!parts[0].empty()) {
if (visitor(Package::fromDescription(parts))) {
return;
}
}
}
}
@ -768,11 +775,11 @@ void Package::addInfoFromPkgInfoFile(const string &info)
static const regex pythonVersionRegex("usr/lib/python(2|3)\\.([0-9]*)(\\..*)?/site-packages");
static const regex perlVersionRegex("usr/lib/perl5/5\\.([0-9]*)(\\..*)?/vendor_perl");
void Package::addDepsAndProvidesFromContainedDirectory(const std::string &directoryPath)
void Package::addDepsAndProvidesFromContainedDirectory(std::string_view directoryPath)
{
// check for Python modules
thread_local smatch match;
if (regex_match(directoryPath, match, pythonVersionRegex)) {
thread_local auto match = std::match_results<std::string_view::const_iterator>();
if (std::regex_match(directoryPath.begin(), directoryPath.end(), match, pythonVersionRegex)) {
const auto majorVersion = match[1].str();
const auto minorVersion = match[2].str();
const char *const pythonPackage(majorVersion == "3" ? "python" : "python2");
@ -783,7 +790,7 @@ void Package::addDepsAndProvidesFromContainedDirectory(const std::string &direct
}
// check for Perl modules
if (regex_match(directoryPath, match, perlVersionRegex)) {
if (std::regex_match(directoryPath.begin(), directoryPath.end(), match, perlVersionRegex)) {
const auto minorVersion = match[1].str();
auto currentVersion = "5." + minorVersion;
auto nextVersion = "5." + numberToString(stringToNumber<unsigned int>(minorVersion) + 1);
@ -793,7 +800,7 @@ void Package::addDepsAndProvidesFromContainedDirectory(const std::string &direct
}
void Package::addDepsAndProvidesFromContainedFile(
const std::string &directoryPath, const ArchiveFile &file, std::set<std::string> &dllsReferencedByImportLibs)
std::string_view directoryPath, const ArchiveFile &file, std::set<std::string> &dllsReferencedByImportLibs)
{
try {
Binary binary;
@ -858,24 +865,26 @@ std::shared_ptr<Package> Package::fromPkgFile(const string &path)
shared_ptr<Package> package;
LibPkg::walkThroughArchive(
path, &LibPkg::Package::isPkgInfoFileOrBinary,
[&package, &tmpPackageForLibraryDeps, &dllsReferencedByImportLibs](std::string &&directoryPath, LibPkg::ArchiveFile &&file) {
[&package, &tmpPackageForLibraryDeps, &dllsReferencedByImportLibs](std::string_view directoryPath, LibPkg::ArchiveFile &&file) {
if (directoryPath.empty() && file.name == ".PKGINFO") {
if (package) {
return; // only consider one .PKGINFO file (multiple ones are likely not possible in any supported archive formats anyways)
return false; // only consider one .PKGINFO file (multiple ones are likely not possible in any supported archive formats anyways)
}
auto packages = fromInfo(file.content, true);
if (!packages.empty()) {
package = std::move(packages.front().pkg);
}
return;
return false;
}
tmpPackageForLibraryDeps.addDepsAndProvidesFromContainedFile(directoryPath, file, dllsReferencedByImportLibs);
return false;
},
[&tmpPackageForLibraryDeps](std::string &&directoryPath) {
[&tmpPackageForLibraryDeps](std::string_view directoryPath) {
if (directoryPath.empty()) {
return;
return false;
}
tmpPackageForLibraryDeps.addDepsAndProvidesFromContainedDirectory(directoryPath);
return false;
});
if (!package) {
throw runtime_error("Package " % path + " does not contain a valid .PKGINFO");

35
libpkg/parser/utils.cpp
View File

@ -21,17 +21,19 @@ using namespace CppUtilities;
namespace LibPkg {
struct AddDirectoryToFileMap {
void operator()(std::string &&path)
bool operator()(std::string_view path)
{
fileMap[std::move(path)];
fileMap[std::string(path)];
return false;
}
FileMap &fileMap;
};
struct AddFileToFileMap {
void operator()(std::string &&directoryPath, ArchiveFile &&file)
bool operator()(std::string_view directoryPath, ArchiveFile &&file)
{
fileMap[std::move(directoryPath)].emplace_back(std::move(file));
fileMap[std::string(directoryPath)].emplace_back(std::move(file));
return false;
}
FileMap &fileMap;
};
@ -40,8 +42,8 @@ void walkThroughArchiveInternal(struct archive *ar, const string &archiveName, c
DirectoryHandler &&directoryHandler)
{
// iterate through all archive entries
struct archive_entry *entry;
while (archive_read_next_header(ar, &entry) == ARCHIVE_OK) {
struct archive_entry *const entry = archive_entry_new();
while (archive_read_next_header2(ar, entry) == ARCHIVE_OK) {
// check entry type (only dirs, files and symlinks relevant here)
const auto entryType(archive_entry_filetype(entry));
if (entryType != AE_IFDIR && entryType != AE_IFREG && entryType != AE_IFLNK) {
@ -70,7 +72,9 @@ void walkThroughArchiveInternal(struct archive *ar, const string &archiveName, c
}
}
directoryHandler(string(filePath, dirEnd));
if (directoryHandler(std::string_view(filePath, dirEnd))) {
return;
}
continue;
}
@ -94,21 +98,23 @@ void walkThroughArchiveInternal(struct archive *ar, const string &archiveName, c
// read symlink
if (entryType == AE_IFLNK) {
fileHandler(string(filePath, static_cast<string::size_type>(dirEnd - filePath)),
ArchiveFile(fileName, string(archive_entry_symlink_utf8(entry)), ArchiveFileType::Link, creationTime, modificationTime));
if (fileHandler(std::string_view(filePath, static_cast<string::size_type>(dirEnd - filePath)),
ArchiveFile(fileName, std::string(archive_entry_symlink_utf8(entry)), ArchiveFileType::Link, creationTime, modificationTime))) {
return;
}
continue;
}
// determine file size to pre-allocate buffer for file content
const la_int64_t fileSize = archive_entry_size(entry);
string fileContent;
std::string fileContent;
if (fileSize > 0) {
fileContent.reserve(static_cast<string::size_type>(fileSize));
}
// read file content
const char *buff;
size_t size;
std::size_t size;
la_int64_t offset;
for (;;) {
int returnCode = archive_read_data_block(ar, reinterpret_cast<const void **>(&buff), &size, &offset);
@ -119,11 +125,14 @@ void walkThroughArchiveInternal(struct archive *ar, const string &archiveName, c
}
// move it to results
fileHandler(string(filePath, static_cast<string::size_type>(dirEnd - filePath)),
ArchiveFile(fileName, move(fileContent), ArchiveFileType::Regular, creationTime, modificationTime));
if (fileHandler(std::string_view(filePath, static_cast<string::size_type>(dirEnd - filePath)),
ArchiveFile(fileName, std::move(fileContent), ArchiveFileType::Regular, creationTime, modificationTime))) {
return;
}
}
// free resources used by libarchive
archive_entry_free(entry);
int returnCode = archive_read_free(ar);
if (returnCode != ARCHIVE_OK) {
throw runtime_error("Unable to free archive: " + archiveName);

4
libpkg/parser/utils.h
View File

@ -37,8 +37,8 @@ struct LIBPKG_EXPORT ArchiveFile {
using FileMap = std::map<std::string, std::vector<ArchiveFile>>;
using FilePredicate = std::function<bool(const char *, const char *, mode_t)>;
using DirectoryHandler = std::function<void(std::string &&path)>;
using FileHandler = std::function<void(std::string &&path, ArchiveFile &&file)>;
using DirectoryHandler = std::function<bool(std::string_view path)>;
using FileHandler = std::function<bool(std::string_view path, ArchiveFile &&file)>;
LIBPKG_EXPORT FileMap extractFiles(const std::string &archivePath, const FilePredicate &isFileRelevant = FilePredicate());
LIBPKG_EXPORT void walkThroughArchive(const std::string &archivePath, const FilePredicate &isFileRelevant = FilePredicate(),

13
librepomgr/buildactions/reloaddatabase.cpp
View File

@ -80,6 +80,7 @@ void ReloadDatabase::run()
boost::asio::post(
m_setup.building.ioContext.get_executor(), [this, force, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
try {
auto configLock = m_setup.config.lockToRead();
auto dbFileLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(dbName, dbArch));
const auto lastModified = LibPkg::lastModified(dbPath);
if (!force) {
@ -93,13 +94,10 @@ void ReloadDatabase::run()
return;
}
}
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
dbFileLock.lock().unlock();
m_buildAction->appendOutput(
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
auto configLock = m_setup.config.lockToRead();
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
if (!destinationDb) {
configLock.unlock();
@ -110,13 +108,8 @@ void ReloadDatabase::run()
}
auto updater = LibPkg::PackageUpdater(*destinationDb, true);
LibPkg::Package::fromDatabaseFile(std::move(dbFile), [&updater](std::shared_ptr<LibPkg::Package> package) {
if (const auto [id, existingPackage] = updater.findPackageWithID(package->name); existingPackage) {
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
}
updater.update(package);
return false;
});
updater.insertFromDatabaseFile(dbPath);
dbFileLock.lock().unlock();
updater.commit();
destinationDb->lastUpdate = lastModified;
} catch (const std::runtime_error &e) {

12
librepomgr/buildactions/reloadlibrarydependencies.cpp
View File

@ -320,21 +320,23 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents()
// extract the binary package's files
try {
std::set<std::string> dllsReferencedByImportLibs;
auto dllsReferencedByImportLibs = std::set<std::string>();
LibPkg::walkThroughArchive(
currentPkg.path, &LibPkg::Package::isPkgInfoFileOrBinary,
[&currentPkg, &dllsReferencedByImportLibs](std::string &&directoryPath, LibPkg::ArchiveFile &&file) {
[&currentPkg, &dllsReferencedByImportLibs](std::string_view directoryPath, LibPkg::ArchiveFile &&file) {
if (directoryPath.empty() && file.name == ".PKGINFO") {
currentPkg.info.addInfoFromPkgInfoFile(file.content);
return;
return false;
}
currentPkg.info.addDepsAndProvidesFromContainedFile(directoryPath, file, dllsReferencedByImportLibs);
return false;
},
[&currentPkg](std::string &&directoryPath) {
[&currentPkg](std::string_view directoryPath) {
if (directoryPath.empty()) {
return;
return false;
}
currentPkg.info.addDepsAndProvidesFromContainedDirectory(directoryPath);
return false;
});
if (auto dllIssues = currentPkg.info.processDllsReferencedByImportLibs(std::move(dllsReferencedByImportLibs)); !dllIssues.empty()) {
std::unique_lock<std::mutex> submitWarningLock(submitWarningMutex);

9
librepomgr/webclient/database.cpp
View File

@ -153,10 +153,6 @@ void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQu
try {
// load packages
auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant);
auto packages = Package::fromDatabaseFile(std::move(files));
// insert packages
auto lock = setup.config.lockToRead();
auto db = setup.config.findDatabase(dbName, dbArch);
if (!db) {
@ -164,7 +160,10 @@ void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQu
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
return;
}
db->replacePackages(packages, lastModified);
auto updater = LibPkg::PackageUpdater(*db, true);
updater.insertFromDatabaseFile(session2.destinationFilePath);
updater.commit();
db->lastUpdate = lastModified;
} catch (const std::runtime_error &e) {
log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
dbQuerySession->addResponse(std::move(dbName));