Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

lmdb: Use package cache when querying packages for dependencies 

* Use package cache when querying packages for dependencies or library
  names
* Avoid manual instantiations of `std::shared_ptr<Package>`
This commit is contained in:
Martchus 2022-01-21 20:35:43 +01:00
parent fcdd4fa7b3
commit f088e54c61
13 changed files with 111 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
libpkg/algo/search.cpp
View File

@ -128,11 +128,10 @@ PackageSearchResult Config::findPackage(const Dependency &dependency)
auto result = PackageSearchResult(); auto result = PackageSearchResult();
auto exactMatch = false; auto exactMatch = false;
for (auto &db : databases) { for (auto &db : databases) {
db.providingPackages(dependency, false, [&](StorageID id, Package &&package) { db.providingPackages(dependency, false, [&](StorageID id, const std::shared_ptr<Package> &package) {
// FIXME: avoid copy exactMatch = dependency.name == package->name;
exactMatch = dependency.name == package.name;
result.db = &db; result.db = &db;
result.pkg = std::make_shared<Package>(std::move(package)); result.pkg = package;
result.id = id; result.id = id;
// prefer package where the name matches exactly; so if we found one no need to look further // prefer package where the name matches exactly; so if we found one no need to look further
return exactMatch; return exactMatch;
@ -152,10 +151,9 @@ std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependen
auto results = std::vector<PackageSearchResult>(); auto results = std::vector<PackageSearchResult>();
for (auto &db : databases) { for (auto &db : databases) {
auto visited = std::unordered_set<StorageID>(); auto visited = std::unordered_set<StorageID>();
db.providingPackages(dependency, reverse, [&](StorageID packageID, Package &&package) { db.providingPackages(dependency, reverse, [&](StorageID packageID, const std::shared_ptr<Package> &package) {
// FIXME: avoid copy
if (visited.emplace(packageID).second) { if (visited.emplace(packageID).second) {
results.emplace_back(db, std::make_shared<Package>(std::move(package)), packageID); results.emplace_back(db, package, packageID);
} }
return false; return false;
}); });
@ -171,10 +169,9 @@ std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std:
auto results = std::vector<PackageSearchResult>(); auto results = std::vector<PackageSearchResult>();
auto visited = std::unordered_set<StorageID>(); auto visited = std::unordered_set<StorageID>();
for (auto &db : databases) { for (auto &db : databases) {
db.providingPackages(library, reverse, [&](StorageID packageID, Package &&package) { db.providingPackages(library, reverse, [&](StorageID packageID, const std::shared_ptr<Package> &package) {
// FIXME: avoid copy
if (visited.emplace(packageID).second) { if (visited.emplace(packageID).second) {
results.emplace_back(db, std::make_shared<Package>(std::move(package)), packageID); results.emplace_back(db, package, packageID);
} }
return false; return false;
}); });
@ -189,10 +186,9 @@ std::vector<PackageSearchResult> Config::findPackages(const std::regex &regex)
{ {
auto pkgs = std::vector<PackageSearchResult>(); auto pkgs = std::vector<PackageSearchResult>();
for (auto &db : databases) { for (auto &db : databases) {
db.allPackages([&](StorageID packageID, Package &&package) { db.allPackages([&](StorageID packageID, const std::shared_ptr<Package> &package) {
if (std::regex_match(package.name, regex)) { if (std::regex_match(package->name, regex)) {
// FIXME: avoid copy pkgs.emplace_back(db, package, packageID);
pkgs.emplace_back(db, std::make_shared<Package>(std::move(package)), packageID);
} }
return false; return false;
}); });
@ -226,10 +222,9 @@ std::vector<PackageSearchResult> Config::findPackages(
if (!databasePred(db)) { if (!databasePred(db)) {
continue; continue;
} }
db.allPackages([&](StorageID packageID, const Package &package) { db.allPackages([&](StorageID packageID, const std::shared_ptr<Package> &package) {
if (packagePred(db, package)) { if (packagePred(db, *package)) {
// FIXME: avoid copy pkgs.emplace_back(db, package, packageID);
pkgs.emplace_back(db, std::make_shared<Package>(package), packageID);
} }
return false; return false;
}); });
@ -244,10 +239,9 @@ std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(c
{ {
auto pkgs = std::vector<PackageSearchResult>(); auto pkgs = std::vector<PackageSearchResult>();
for (auto &db : databases) { for (auto &db : databases) {
db.allPackages([&](StorageID packageID, const Package &package) { db.allPackages([&](StorageID packageID, const std::shared_ptr<Package> &package) {
if (pred(db, package)) { if (pred(db, *package)) {
// FIXME: avoid copy pkgs.emplace_back(db, package, packageID);
pkgs.emplace_back(db, std::make_shared<Package>(package), packageID);
} }
return false; return false;
}); });

5
libpkg/data/config.cpp
View File

@ -151,13 +151,12 @@ void Config::pullDependentPackages(const std::vector<Dependency> &dependencies,
if (relevantDbs.find(&db) == relevantDbs.end()) { if (relevantDbs.find(&db) == relevantDbs.end()) {
continue; continue;
} }
db.providingPackages(dependency, false, [&](StorageID packageID, Package &&package) { db.providingPackages(dependency, false, [&](StorageID packageID, const std::shared_ptr<Package> &package) {
found = true; found = true;
// FIXME: avoid copy
if (visited.emplace(packageID).second) { if (visited.emplace(packageID).second) {
const auto &[i, inserted] = runtimeDependencies.try_emplace(packageID); const auto &[i, inserted] = runtimeDependencies.try_emplace(packageID);
if (inserted) { if (inserted) {
i->second = std::make_shared<Package>(std::move(package)); i->second = package;
} }
pullDependentPackages(i->second, relevantDbs, runtimeDependencies, missingDependencies, visited); pullDependentPackages(i->second, relevantDbs, runtimeDependencies, missingDependencies, visited);
} }

32
libpkg/data/database.cpp
View File

@ -265,11 +265,10 @@ void Database::addPackageDependencies(StorageID packageID, const std::shared_ptr
void Database::allPackages(const PackageVisitor &visitor) void Database::allPackages(const PackageVisitor &visitor)
{ {
// TODO: use cache here // TODO: use cache here, e.g. implement a "lazy" iterator for LMDB that also directly yields a std::shared_ptr
//auto &cachedPackages = m_storage->packageCache;
auto txn = m_storage->packages.getROTransaction(); auto txn = m_storage->packages.getROTransaction();
for (auto i = txn.begin(); i != txn.end(); ++i) { for (auto i = txn.begin(); i != txn.end(); ++i) {
if (visitor(i.getID(), std::move(i.value()))) { if (visitor(i.getID(), std::make_shared<Package>(std::move(i.value())))) {
return; return;
} }
} }
@ -282,8 +281,6 @@ std::size_t Database::packageCount() const
void Database::providingPackages(const Dependency &dependency, bool reverse, const PackageVisitor &visitor) void Database::providingPackages(const Dependency &dependency, bool reverse, const PackageVisitor &visitor)
{ {
// TODO: use cache here
auto package = Package();
auto providesTxn = (reverse ? m_storage->requiredDeps : m_storage->providedDeps).getROTransaction(); auto providesTxn = (reverse ? m_storage->requiredDeps : m_storage->providedDeps).getROTransaction();
auto packagesTxn = m_storage->packages.getROTransaction(); auto packagesTxn = m_storage->packages.getROTransaction();
for (auto [i, end] = providesTxn.equal_range<0>(dependency.name); i != end; ++i) { for (auto [i, end] = providesTxn.equal_range<0>(dependency.name); i != end; ++i) {
@ -292,7 +289,8 @@ void Database::providingPackages(const Dependency &dependency, bool reverse, con
continue; continue;
} }
for (const auto packageID : i->relevantPackages) { for (const auto packageID : i->relevantPackages) {
if (packagesTxn.get(packageID, package) && visitor(packageID, std::move(package))) { const auto res = m_storage->packageCache.retrieve(*m_storage, &packagesTxn, packageID);
if (res.pkg && visitor(packageID, res.pkg)) {
return; return;
} }
} }
@ -301,13 +299,12 @@ void Database::providingPackages(const Dependency &dependency, bool reverse, con
void Database::providingPackages(const std::string &libraryName, bool reverse, const PackageVisitor &visitor) void Database::providingPackages(const std::string &libraryName, bool reverse, const PackageVisitor &visitor)
{ {
// TODO: use cache here
auto package = Package();
auto providesTxn = (reverse ? m_storage->requiredLibs : m_storage->providedLibs).getROTransaction(); auto providesTxn = (reverse ? m_storage->requiredLibs : m_storage->providedLibs).getROTransaction();
auto packagesTxn = m_storage->packages.getROTransaction(); auto packagesTxn = m_storage->packages.getROTransaction();
for (auto [i, end] = providesTxn.equal_range<0>(libraryName); i != end; ++i) { for (auto [i, end] = providesTxn.equal_range<0>(libraryName); i != end; ++i) {
for (const auto packageID : i->relevantPackages) { for (const auto packageID : i->relevantPackages) {
if (packagesTxn.get(packageID, package) && visitor(packageID, std::move(package))) { const auto res = m_storage->packageCache.retrieve(*m_storage, &packagesTxn, packageID);
if (res.pkg && visitor(packageID, res.pkg)) {
return; return;
} }
} }
@ -506,23 +503,22 @@ std::unordered_map<PackageSpec, UnresolvedDependencies> Database::detectUnresolv
LibPkg::PackageUpdates LibPkg::Database::checkForUpdates(const std::vector<LibPkg::Database *> &updateSources, UpdateCheckOptions options) LibPkg::PackageUpdates LibPkg::Database::checkForUpdates(const std::vector<LibPkg::Database *> &updateSources, UpdateCheckOptions options)
{ {
auto results = PackageUpdates(); auto results = PackageUpdates();
allPackages([&](StorageID myPackageID, Package &&package) { allPackages([&](StorageID myPackageID, const std::shared_ptr<Package> &package) {
auto myPackage = std::make_shared<Package>(std::move(package));
auto regularName = std::string(); auto regularName = std::string();
if (options & UpdateCheckOptions::ConsiderRegularPackage) { if (options & UpdateCheckOptions::ConsiderRegularPackage) {
const auto decomposedName = myPackage->decomposeName(); const auto decomposedName = package->decomposeName();
if ((!decomposedName.targetPrefix.empty() || !decomposedName.vcsSuffix.empty()) && !decomposedName.isVcsPackage()) { if ((!decomposedName.targetPrefix.empty() || !decomposedName.vcsSuffix.empty()) && !decomposedName.isVcsPackage()) {
regularName = decomposedName.actualName; regularName = decomposedName.actualName;
} }
} }
auto foundPackage = false; auto foundPackage = false;
for (auto *const updateSource : updateSources) { for (auto *const updateSource : updateSources) {
const auto [updatePackageID, updatePackage] = updateSource->findPackageWithID(myPackage->name); const auto [updatePackageID, updatePackage] = updateSource->findPackageWithID(package->name);
if (!updatePackage) { if (!updatePackage) {
continue; continue;
} }
foundPackage = true; foundPackage = true;
const auto versionDiff = myPackage->compareVersion(*updatePackage); const auto versionDiff = package->compareVersion(*updatePackage);
std::vector<PackageUpdate> *list = nullptr; std::vector<PackageUpdate> *list = nullptr;
switch (versionDiff) { switch (versionDiff) {
case PackageVersionComparison::SoftwareUpgrade: case PackageVersionComparison::SoftwareUpgrade:
@ -538,11 +534,11 @@ LibPkg::PackageUpdates LibPkg::Database::checkForUpdates(const std::vector<LibPk
} }
if (list) { if (list) {
list->emplace_back( list->emplace_back(
PackageSearchResult(*this, myPackage, myPackageID), PackageSearchResult(*updateSource, updatePackage, updatePackageID)); PackageSearchResult(*this, package, myPackageID), PackageSearchResult(*updateSource, updatePackage, updatePackageID));
} }
} }
if (!foundPackage) { if (!foundPackage) {
results.orphans.emplace_back(PackageSearchResult(*this, myPackage, myPackageID)); results.orphans.emplace_back(PackageSearchResult(*this, package, myPackageID));
} }
if (regularName.empty()) { if (regularName.empty()) {
return false; return false;
@ -552,7 +548,7 @@ LibPkg::PackageUpdates LibPkg::Database::checkForUpdates(const std::vector<LibPk
if (!updatePackage) { if (!updatePackage) {
continue; continue;
} }
const auto versionDiff = myPackage->compareVersion(*updatePackage); const auto versionDiff = package->compareVersion(*updatePackage);
std::vector<PackageUpdate> *list = nullptr; std::vector<PackageUpdate> *list = nullptr;
switch (versionDiff) { switch (versionDiff) {
case PackageVersionComparison::SoftwareUpgrade: case PackageVersionComparison::SoftwareUpgrade:
@ -568,7 +564,7 @@ LibPkg::PackageUpdates LibPkg::Database::checkForUpdates(const std::vector<LibPk
} }
if (list) { if (list) {
list->emplace_back( list->emplace_back(
PackageSearchResult(*this, myPackage, myPackageID), PackageSearchResult(*updateSource, updatePackage, updatePackageID)); PackageSearchResult(*this, package, myPackageID), PackageSearchResult(*updateSource, updatePackage, updatePackageID));
} }
} }
return false; return false;

2
libpkg/data/database.h
View File

@ -111,7 +111,7 @@ private:
}; };
struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Database>, public ReflectiveRapidJSON::BinarySerializable<Database> { struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Database>, public ReflectiveRapidJSON::BinarySerializable<Database> {
using PackageVisitor = std::function<bool(StorageID, Package &&)>; using PackageVisitor = std::function<bool(StorageID, const std::shared_ptr<Package> &)>;
friend struct PackageUpdater; friend struct PackageUpdater;

41
libpkg/data/storage.cpp
View File

@ -43,8 +43,8 @@ template <typename StorageEntryType> std::size_t StorageCacheEntries<StorageEntr
return count; return count;
} }
template <typename StorageEntriesType, typename TransactionType, typename SpecType> template <typename StorageEntriesType, typename StorageType, typename SpecType>
auto StorageCache<StorageEntriesType, TransactionType, SpecType>::retrieve(Storage &storage, StorageID storageID) -> SpecType auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, ROTxn *txn, StorageID storageID) -> SpecType
{ {
// check for package in cache // check for package in cache
const auto ref = typename StorageEntryByID<typename Entries::StorageEntry>::result_type{ storageID, &storage }; const auto ref = typename StorageEntryByID<typename Entries::StorageEntry>::result_type{ storageID, &storage };
@ -55,8 +55,7 @@ auto StorageCache<StorageEntriesType, TransactionType, SpecType>::retrieve(Stora
// check for package in storage, populate cache entry // check for package in storage, populate cache entry
lock.unlock(); lock.unlock();
auto entry = std::make_shared<Entry>(); auto entry = std::make_shared<Entry>();
auto txn = storage.packages.getROTransaction(); if (auto id = txn ? txn->get(storageID, *entry) : storage.packages.getROTransaction().get(storageID, *entry)) {
if (auto id = txn.get(storageID, *entry)) {
using CacheEntry = typename Entries::StorageEntry; using CacheEntry = typename Entries::StorageEntry;
using CacheRef = typename Entries::Ref; using CacheRef = typename Entries::Ref;
auto newCacheEntry = CacheEntry(CacheRef(storage, entry), id); auto newCacheEntry = CacheEntry(CacheRef(storage, entry), id);
@ -69,8 +68,14 @@ auto StorageCache<StorageEntriesType, TransactionType, SpecType>::retrieve(Stora
return SpecType(0, std::shared_ptr<Entry>()); return SpecType(0, std::shared_ptr<Entry>());
} }
template <typename StorageEntriesType, typename TransactionType, typename SpecType> template <typename StorageEntriesType, typename StorageType, typename SpecType>
auto StorageCache<StorageEntriesType, TransactionType, SpecType>::retrieve(Storage &storage, const std::string &entryName) -> SpecType auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, StorageID storageID) -> SpecType
{
return retrieve(storage, nullptr, storageID);
}
template <typename StorageEntriesType, typename StorageType, typename SpecType>
auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, const std::string &entryName) -> SpecType
{ {
// check for package in cache // check for package in cache
using CacheRef = typename Entries::Ref; using CacheRef = typename Entries::Ref;
@ -95,9 +100,8 @@ auto StorageCache<StorageEntriesType, TransactionType, SpecType>::retrieve(Stora
return SpecType(0, std::shared_ptr<Entry>()); return SpecType(0, std::shared_ptr<Entry>());
} }
template <typename StorageEntriesType, typename TransactionType, typename SpecType> template <typename StorageEntriesType, typename StorageType, typename SpecType>
auto StorageCache<StorageEntriesType, TransactionType, SpecType>::store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force) auto StorageCache<StorageEntriesType, StorageType, SpecType>::store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force) -> StoreResult
-> StoreResult
{ {
// check for package in cache // check for package in cache
using CacheEntry = typename Entries::StorageEntry; using CacheEntry = typename Entries::StorageEntry;
@ -144,9 +148,8 @@ auto StorageCache<StorageEntriesType, TransactionType, SpecType>::store(Storage
return res; return res;
} }
template <typename StorageEntriesType, typename TransactionType, typename SpecType> template <typename StorageEntriesType, typename StorageType, typename SpecType>
auto StorageCache<StorageEntriesType, TransactionType, SpecType>::store(Storage &storage, Txn &txn, const std::shared_ptr<Entry> &entry) auto StorageCache<StorageEntriesType, StorageType, SpecType>::store(Storage &storage, RWTxn &txn, const std::shared_ptr<Entry> &entry) -> StoreResult
-> StoreResult
{ {
// check for package in cache // check for package in cache
using CacheEntry = typename Entries::StorageEntry; using CacheEntry = typename Entries::StorageEntry;
@ -185,8 +188,8 @@ auto StorageCache<StorageEntriesType, TransactionType, SpecType>::store(Storage
return res; return res;
} }
template <typename StorageEntriesType, typename TransactionType, typename SpecType> template <typename StorageEntriesType, typename StorageType, typename SpecType>
bool StorageCache<StorageEntriesType, TransactionType, SpecType>::invalidate(Storage &storage, const std::string &entryName) bool StorageCache<StorageEntriesType, StorageType, SpecType>::invalidate(Storage &storage, const std::string &entryName)
{ {
// remove package from cache // remove package from cache
const auto ref = typename Entries::Ref(storage, entryName); const auto ref = typename Entries::Ref(storage, entryName);
@ -203,8 +206,8 @@ bool StorageCache<StorageEntriesType, TransactionType, SpecType>::invalidate(Sto
return false; return false;
} }
template <typename StorageEntriesType, typename TransactionType, typename SpecType> template <typename StorageEntriesType, typename StorageType, typename SpecType>
void StorageCache<StorageEntriesType, TransactionType, SpecType>::clear(Storage &storage) void StorageCache<StorageEntriesType, StorageType, SpecType>::clear(Storage &storage)
{ {
clearCacheOnly(storage); clearCacheOnly(storage);
auto packagesTxn = storage.packages.getRWTransaction(); auto packagesTxn = storage.packages.getRWTransaction();
@ -224,8 +227,8 @@ void StorageCache<StorageEntriesType, TransactionType, SpecType>::clear(Storage
requiredLibsTxn.commit(); requiredLibsTxn.commit();
} }
template <typename StorageEntriesType, typename TransactionType, typename SpecType> template <typename StorageEntriesType, typename StorageType, typename SpecType>
void StorageCache<StorageEntriesType, TransactionType, SpecType>::clearCacheOnly(Storage &storage) void StorageCache<StorageEntriesType, StorageType, SpecType>::clearCacheOnly(Storage &storage)
{ {
const auto lock = std::unique_lock(m_mutex); const auto lock = std::unique_lock(m_mutex);
m_entries.clear(storage); m_entries.clear(storage);
@ -234,7 +237,7 @@ void StorageCache<StorageEntriesType, TransactionType, SpecType>::clearCacheOnly
template struct StorageCacheRef<DatabaseStorage, Package>; template struct StorageCacheRef<DatabaseStorage, Package>;
template struct StorageCacheEntry<PackageCacheRef, Package>; template struct StorageCacheEntry<PackageCacheRef, Package>;
template class StorageCacheEntries<PackageCacheEntry>; template class StorageCacheEntries<PackageCacheEntry>;
template struct StorageCache<PackageCacheEntries, PackageStorage::RWTransaction, PackageSpec>; template struct StorageCache<PackageCacheEntries, PackageStorage, PackageSpec>;
StorageDistribution::StorageDistribution(const char *path, std::uint32_t maxDbs) StorageDistribution::StorageDistribution(const char *path, std::uint32_t maxDbs)
{ {

12
libpkg/data/storageprivate.h
View File

@ -136,10 +136,11 @@ template <typename StorageEntryType> inline auto StorageCacheEntries<StorageEntr
return m_entries.end(); return m_entries.end();
} }
template <typename StorageEntriesType, typename TransactionType, typename SpecType> struct StorageCache { template <typename StorageEntriesType, typename StorageType, typename SpecType> struct StorageCache {
using Entries = StorageEntriesType; using Entries = StorageEntriesType;
using Entry = typename Entries::Entry; using Entry = typename Entries::Entry;
using Txn = TransactionType; using ROTxn = typename StorageType::ROTransaction;
using RWTxn = typename StorageType::RWTransaction;
using Storage = typename Entries::Storage; using Storage = typename Entries::Storage;
struct StoreResult { struct StoreResult {
StorageID id = 0; StorageID id = 0;
@ -147,10 +148,11 @@ template <typename StorageEntriesType, typename TransactionType, typename SpecTy
std::shared_ptr<typename Entries::Entry> oldEntry; std::shared_ptr<typename Entries::Entry> oldEntry;
}; };
SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID);
SpecType retrieve(Storage &storage, StorageID storageID); SpecType retrieve(Storage &storage, StorageID storageID);
SpecType retrieve(Storage &storage, const std::string &entryName); SpecType retrieve(Storage &storage, const std::string &entryName);
StoreResult store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force); StoreResult store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force);
StoreResult store(Storage &storage, Txn &txn, const std::shared_ptr<Entry> &entry); StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr<Entry> &entry);
bool invalidate(Storage &storage, const std::string &entryName); bool invalidate(Storage &storage, const std::string &entryName);
void clear(Storage &storage); void clear(Storage &storage);
void clearCacheOnly(Storage &storage); void clearCacheOnly(Storage &storage);
@ -168,12 +170,12 @@ using PackageCacheRef = StorageCacheRef<DatabaseStorage, Package>;
using PackageCacheEntry = StorageCacheEntry<PackageCacheRef, Package>; using PackageCacheEntry = StorageCacheEntry<PackageCacheRef, Package>;
using PackageCacheEntries = StorageCacheEntries<PackageCacheEntry>; using PackageCacheEntries = StorageCacheEntries<PackageCacheEntry>;
using PackageCacheEntryByID = typename PackageCacheEntries::ByID::result_type; using PackageCacheEntryByID = typename PackageCacheEntries::ByID::result_type;
using PackageCache = StorageCache<PackageCacheEntries, PackageStorage::RWTransaction, PackageSpec>; using PackageCache = StorageCache<PackageCacheEntries, PackageStorage, PackageSpec>;
extern template struct StorageCacheRef<DatabaseStorage, Package>; extern template struct StorageCacheRef<DatabaseStorage, Package>;
extern template struct StorageCacheEntry<PackageCacheRef, Package>; extern template struct StorageCacheEntry<PackageCacheRef, Package>;
extern template class StorageCacheEntries<PackageCacheEntry>; extern template class StorageCacheEntries<PackageCacheEntry>;
extern template struct StorageCache<PackageCacheEntries, PackageStorage::RWTransaction, PackageSpec>; extern template struct StorageCache<PackageCacheEntries, PackageStorage, PackageSpec>;
struct StorageDistribution { struct StorageDistribution {
explicit StorageDistribution(const char *path, std::uint32_t maxDbs); explicit StorageDistribution(const char *path, std::uint32_t maxDbs);

16
librepomgr/buildactions/conductbuild.cpp
View File

@ -1520,31 +1520,31 @@ PackageStagingNeeded ConductBuild::checkWhetherStagingIsNeededAndPopulateRebuild
const auto &removedDependencyName = removedDependency.first; const auto &removedDependencyName = removedDependency.first;
const auto &removedDependencyDetail = removedDependency.second; const auto &removedDependencyDetail = removedDependency.second;
db->providingPackages(LibPkg::Dependency(removedDependencyName, removedDependencyDetail.version, removedDependencyDetail.mode), true, db->providingPackages(LibPkg::Dependency(removedDependencyName, removedDependencyDetail.version, removedDependencyDetail.mode), true,
[&](LibPkg::StorageID, LibPkg::Package &&affectedPackage) { [&](LibPkg::StorageID, const std::shared_ptr<LibPkg::Package> &affectedPackage) {
if (isDestinationDb && isPackageWeWantToUpdateItself(affectedPackage)) { if (isDestinationDb && isPackageWeWantToUpdateItself(*affectedPackage)) {
return false; // skip if that's just the package we want to update itself return false; // skip if that's just the package we want to update itself
} }
if (!rebuildInfoForDb) { if (!rebuildInfoForDb) {
rebuildInfoForDb = &m_buildProgress.rebuildList[db->name]; rebuildInfoForDb = &m_buildProgress.rebuildList[db->name];
} }
needsStaging = true; needsStaging = true;
(*rebuildInfoForDb)[affectedPackage.name].provides.emplace_back( (*rebuildInfoForDb)[affectedPackage->name].provides.emplace_back(
removedDependencyName, removedDependencyDetail.version, removedDependencyDetail.mode); removedDependencyName, removedDependencyDetail.version, removedDependencyDetail.mode);
listOfAffectedPackages.emplace_back(db->name % '/' + affectedPackage.name); listOfAffectedPackages.emplace_back(db->name % '/' + affectedPackage->name);
return false; return false;
}); });
} }
for (const auto &removedLibProvide : removedLibProvides) { for (const auto &removedLibProvide : removedLibProvides) {
db->providingPackages(removedLibProvide, true, [&](LibPkg::StorageID, LibPkg::Package &&affectedPackage) { db->providingPackages(removedLibProvide, true, [&](LibPkg::StorageID, const std::shared_ptr<LibPkg::Package> &affectedPackage) {
if (isDestinationDb && isPackageWeWantToUpdateItself(affectedPackage)) { if (isDestinationDb && isPackageWeWantToUpdateItself(*affectedPackage)) {
return false; // skip if that's just the package we want to update itself return false; // skip if that's just the package we want to update itself
} }
if (!rebuildInfoForDb) { if (!rebuildInfoForDb) {
rebuildInfoForDb = &m_buildProgress.rebuildList[db->name]; rebuildInfoForDb = &m_buildProgress.rebuildList[db->name];
} }
needsStaging = true; needsStaging = true;
(*rebuildInfoForDb)[affectedPackage.name].libprovides.emplace_back(removedLibProvide); (*rebuildInfoForDb)[affectedPackage->name].libprovides.emplace_back(removedLibProvide);
listOfAffectedPackages.emplace_back(db->name % '/' + affectedPackage.name); listOfAffectedPackages.emplace_back(db->name % '/' + affectedPackage->name);
return false; return false;
}); });
} }

31
librepomgr/buildactions/reloadlibrarydependencies.cpp
View File

@ -85,9 +85,8 @@ void ReloadLibraryDependencies::run()
} }
} }
for (auto *const destinationDb : m_destinationDbs) { for (auto *const destinationDb : m_destinationDbs) {
destinationDb->allPackages([&, this](LibPkg::StorageID, LibPkg::Package &&package) { destinationDb->allPackages([&, this](LibPkg::StorageID, const std::shared_ptr<LibPkg::Package> &package) {
m_setup.config.pullDependentPackages( m_setup.config.pullDependentPackages(package, relevantDbs, relevantPkgs, missingDeps, visitedPackages);
std::make_shared<LibPkg::Package>(std::move(package)), relevantDbs, relevantPkgs, missingDeps, visitedPackages);
return false; return false;
}); });
} }
@ -104,20 +103,20 @@ void ReloadLibraryDependencies::run()
for (auto *const db : relevantDbs) { for (auto *const db : relevantDbs) {
const auto isDestinationDb = m_destinationDbs.empty() || m_destinationDbs.find(db) != m_destinationDbs.end(); const auto isDestinationDb = m_destinationDbs.empty() || m_destinationDbs.find(db) != m_destinationDbs.end();
auto &relevantDbInfo = m_relevantPackagesByDatabase.emplace_back(DatabaseToConsider{ .name = db->name, .arch = db->arch }); auto &relevantDbInfo = m_relevantPackagesByDatabase.emplace_back(DatabaseToConsider{ .name = db->name, .arch = db->arch });
db->allPackages([&](LibPkg::StorageID packageID, LibPkg::Package &&package) { db->allPackages([&](LibPkg::StorageID packageID, const std::shared_ptr<LibPkg::Package> &package) {
// allow aborting the build action // allow aborting the build action
if (reportAbortedIfAborted()) { if (reportAbortedIfAborted()) {
return true; return true;
} }
// skip if package should be excluded // skip if package should be excluded
if (!packageExcludeRegexValue.empty() && std::regex_match(package.name, packageExcludeRegex)) { if (!packageExcludeRegexValue.empty() && std::regex_match(package->name, packageExcludeRegex)) {
m_messages.notes.emplace_back(db->name % '/' % package.name + ": matches exclude regex"); m_messages.notes.emplace_back(db->name % '/' % package->name + ": matches exclude regex");
return false; return false;
} }
// skip if the package info is missing (we need the binary package's file name here) // skip if the package info is missing (we need the binary package's file name here)
const auto &packageInfo = package.packageInfo; const auto &packageInfo = package->packageInfo;
if (!packageInfo) { if (!packageInfo) {
m_messages.errors.emplace_back(db->name % '/' % package.name + ": no package info"); m_messages.errors.emplace_back(db->name % '/' % package->name + ": no package info");
return false; return false;
} }
// skip the package if it is not part of the destination DB or required by a package of the destination DB // skip the package if it is not part of the destination DB or required by a package of the destination DB
@ -125,7 +124,7 @@ void ReloadLibraryDependencies::run()
if (m_skippingNote.tellp()) { if (m_skippingNote.tellp()) {
m_skippingNote << ", "; m_skippingNote << ", ";
} }
m_skippingNote << db->name << '/' << package.name; m_skippingNote << db->name << '/' << package->name;
return false; return false;
} }
// find the package on disk; otherwise add an URL to download it from the configured mirror // find the package on disk; otherwise add an URL to download it from the configured mirror
@ -162,16 +161,16 @@ void ReloadLibraryDependencies::run()
} }
} }
if (path.empty()) { if (path.empty()) {
m_messages.errors.emplace_back(db->name % '/' % package.name + ": binary package not found and no mirror configured"); m_messages.errors.emplace_back(db->name % '/' % package->name + ": binary package not found and no mirror configured");
return false; return false;
} }
// skip if the package info has already been loaded from package contents and the present binary package is not newer // skip if the package info has already been loaded from package contents and the present binary package is not newer
auto lastModified = DateTime(); auto lastModified = DateTime();
if (url.empty()) { if (url.empty()) {
lastModified = LibPkg::lastModified(path); lastModified = LibPkg::lastModified(path);
if (!force && package.origin == LibPkg::PackageOrigin::PackageContents && package.timestamp >= lastModified) { if (!force && package->origin == LibPkg::PackageOrigin::PackageContents && package->timestamp >= lastModified) {
m_messages.notes.emplace_back(db->name % '/' % package.name % ": skipping because \"" % path % "\" is newer (" m_messages.notes.emplace_back(db->name % '/' % package->name % ": skipping because \"" % path % "\" is newer ("
% package.timestamp.toString() % " >= " % lastModified.toString() % package->timestamp.toString() % " >= " % lastModified.toString()
+ ")\n"); + ")\n");
return false; return false;
} }
@ -180,11 +179,11 @@ void ReloadLibraryDependencies::run()
auto &relevantPkg = relevantDbInfo.packages.emplace_back( auto &relevantPkg = relevantDbInfo.packages.emplace_back(
PackageToConsider{ .path = std::move(path), .url = std::move(url), .lastModified = lastModified }); PackageToConsider{ .path = std::move(path), .url = std::move(url), .lastModified = lastModified });
// create a temporary package object to hold the info parsed from the .PKGINFO file // create a temporary package object to hold the info parsed from the .PKGINFO file
relevantPkg.info.name = package.name; relevantPkg.info.name = package->name;
// -> assign certain fields which are used by addDepsAndProvidesFromOtherPackage() to check whether the packages are matching // -> assign certain fields which are used by addDepsAndProvidesFromOtherPackage() to check whether the packages are matching
relevantPkg.info.version = package.version; relevantPkg.info.version = package->version;
relevantPkg.info.packageInfo = std::make_unique<LibPkg::PackageInfo>(); relevantPkg.info.packageInfo = std::make_unique<LibPkg::PackageInfo>();
relevantPkg.info.packageInfo->buildDate = package.packageInfo->buildDate; relevantPkg.info.packageInfo->buildDate = package->packageInfo->buildDate;
// -> gather source info such as make and check dependencies as well // -> gather source info such as make and check dependencies as well
relevantPkg.info.sourceInfo = std::make_shared<LibPkg::SourceInfo>(); relevantPkg.info.sourceInfo = std::make_shared<LibPkg::SourceInfo>();
++m_remainingPackages; ++m_remainingPackages;

14
librepomgr/buildactions/repomanagement.cpp
View File

@ -444,21 +444,21 @@ void CheckForProblems::run()
problems.emplace_back( problems.emplace_back(
RepositoryProblem{ .desc = "configured local package directory \"" % db->localPkgDir + "\" is not a directory" }); RepositoryProblem{ .desc = "configured local package directory \"" % db->localPkgDir + "\" is not a directory" });
} }
db->allPackages([&](LibPkg::StorageID, LibPkg::Package &&package) { db->allPackages([&](LibPkg::StorageID, const std::shared_ptr<LibPkg::Package> &package) {
if (!package.packageInfo) { if (!package->packageInfo) {
problems.emplace_back(RepositoryProblem{ .desc = "no package info present", .pkg = package.name }); problems.emplace_back(RepositoryProblem{ .desc = "no package info present", .pkg = package->name });
return false; return false;
} }
const auto packageLocation = db->locatePackage(package.packageInfo->fileName); const auto packageLocation = db->locatePackage(package->packageInfo->fileName);
if (!packageLocation.exists) { if (!packageLocation.exists) {
problems.emplace_back( problems.emplace_back(
RepositoryProblem{ .desc = "binary package \"" % package.packageInfo->fileName + "\" not present", .pkg = package.name }); RepositoryProblem{ .desc = "binary package \"" % package->packageInfo->fileName + "\" not present", .pkg = package->name });
} }
if (m_requirePackageSignatures) { if (m_requirePackageSignatures) {
const auto signatureLocation = db->locatePackage(package.packageInfo->fileName + ".sig"); const auto signatureLocation = db->locatePackage(package->packageInfo->fileName + ".sig");
if (!signatureLocation.exists) { if (!signatureLocation.exists) {
problems.emplace_back(RepositoryProblem{ problems.emplace_back(RepositoryProblem{
.desc = "signature file for package \"" % package.packageInfo->fileName + "\" not present", .pkg = package.name }); .desc = "signature file for package \"" % package->packageInfo->fileName + "\" not present", .pkg = package->name });
} }
} }
return false; return false;

20
librepomgr/serversetup.cpp
View File

@ -399,13 +399,13 @@ RAPIDJSON_NAMESPACE::Document ServiceSetup::libraryDependenciesToJson()
auto &alloc = document.GetAllocator(); auto &alloc = document.GetAllocator();
for (auto &db : config.databases) { for (auto &db : config.databases) {
auto dbValue = RAPIDJSON_NAMESPACE::Value(RAPIDJSON_NAMESPACE::Type::kObjectType); auto dbValue = RAPIDJSON_NAMESPACE::Value(RAPIDJSON_NAMESPACE::Type::kObjectType);
db.allPackages([&](StorageID, Package &&package) { db.allPackages([&](StorageID, const std::shared_ptr<Package> &package) {
if (!package.packageInfo) { if (!package->packageInfo) {
return false; return false;
} }
if (package.libdepends.empty() && package.libprovides.empty()) { if (package->libdepends.empty() && package->libprovides.empty()) {
auto hasVersionedPythonOrPerlDep = false; auto hasVersionedPythonOrPerlDep = false;
for (const auto &dependency : package.dependencies) { for (const auto &dependency : package->dependencies) {
if (dependency.mode == DependencyMode::Any || dependency.version.empty() if (dependency.mode == DependencyMode::Any || dependency.version.empty()
|| (dependency.name != "python" && dependency.name != "python2" && dependency.name != "perl")) { || (dependency.name != "python" && dependency.name != "python2" && dependency.name != "perl")) {
return false; return false;
@ -419,12 +419,12 @@ RAPIDJSON_NAMESPACE::Document ServiceSetup::libraryDependenciesToJson()
} }
auto pkgValue = RAPIDJSON_NAMESPACE::Value(RAPIDJSON_NAMESPACE::Type::kObjectType); auto pkgValue = RAPIDJSON_NAMESPACE::Value(RAPIDJSON_NAMESPACE::Type::kObjectType);
auto pkgObj = pkgValue.GetObject(); auto pkgObj = pkgValue.GetObject();
JR::push(package.version, "v", pkgObj, alloc); JR::push(package->version, "v", pkgObj, alloc);
JR::push(package.packageInfo->buildDate, "t", pkgObj, alloc); JR::push(package->packageInfo->buildDate, "t", pkgObj, alloc);
JR::push(package.dependencies, "d", pkgObj, alloc); // for versioned Python/Perl deps JR::push(package->dependencies, "d", pkgObj, alloc); // for versioned Python/Perl deps
JR::push(package.libdepends, "ld", pkgObj, alloc); JR::push(package->libdepends, "ld", pkgObj, alloc);
JR::push(package.libprovides, "lp", pkgObj, alloc); JR::push(package->libprovides, "lp", pkgObj, alloc);
dbValue.AddMember(RAPIDJSON_NAMESPACE::StringRef(package.name.data(), JR::rapidJsonSize(package.name.size())), pkgValue, alloc); dbValue.AddMember(RAPIDJSON_NAMESPACE::StringRef(package->name.data(), JR::rapidJsonSize(package->name.size())), pkgValue, alloc);
return false; return false;
}); });
document.AddMember(RAPIDJSON_NAMESPACE::Value(db.name % '@' + db.arch, alloc), dbValue, alloc); document.AddMember(RAPIDJSON_NAMESPACE::Value(db.name % '@' + db.arch, alloc), dbValue, alloc);

4
librepomgr/tests/buildactions.cpp
View File

@ -149,8 +149,8 @@ void BuildActionsTests::logTestSetup()
{ {
for (auto &db : m_setup.config.databases) { for (auto &db : m_setup.config.databases) {
cout << EscapeCodes::Phrases::Info << "Packages of " << db.name << ':' << EscapeCodes::Phrases::End; cout << EscapeCodes::Phrases::Info << "Packages of " << db.name << ':' << EscapeCodes::Phrases::End;
db.allPackages([](LibPkg::StorageID, LibPkg::Package &&package) { db.allPackages([](LibPkg::StorageID, const std::shared_ptr<LibPkg::Package> &package) {
cout << " - " << package.name << '\n'; cout << " - " << package->name << '\n';
return false; return false;
}); });
} }

6
librepomgr/webclient/aur.cpp
View File

@ -169,9 +169,9 @@ std::shared_ptr<AurQuerySession> queryAurPackagesForDatabase(LogContext &log, Se
configReadLock = &ownConfigReadLock; configReadLock = &ownConfigReadLock;
} }
auto &aurDb = setup.config.aur; auto &aurDb = setup.config.aur;
database.allPackages([&aurDb, &missingPackages](StorageID, Package &&package) { database.allPackages([&aurDb, &missingPackages](StorageID, const std::shared_ptr<Package> &package) {
if (const auto aurPackage = aurDb.findPackage(package.name); !aurPackage) { if (const auto aurPackage = aurDb.findPackage(package->name); !aurPackage) {
missingPackages.emplace_back(package.name); missingPackages.emplace_back(package->name);
} }
return false; return false;
}); });

8
pacfind/main.cpp
View File

@ -131,8 +131,8 @@ int main(int argc, const char *argv[])
} }
} }
for (auto &db : cfg.databases) { for (auto &db : cfg.databases) {
db.allPackages([&](LibPkg::StorageID, LibPkg::Package &&package) { db.allPackages([&](LibPkg::StorageID, const std::shared_ptr<LibPkg::Package> &package) {
const auto &pkgInfo = package.packageInfo; const auto &pkgInfo = package->packageInfo;
if (!pkgInfo) { if (!pkgInfo) {
return false; return false;
} }
@ -154,7 +154,7 @@ int main(int argc, const char *argv[])
if (!db.name.empty()) { if (!db.name.empty()) {
std::cout << db.name << '/'; std::cout << db.name << '/';
} }
std::cout << package.name << '\n'; std::cout << package->name << '\n';
foundOne = true; foundOne = true;
} }
std::cout << " - " << file << '\n'; std::cout << " - " << file << '\n';
@ -163,7 +163,7 @@ int main(int argc, const char *argv[])
if (!db.name.empty()) { if (!db.name.empty()) {
std::cout << db.name << '/'; std::cout << db.name << '/';
} }
std::cout << package.name << '\n'; std::cout << package->name << '\n';
} }
return false; return false;
}); });