From 66f59fecb985439c1929e49b496457491279afdb Mon Sep 17 00:00:00 2001 From: Martchus Date: Mon, 31 Jan 2022 20:51:45 +0100 Subject: [PATCH] Make storing lib deps more efficient and allow for intermediate reads --- libpkg/data/database.cpp | 5 +++++ libpkg/data/database.h | 1 + libpkg/data/storage.cpp | 11 ++++++++--- libpkg/data/storageprivate.h | 1 + librepomgr/buildactions/reloadlibrarydependencies.cpp | 9 +++++---- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/libpkg/data/database.cpp b/libpkg/data/database.cpp index 76f6491..0a8b94a 100644 --- a/libpkg/data/database.cpp +++ b/libpkg/data/database.cpp @@ -718,6 +718,11 @@ PackageUpdater::~PackageUpdater() { } +LibPkg::PackageSpec LibPkg::PackageUpdater::findPackageWithID(const std::string &packageName) +{ + return m_database.m_storage->packageCache.retrieve(*m_database.m_storage, &m_d->packagesTxn, packageName); +} + StorageID PackageUpdater::update(const std::shared_ptr &package) { const auto res = m_database.m_storage->packageCache.store(*m_database.m_storage, m_d->packagesTxn, package); diff --git a/libpkg/data/database.h b/libpkg/data/database.h index 37e3be1..dd79161 100644 --- a/libpkg/data/database.h +++ b/libpkg/data/database.h @@ -102,6 +102,7 @@ struct LIBPKG_EXPORT PackageUpdater { explicit PackageUpdater(Database &database); ~PackageUpdater(); + PackageSpec findPackageWithID(const std::string &packageName); StorageID update(const std::shared_ptr &package); void commit(); diff --git a/libpkg/data/storage.cpp b/libpkg/data/storage.cpp index 8e4d8b0..04cdc1b 100644 --- a/libpkg/data/storage.cpp +++ b/libpkg/data/storage.cpp @@ -83,7 +83,7 @@ auto StorageCache::retrieve(Storage & } template -auto StorageCache::retrieve(Storage &storage, const std::string &entryName) -> SpecType +auto StorageCache::retrieve(Storage &storage, RWTxn *txn, const std::string &entryName) -> SpecType { // check for package in cache using CacheRef = typename Entries::Ref; @@ -95,8 +95,7 @@ auto StorageCache::retrieve(Storage & lock.unlock(); // check for package in storage, populate cache entry auto entry = std::make_shared(); - auto txn = storage.packages.getROTransaction(); - if (auto id = txn.template get<0>(entryName, *entry)) { + if (auto id = txn ? txn->template get<0>(entryName, *entry) : storage.packages.getROTransaction().template get<0>(entryName, *entry)) { using CacheEntry = typename Entries::StorageEntry; auto newCacheEntry = CacheEntry(ref, id); newCacheEntry.entry = entry; @@ -108,6 +107,12 @@ auto StorageCache::retrieve(Storage & return SpecType(0, std::shared_ptr()); } +template +auto StorageCache::retrieve(Storage &storage, const std::string &entryName) -> SpecType +{ + return retrieve(storage, nullptr, entryName); +} + template auto StorageCache::store(Storage &storage, const std::shared_ptr &entry, bool force) -> StoreResult { diff --git a/libpkg/data/storageprivate.h b/libpkg/data/storageprivate.h index a268ee7..0612859 100644 --- a/libpkg/data/storageprivate.h +++ b/libpkg/data/storageprivate.h @@ -145,6 +145,7 @@ template SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID); SpecType retrieve(Storage &storage, StorageID storageID); + SpecType retrieve(Storage &storage, RWTxn *, const std::string &entryName); SpecType retrieve(Storage &storage, const std::string &entryName); StoreResult store(Storage &storage, const std::shared_ptr &entry, bool force); StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr &entry); diff --git a/librepomgr/buildactions/reloadlibrarydependencies.cpp b/librepomgr/buildactions/reloadlibrarydependencies.cpp index ba54c64..83f6cab 100644 --- a/librepomgr/buildactions/reloadlibrarydependencies.cpp +++ b/librepomgr/buildactions/reloadlibrarydependencies.cpp @@ -337,19 +337,20 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents() // store the information in the database m_buildAction->appendOutput(Phrases::SuccessMessage, "Adding parsed information to databases ...\n"); std::size_t counter = 0; - auto configWritelock = m_setup.config.lockToWrite(); for (DatabaseToConsider &relevantDb : m_relevantPackagesByDatabase) { + auto configWritelock = m_setup.config.lockToWrite(); // acquire lock within loop to allow intermediate reads auto *const db = m_setup.config.findDatabase(relevantDb.name, relevantDb.arch); if (!db) { continue; // the whole database has been removed while we were loading package contents } + auto updater = LibPkg::PackageUpdater(*db); for (PackageToConsider &package : relevantDb.packages) { // skip if package info could not be parsed from package contents if (package.info.origin != LibPkg::PackageOrigin::PackageContents) { continue; } // find the package in the database again - const auto [packageID, existingPackage] = db->findPackageWithID(package.info.name); + const auto [packageID, existingPackage] = updater.findPackageWithID(package.info.name); if (!existingPackage) { continue; // the package has been removed while we were loading package contents } @@ -362,11 +363,11 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents() existingPackage->timestamp = package.lastModified; } // add the new dependencies on database-level - db->forceUpdatePackage(existingPackage); + updater.update(existingPackage); ++counter; } + updater.commit(); } - configWritelock.unlock(); m_buildAction->appendOutput(Phrases::SuccessMessage, "Added dependency information for ", counter, " packages\n"); conclude();