Make storing lib deps more efficient and allow for intermediate reads

libpkg/data/database.cpp

@@ -718,6 +718,11 @@ PackageUpdater::~PackageUpdater()
 {
 }
 
+LibPkg::PackageSpec LibPkg::PackageUpdater::findPackageWithID(const std::string &packageName)
+{
+    return m_database.m_storage->packageCache.retrieve(*m_database.m_storage, &m_d->packagesTxn, packageName);
+}
+
 StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
 {
     const auto res = m_database.m_storage->packageCache.store(*m_database.m_storage, m_d->packagesTxn, package);

libpkg/data/database.h

@@ -102,6 +102,7 @@ struct LIBPKG_EXPORT PackageUpdater {
     explicit PackageUpdater(Database &database);
     ~PackageUpdater();
 
+    PackageSpec findPackageWithID(const std::string &packageName);
     StorageID update(const std::shared_ptr<Package> &package);
     void commit();
 

libpkg/data/storage.cpp

@@ -83,7 +83,7 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
 }
 
 template <typename StorageEntriesType, typename StorageType, typename SpecType>
-auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, const std::string &entryName) -> SpecType
+auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, RWTxn *txn, const std::string &entryName) -> SpecType
 {
     // check for package in cache
     using CacheRef = typename Entries::Ref;
@@ -95,8 +95,7 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
     lock.unlock();
     // check for package in storage, populate cache entry
     auto entry = std::make_shared<Entry>();
-    auto txn = storage.packages.getROTransaction();
-    if (auto id = txn.template get<0>(entryName, *entry)) {
+    if (auto id = txn ? txn->template get<0>(entryName, *entry) : storage.packages.getROTransaction().template get<0>(entryName, *entry)) {
         using CacheEntry = typename Entries::StorageEntry;
         auto newCacheEntry = CacheEntry(ref, id);
         newCacheEntry.entry = entry;
@@ -108,6 +107,12 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
     return SpecType(0, std::shared_ptr<Entry>());
 }
 
+template <typename StorageEntriesType, typename StorageType, typename SpecType>
+auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, const std::string &entryName) -> SpecType
+{
+    return retrieve(storage, nullptr, entryName);
+}
+
 template <typename StorageEntriesType, typename StorageType, typename SpecType>
 auto StorageCache<StorageEntriesType, StorageType, SpecType>::store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force) -> StoreResult
 {

libpkg/data/storageprivate.h

@@ -145,6 +145,7 @@ template <typename StorageEntriesType, typename StorageType, typename SpecType>
 
     SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID);
     SpecType retrieve(Storage &storage, StorageID storageID);
+    SpecType retrieve(Storage &storage, RWTxn *, const std::string &entryName);
     SpecType retrieve(Storage &storage, const std::string &entryName);
     StoreResult store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force);
     StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr<Entry> &entry);

librepomgr/buildactions/reloadlibrarydependencies.cpp

@@ -337,19 +337,20 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents()
     // store the information in the database
     m_buildAction->appendOutput(Phrases::SuccessMessage, "Adding parsed information to databases ...\n");
     std::size_t counter = 0;
-    auto configWritelock = m_setup.config.lockToWrite();
     for (DatabaseToConsider &relevantDb : m_relevantPackagesByDatabase) {
+        auto configWritelock = m_setup.config.lockToWrite(); // acquire lock within loop to allow intermediate reads
         auto *const db = m_setup.config.findDatabase(relevantDb.name, relevantDb.arch);
         if (!db) {
             continue; // the whole database has been removed while we were loading package contents
         }
+        auto updater = LibPkg::PackageUpdater(*db);
         for (PackageToConsider &package : relevantDb.packages) {
             // skip if package info could not be parsed from package contents
             if (package.info.origin != LibPkg::PackageOrigin::PackageContents) {
                 continue;
             }
             // find the package in the database again
-            const auto [packageID, existingPackage] = db->findPackageWithID(package.info.name);
+            const auto [packageID, existingPackage] = updater.findPackageWithID(package.info.name);
             if (!existingPackage) {
                 continue; // the package has been removed while we were loading package contents
             }
@@ -362,11 +363,11 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents()
                 existingPackage->timestamp = package.lastModified;
             }
             // add the new dependencies on database-level
-            db->forceUpdatePackage(existingPackage);
+            updater.update(existingPackage);
             ++counter;
         }
+        updater.commit();
     }
-    configWritelock.unlock();
 
     m_buildAction->appendOutput(Phrases::SuccessMessage, "Added dependency information for ", counter, " packages\n");
     conclude();