lmdb: Use caching when finding package via ID

libpkg/data/database.cpp

@@ -334,10 +334,7 @@ bool Database::provides(const std::string &libraryName, bool reverse) const
 
 std::shared_ptr<Package> Database::findPackage(StorageID packageID)
 {
-    // TODO: use cache here
+    return m_storage->packageCache.retrieve(*m_storage, packageID).pkg;
-    auto package = std::make_shared<Package>();
-    auto txn = m_storage->packages.getROTransaction();
-    return txn.get(packageID, *package) ? package : std::shared_ptr<Package>();
 }
 
 std::shared_ptr<Package> Database::findPackage(const std::string &packageName)

libpkg/data/storage.cpp

@@ -6,14 +6,21 @@ using namespace CppUtilities;
 
 namespace LibPkg {
 
-template <typename StorageEntryType> auto StorageCacheEntries<StorageEntryType>::findOrCreate(const Ref &ref) -> StorageEntry &
+template <typename StorageEntryType>
+template <typename IndexType>
+auto StorageCacheEntries<StorageEntryType>::find(const IndexType &ref) -> StorageEntry *
 {
-    const auto &index = m_entries.template get<Ref>();
+    const auto &index = m_entries.template get<IndexType>();
     if (auto i = index.find(ref); i != index.end()) {
         m_entries.relocate(m_entries.begin(), m_entries.template project<0>(i));
-        return i.get_node()->value();
+        return &i.get_node()->value();
     }
-    const auto [i, newItem] = m_entries.emplace_front(ref);
+    return nullptr;
+}
+
+template <typename StorageEntryType> auto StorageCacheEntries<StorageEntryType>::insert(StorageEntry &&entry) -> StorageEntry &
+{
+    const auto [i, newItem] = m_entries.emplace_front(entry);
     if (!newItem) {
         m_entries.relocate(m_entries.begin(), i);
     } else if (m_entries.size() > m_limit) {
@@ -36,25 +43,56 @@ template <typename StorageEntryType> std::size_t StorageCacheEntries<StorageEntr
     return count;
 }
 
+template <typename StorageEntriesType, typename TransactionType, typename SpecType>
+auto StorageCache<StorageEntriesType, TransactionType, SpecType>::retrieve(Storage &storage, StorageID storageID) -> SpecType
+{
+    // check for package in cache
+    const auto ref = typename StorageEntryByID<typename Entries::StorageEntry>::result_type{ storageID, &storage };
+    auto lock = std::unique_lock(m_mutex);
+    if (auto *const existingCacheEntry = m_entries.find(ref)) {
+        return SpecType(existingCacheEntry->id, existingCacheEntry->entry);
+    }
+    // check for package in storage, populate cache entry
+    lock.unlock();
+    auto entry = std::make_shared<Entry>();
+    auto txn = storage.packages.getROTransaction();
+    if (auto id = txn.get(storageID, *entry)) {
+        using CacheEntry = typename Entries::StorageEntry;
+        using CacheRef = typename Entries::Ref;
+        auto newCacheEntry = CacheEntry(CacheRef(storage, entry), id);
+        newCacheEntry.entry = entry;
+        lock = std::unique_lock(m_mutex);
+        m_entries.insert(std::move(newCacheEntry));
+        lock.unlock();
+        return SpecType(id, entry);
+    }
+    return SpecType(0, std::shared_ptr<Entry>());
+}
+
 template <typename StorageEntriesType, typename TransactionType, typename SpecType>
 auto StorageCache<StorageEntriesType, TransactionType, SpecType>::retrieve(Storage &storage, const std::string &entryName) -> SpecType
 {
     // check for package in cache
-    const auto ref = typename Entries::Ref(storage, entryName);
+    using CacheRef = typename Entries::Ref;
-    const auto lock = std::unique_lock(m_mutex);
+    const auto ref = CacheRef(storage, entryName);
-    auto &cacheEntry = m_entries.findOrCreate(ref);
+    auto lock = std::unique_lock(m_mutex);
-    if (cacheEntry.entry) {
+    if (auto *const existingCacheEntry = m_entries.find(ref)) {
-        return PackageSpec(cacheEntry.id, cacheEntry.entry);
+        return SpecType(existingCacheEntry->id, existingCacheEntry->entry);
     }
+    lock.unlock();
     // check for package in storage, populate cache entry
-    cacheEntry.entry = std::make_shared<Entry>();
+    auto entry = std::make_shared<Entry>();
     auto txn = storage.packages.getROTransaction();
-    if ((cacheEntry.id = txn.template get<0>(entryName, *cacheEntry.entry))) {
+    if (auto id = txn.template get<0>(entryName, *entry)) {
-        cacheEntry.ref.entryName = &cacheEntry.entry->name;
+        using CacheEntry = typename Entries::StorageEntry;
-        return PackageSpec(cacheEntry.id, cacheEntry.entry);
+        auto newCacheEntry = CacheEntry(ref, id);
+        newCacheEntry.entry = entry;
+        lock = std::unique_lock(m_mutex);
+        m_entries.insert(std::move(newCacheEntry));
+        lock.unlock();
+        return SpecType(id, entry);
     }
-    m_entries.undo();
+    return SpecType(0, std::shared_ptr<Entry>());
-    return PackageSpec(0, std::shared_ptr<Entry>());
 }
 
 template <typename StorageEntriesType, typename TransactionType, typename SpecType>
@@ -62,32 +100,46 @@ auto StorageCache<StorageEntriesType, TransactionType, SpecType>::store(Storage
     -> StoreResult
 {
     // check for package in cache
-    const auto ref = typename Entries::Ref(storage, entry->name);
+    using CacheEntry = typename Entries::StorageEntry;
+    using CacheRef = typename Entries::Ref;
+    const auto ref = CacheRef(storage, entry->name);
     auto res = StorageCache::StoreResult();
     auto lock = std::unique_lock(m_mutex);
-    auto &cacheEntry = m_entries.findOrCreate(ref);
+    auto *cacheEntry = m_entries.find(ref);
-    if (cacheEntry.entry == entry && !force) {
+    if (cacheEntry) {
-        // do nothing if cached package is the same as specified one
+        res.id = cacheEntry->id;
-        res.id = cacheEntry.id;
+        res.oldEntry = cacheEntry->entry;
-        return res;
+        if (cacheEntry->entry == entry && !force) {
-    } else if (cacheEntry.entry) {
+            // do nothing if cached package is the same as specified one
-        // retain certain information obtained from package contents if this is actually the same package as before
+            return res;
-        entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry));
+        } else {
-    } else {
+            // retain certain information obtained from package contents if this is actually the same package as before
-        cacheEntry.entry = std::make_shared<Entry>();
+            entry->addDepsAndProvidesFromOtherPackage(*cacheEntry->entry);
+        }
     }
+    lock.unlock();
     // check for package in storage
     auto txn = storage.packages.getRWTransaction();
-    if (!res.oldEntry && (cacheEntry.id = txn.template get<0>(entry->name, *cacheEntry.entry))) {
+    if (!res.oldEntry) {
-        entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry));
+        res.oldEntry = std::make_shared<Entry>();
+        if (txn.template get<0>(entry->name, *res.oldEntry)) {
+            entry->addDepsAndProvidesFromOtherPackage(*res.oldEntry);
+        } else {
+            res.oldEntry.reset();
+        }
     }
-    // update cache entry
-    cacheEntry.ref.entryName = &entry->name;
-    cacheEntry.entry = entry;
     // update package in storage
-    cacheEntry.id = txn.put(*entry, cacheEntry.id);
+    res.id = txn.put(*entry, res.id);
+    // update cache entry
+    lock = std::unique_lock(m_mutex);
+    if (cacheEntry) {
+        cacheEntry->ref.entryName = &entry->name;
+    } else {
+        cacheEntry = &m_entries.insert(CacheEntry(ref, res.id));
+    }
+    cacheEntry->entry = entry;
+    lock.unlock();
     txn.commit();
-    res.id = cacheEntry.id;
     res.updated = true;
     return res;
 }
@@ -97,26 +149,38 @@ auto StorageCache<StorageEntriesType, TransactionType, SpecType>::store(Storage
     -> StoreResult
 {
     // check for package in cache
-    const auto ref = typename Entries::Ref(storage, entry->name);
+    using CacheEntry = typename Entries::StorageEntry;
+    using CacheRef = typename Entries::Ref;
+    const auto ref = CacheRef(storage, entry->name);
     auto res = StorageCache::StoreResult();
     auto lock = std::unique_lock(m_mutex);
-    auto &cacheEntry = m_entries.findOrCreate(ref);
+    auto *cacheEntry = m_entries.find(ref);
-    if (cacheEntry.entry) {
+    if (cacheEntry) {
         // retain certain information obtained from package contents if this is actually the same package as before
-        res.id = cacheEntry.id;
+        res.id = cacheEntry->id;
-        entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry));
+        entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry->entry));
-    } else {
+    }
-        // check for package in storage
+    lock.unlock();
-        cacheEntry.entry = std::make_shared<Entry>();
+    // check for package in storage
-        if ((cacheEntry.id = txn.template get<0>(entry->name, *cacheEntry.entry))) {
+    if (!res.oldEntry) {
-            entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry));
+        res.oldEntry = std::make_shared<Entry>();
+        if (txn.template get<0>(entry->name, *res.oldEntry)) {
+            entry->addDepsAndProvidesFromOtherPackage(*res.oldEntry);
+        } else {
+            res.oldEntry.reset();
         }
     }
-    // update cache entry
-    cacheEntry.ref.entryName = &entry->name;
-    cacheEntry.entry = entry;
     // update package in storage
-    res.id = cacheEntry.id = txn.put(*entry, cacheEntry.id);
+    res.id = txn.put(*entry, res.id);
+    // update cache entry
+    lock = std::unique_lock(m_mutex);
+    if (cacheEntry) {
+        cacheEntry->ref.entryName = &entry->name;
+    } else {
+        cacheEntry = &m_entries.insert(CacheEntry(ref, res.id));
+    }
+    cacheEntry->entry = entry;
+    lock.unlock();
     res.updated = true;
     return res;
 }
@@ -195,4 +259,11 @@ std::size_t hash_value(const PackageCacheRef &ref)
     return ((hasher1(ref.relatedStorage) ^ (hasher2(*ref.entryName) << 1)) >> 1);
 }
 
+std::size_t hash_value(const PackageCacheEntryByID &entryByID)
+{
+    const auto hasher1 = boost::hash<StorageID>();
+    const auto hasher2 = boost::hash<const LibPkg::DatabaseStorage *>();
+    return ((hasher1(entryByID.id) ^ (hasher2(entryByID.storage) << 1)) >> 1);
+}
+
 } // namespace LibPkg

libpkg/data/storageprivate.h

@@ -52,33 +52,53 @@ template <typename StorageRefType, typename EntryType> struct StorageCacheEntry
     using Ref = StorageRefType;
     using Entry = EntryType;
     using Storage = typename Ref::Storage;
-    explicit StorageCacheEntry(const StorageRefType &ref);
+    explicit StorageCacheEntry(const StorageRefType &ref, StorageID id);
     StorageRefType ref;
     StorageID id;
     std::shared_ptr<EntryType> entry;
 };
 
 template <typename StorageRefType, typename EntryType>
-inline StorageCacheEntry<StorageRefType, EntryType>::StorageCacheEntry(const StorageRefType &ref)
+inline StorageCacheEntry<StorageRefType, EntryType>::StorageCacheEntry(const StorageRefType &ref, StorageID id)
     : ref(ref)
-    , id(0)
+    , id(id)
 {
 }
 
+template <typename StorageEntryType> struct StorageEntryByID {
+    struct result_type {
+        StorageID id = 0;
+        const typename StorageEntryType::Storage *storage = nullptr;
+
+        bool operator==(const result_type &other) const
+        {
+            return id == other.id && storage == other.storage;
+        }
+    };
+
+    result_type operator()(const StorageEntryType &storageEntry) const
+    {
+        return result_type{ storageEntry.id, storageEntry.ref.relatedStorage };
+    }
+};
+
 template <typename StorageEntryType> class StorageCacheEntries {
 public:
     using Ref = typename StorageEntryType::Ref;
     using Entry = typename StorageEntryType::Entry;
     using Storage = typename StorageEntryType::Storage;
     using StorageEntry = StorageEntryType;
+    using ByID = StorageEntryByID<StorageEntry>;
     using EntryList = boost::multi_index::multi_index_container<StorageEntry,
         boost::multi_index::indexed_by<boost::multi_index::sequenced<>,
+            boost::multi_index::hashed_unique<boost::multi_index::tag<typename ByID::result_type>, ByID>,
             boost::multi_index::hashed_unique<boost::multi_index::tag<Ref>, BOOST_MULTI_INDEX_MEMBER(StorageEntryType, Ref, ref)>>>;
     using iterator = typename EntryList::iterator;
 
     explicit StorageCacheEntries(std::size_t limit = 1000);
 
-    StorageEntry &findOrCreate(const Ref &ref);
+    template <typename IndexType> StorageEntry *find(const IndexType &ref);
+    StorageEntry &insert(StorageEntry &&entry);
     void undo();
     std::size_t erase(const Ref &ref);
     std::size_t clear(const Storage &storage);
@@ -127,6 +147,7 @@ template <typename StorageEntriesType, typename TransactionType, typename SpecTy
         std::shared_ptr<typename Entries::Entry> oldEntry;
     };
 
+    SpecType retrieve(Storage &storage, StorageID storageID);
     SpecType retrieve(Storage &storage, const std::string &entryName);
     StoreResult store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force);
     StoreResult store(Storage &storage, Txn &txn, const std::shared_ptr<Entry> &entry);
@@ -146,6 +167,7 @@ using LibraryDependencyStorage
 using PackageCacheRef = StorageCacheRef<DatabaseStorage, Package>;
 using PackageCacheEntry = StorageCacheEntry<PackageCacheRef, Package>;
 using PackageCacheEntries = StorageCacheEntries<PackageCacheEntry>;
+using PackageCacheEntryByID = typename PackageCacheEntries::ByID::result_type;
 using PackageCache = StorageCache<PackageCacheEntries, PackageStorage::RWTransaction, PackageSpec>;
 
 extern template struct StorageCacheRef<DatabaseStorage, Package>;
@@ -182,6 +204,7 @@ private:
 };
 
 std::size_t hash_value(const PackageCacheRef &ref);
+std::size_t hash_value(const PackageCacheEntryByID &entryByID);
 
 } // namespace LibPkg