From f73753792c28529b3f55027f2e19ad7c7966edaa Mon Sep 17 00:00:00 2001 From: Martchus Date: Thu, 20 Jan 2022 23:33:02 +0100 Subject: [PATCH] lmdb: Use caching when finding package via ID --- libpkg/data/database.cpp | 5 +- libpkg/data/storage.cpp | 165 +++++++++++++++++++++++++---------- libpkg/data/storageprivate.h | 31 ++++++- 3 files changed, 146 insertions(+), 55 deletions(-) diff --git a/libpkg/data/database.cpp b/libpkg/data/database.cpp index 913fd13..3b61dc6 100644 --- a/libpkg/data/database.cpp +++ b/libpkg/data/database.cpp @@ -334,10 +334,7 @@ bool Database::provides(const std::string &libraryName, bool reverse) const std::shared_ptr Database::findPackage(StorageID packageID) { - // TODO: use cache here - auto package = std::make_shared(); - auto txn = m_storage->packages.getROTransaction(); - return txn.get(packageID, *package) ? package : std::shared_ptr(); + return m_storage->packageCache.retrieve(*m_storage, packageID).pkg; } std::shared_ptr Database::findPackage(const std::string &packageName) diff --git a/libpkg/data/storage.cpp b/libpkg/data/storage.cpp index 8336b87..8d25946 100644 --- a/libpkg/data/storage.cpp +++ b/libpkg/data/storage.cpp @@ -6,14 +6,21 @@ using namespace CppUtilities; namespace LibPkg { -template auto StorageCacheEntries::findOrCreate(const Ref &ref) -> StorageEntry & +template +template +auto StorageCacheEntries::find(const IndexType &ref) -> StorageEntry * { - const auto &index = m_entries.template get(); + const auto &index = m_entries.template get(); if (auto i = index.find(ref); i != index.end()) { m_entries.relocate(m_entries.begin(), m_entries.template project<0>(i)); - return i.get_node()->value(); + return &i.get_node()->value(); } - const auto [i, newItem] = m_entries.emplace_front(ref); + return nullptr; +} + +template auto StorageCacheEntries::insert(StorageEntry &&entry) -> StorageEntry & +{ + const auto [i, newItem] = m_entries.emplace_front(entry); if (!newItem) { m_entries.relocate(m_entries.begin(), i); } else if (m_entries.size() > m_limit) { @@ -36,25 +43,56 @@ template std::size_t StorageCacheEntries +auto StorageCache::retrieve(Storage &storage, StorageID storageID) -> SpecType +{ + // check for package in cache + const auto ref = typename StorageEntryByID::result_type{ storageID, &storage }; + auto lock = std::unique_lock(m_mutex); + if (auto *const existingCacheEntry = m_entries.find(ref)) { + return SpecType(existingCacheEntry->id, existingCacheEntry->entry); + } + // check for package in storage, populate cache entry + lock.unlock(); + auto entry = std::make_shared(); + auto txn = storage.packages.getROTransaction(); + if (auto id = txn.get(storageID, *entry)) { + using CacheEntry = typename Entries::StorageEntry; + using CacheRef = typename Entries::Ref; + auto newCacheEntry = CacheEntry(CacheRef(storage, entry), id); + newCacheEntry.entry = entry; + lock = std::unique_lock(m_mutex); + m_entries.insert(std::move(newCacheEntry)); + lock.unlock(); + return SpecType(id, entry); + } + return SpecType(0, std::shared_ptr()); +} + template auto StorageCache::retrieve(Storage &storage, const std::string &entryName) -> SpecType { // check for package in cache - const auto ref = typename Entries::Ref(storage, entryName); - const auto lock = std::unique_lock(m_mutex); - auto &cacheEntry = m_entries.findOrCreate(ref); - if (cacheEntry.entry) { - return PackageSpec(cacheEntry.id, cacheEntry.entry); + using CacheRef = typename Entries::Ref; + const auto ref = CacheRef(storage, entryName); + auto lock = std::unique_lock(m_mutex); + if (auto *const existingCacheEntry = m_entries.find(ref)) { + return SpecType(existingCacheEntry->id, existingCacheEntry->entry); } + lock.unlock(); // check for package in storage, populate cache entry - cacheEntry.entry = std::make_shared(); + auto entry = std::make_shared(); auto txn = storage.packages.getROTransaction(); - if ((cacheEntry.id = txn.template get<0>(entryName, *cacheEntry.entry))) { - cacheEntry.ref.entryName = &cacheEntry.entry->name; - return PackageSpec(cacheEntry.id, cacheEntry.entry); + if (auto id = txn.template get<0>(entryName, *entry)) { + using CacheEntry = typename Entries::StorageEntry; + auto newCacheEntry = CacheEntry(ref, id); + newCacheEntry.entry = entry; + lock = std::unique_lock(m_mutex); + m_entries.insert(std::move(newCacheEntry)); + lock.unlock(); + return SpecType(id, entry); } - m_entries.undo(); - return PackageSpec(0, std::shared_ptr()); + return SpecType(0, std::shared_ptr()); } template @@ -62,32 +100,46 @@ auto StorageCache::store(Storage -> StoreResult { // check for package in cache - const auto ref = typename Entries::Ref(storage, entry->name); + using CacheEntry = typename Entries::StorageEntry; + using CacheRef = typename Entries::Ref; + const auto ref = CacheRef(storage, entry->name); auto res = StorageCache::StoreResult(); auto lock = std::unique_lock(m_mutex); - auto &cacheEntry = m_entries.findOrCreate(ref); - if (cacheEntry.entry == entry && !force) { - // do nothing if cached package is the same as specified one - res.id = cacheEntry.id; - return res; - } else if (cacheEntry.entry) { - // retain certain information obtained from package contents if this is actually the same package as before - entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry)); - } else { - cacheEntry.entry = std::make_shared(); + auto *cacheEntry = m_entries.find(ref); + if (cacheEntry) { + res.id = cacheEntry->id; + res.oldEntry = cacheEntry->entry; + if (cacheEntry->entry == entry && !force) { + // do nothing if cached package is the same as specified one + return res; + } else { + // retain certain information obtained from package contents if this is actually the same package as before + entry->addDepsAndProvidesFromOtherPackage(*cacheEntry->entry); + } } + lock.unlock(); // check for package in storage auto txn = storage.packages.getRWTransaction(); - if (!res.oldEntry && (cacheEntry.id = txn.template get<0>(entry->name, *cacheEntry.entry))) { - entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry)); + if (!res.oldEntry) { + res.oldEntry = std::make_shared(); + if (txn.template get<0>(entry->name, *res.oldEntry)) { + entry->addDepsAndProvidesFromOtherPackage(*res.oldEntry); + } else { + res.oldEntry.reset(); + } } - // update cache entry - cacheEntry.ref.entryName = &entry->name; - cacheEntry.entry = entry; // update package in storage - cacheEntry.id = txn.put(*entry, cacheEntry.id); + res.id = txn.put(*entry, res.id); + // update cache entry + lock = std::unique_lock(m_mutex); + if (cacheEntry) { + cacheEntry->ref.entryName = &entry->name; + } else { + cacheEntry = &m_entries.insert(CacheEntry(ref, res.id)); + } + cacheEntry->entry = entry; + lock.unlock(); txn.commit(); - res.id = cacheEntry.id; res.updated = true; return res; } @@ -97,26 +149,38 @@ auto StorageCache::store(Storage -> StoreResult { // check for package in cache - const auto ref = typename Entries::Ref(storage, entry->name); + using CacheEntry = typename Entries::StorageEntry; + using CacheRef = typename Entries::Ref; + const auto ref = CacheRef(storage, entry->name); auto res = StorageCache::StoreResult(); auto lock = std::unique_lock(m_mutex); - auto &cacheEntry = m_entries.findOrCreate(ref); - if (cacheEntry.entry) { + auto *cacheEntry = m_entries.find(ref); + if (cacheEntry) { // retain certain information obtained from package contents if this is actually the same package as before - res.id = cacheEntry.id; - entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry)); - } else { - // check for package in storage - cacheEntry.entry = std::make_shared(); - if ((cacheEntry.id = txn.template get<0>(entry->name, *cacheEntry.entry))) { - entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry)); + res.id = cacheEntry->id; + entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry->entry)); + } + lock.unlock(); + // check for package in storage + if (!res.oldEntry) { + res.oldEntry = std::make_shared(); + if (txn.template get<0>(entry->name, *res.oldEntry)) { + entry->addDepsAndProvidesFromOtherPackage(*res.oldEntry); + } else { + res.oldEntry.reset(); } } - // update cache entry - cacheEntry.ref.entryName = &entry->name; - cacheEntry.entry = entry; // update package in storage - res.id = cacheEntry.id = txn.put(*entry, cacheEntry.id); + res.id = txn.put(*entry, res.id); + // update cache entry + lock = std::unique_lock(m_mutex); + if (cacheEntry) { + cacheEntry->ref.entryName = &entry->name; + } else { + cacheEntry = &m_entries.insert(CacheEntry(ref, res.id)); + } + cacheEntry->entry = entry; + lock.unlock(); res.updated = true; return res; } @@ -195,4 +259,11 @@ std::size_t hash_value(const PackageCacheRef &ref) return ((hasher1(ref.relatedStorage) ^ (hasher2(*ref.entryName) << 1)) >> 1); } +std::size_t hash_value(const PackageCacheEntryByID &entryByID) +{ + const auto hasher1 = boost::hash(); + const auto hasher2 = boost::hash(); + return ((hasher1(entryByID.id) ^ (hasher2(entryByID.storage) << 1)) >> 1); +} + } // namespace LibPkg diff --git a/libpkg/data/storageprivate.h b/libpkg/data/storageprivate.h index 96cec3f..9f2f6d1 100644 --- a/libpkg/data/storageprivate.h +++ b/libpkg/data/storageprivate.h @@ -52,33 +52,53 @@ template struct StorageCacheEntry using Ref = StorageRefType; using Entry = EntryType; using Storage = typename Ref::Storage; - explicit StorageCacheEntry(const StorageRefType &ref); + explicit StorageCacheEntry(const StorageRefType &ref, StorageID id); StorageRefType ref; StorageID id; std::shared_ptr entry; }; template -inline StorageCacheEntry::StorageCacheEntry(const StorageRefType &ref) +inline StorageCacheEntry::StorageCacheEntry(const StorageRefType &ref, StorageID id) : ref(ref) - , id(0) + , id(id) { } +template struct StorageEntryByID { + struct result_type { + StorageID id = 0; + const typename StorageEntryType::Storage *storage = nullptr; + + bool operator==(const result_type &other) const + { + return id == other.id && storage == other.storage; + } + }; + + result_type operator()(const StorageEntryType &storageEntry) const + { + return result_type{ storageEntry.id, storageEntry.ref.relatedStorage }; + } +}; + template class StorageCacheEntries { public: using Ref = typename StorageEntryType::Ref; using Entry = typename StorageEntryType::Entry; using Storage = typename StorageEntryType::Storage; using StorageEntry = StorageEntryType; + using ByID = StorageEntryByID; using EntryList = boost::multi_index::multi_index_container, + boost::multi_index::hashed_unique, ByID>, boost::multi_index::hashed_unique, BOOST_MULTI_INDEX_MEMBER(StorageEntryType, Ref, ref)>>>; using iterator = typename EntryList::iterator; explicit StorageCacheEntries(std::size_t limit = 1000); - StorageEntry &findOrCreate(const Ref &ref); + template StorageEntry *find(const IndexType &ref); + StorageEntry &insert(StorageEntry &&entry); void undo(); std::size_t erase(const Ref &ref); std::size_t clear(const Storage &storage); @@ -127,6 +147,7 @@ template oldEntry; }; + SpecType retrieve(Storage &storage, StorageID storageID); SpecType retrieve(Storage &storage, const std::string &entryName); StoreResult store(Storage &storage, const std::shared_ptr &entry, bool force); StoreResult store(Storage &storage, Txn &txn, const std::shared_ptr &entry); @@ -146,6 +167,7 @@ using LibraryDependencyStorage using PackageCacheRef = StorageCacheRef; using PackageCacheEntry = StorageCacheEntry; using PackageCacheEntries = StorageCacheEntries; +using PackageCacheEntryByID = typename PackageCacheEntries::ByID::result_type; using PackageCache = StorageCache; extern template struct StorageCacheRef; @@ -182,6 +204,7 @@ private: }; std::size_t hash_value(const PackageCacheRef &ref); +std::size_t hash_value(const PackageCacheEntryByID &entryByID); } // namespace LibPkg