From 231571f2d17db64e35203c6b2249d1c5eff0c7c1 Mon Sep 17 00:00:00 2001 From: Martchus Date: Wed, 19 Jan 2022 23:27:14 +0100 Subject: [PATCH] lmdb: Generalize caching to be able to use it for other types than packages --- libpkg/data/database.cpp | 12 +-- libpkg/data/storage.cpp | 184 +++++++++++++++++++---------------- libpkg/data/storageprivate.h | 164 ++++++++++++++++++------------- 3 files changed, 198 insertions(+), 162 deletions(-) diff --git a/libpkg/data/database.cpp b/libpkg/data/database.cpp index a45cff8..913fd13 100644 --- a/libpkg/data/database.cpp +++ b/libpkg/data/database.cpp @@ -365,8 +365,8 @@ StorageID Database::updatePackage(const std::shared_ptr &package) if (!res.updated) { return res.id; } - if (res.oldPackage) { - removePackageDependencies(res.id, res.oldPackage); + if (res.oldEntry) { + removePackageDependencies(res.id, res.oldEntry); } addPackageDependencies(res.id, package); return res.id; @@ -375,8 +375,8 @@ StorageID Database::updatePackage(const std::shared_ptr &package) StorageID Database::forceUpdatePackage(const std::shared_ptr &package) { const auto res = m_storage->packageCache.store(*m_storage, package, true); - if (res.oldPackage) { - removePackageDependencies(res.id, res.oldPackage); + if (res.oldEntry) { + removePackageDependencies(res.id, res.oldEntry); } addPackageDependencies(res.id, package); return res.id; @@ -628,8 +628,8 @@ PackageUpdaterPrivate::PackageUpdaterPrivate(DatabaseStorage &storage) void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr &package) { update(res.id, false, package); - if (res.oldPackage) { - update(res.id, true, res.oldPackage); + if (res.oldEntry) { + update(res.id, true, res.oldEntry); } } diff --git a/libpkg/data/storage.cpp b/libpkg/data/storage.cpp index db1ecb5..8336b87 100644 --- a/libpkg/data/storage.cpp +++ b/libpkg/data/storage.cpp @@ -6,102 +6,132 @@ using namespace CppUtilities; namespace LibPkg { -StorageDistribution::StorageDistribution(const char *path, std::uint32_t maxDbs) +template auto StorageCacheEntries::findOrCreate(const Ref &ref) -> StorageEntry & { - m_env = LMDBSafe::getMDBEnv(path, MDB_NOSUBDIR, 0600, maxDbs); + const auto &index = m_entries.template get(); + if (auto i = index.find(ref); i != index.end()) { + m_entries.relocate(m_entries.begin(), m_entries.template project<0>(i)); + return i.get_node()->value(); + } + const auto [i, newItem] = m_entries.emplace_front(ref); + if (!newItem) { + m_entries.relocate(m_entries.begin(), i); + } else if (m_entries.size() > m_limit) { + m_entries.pop_back(); + } + return i.get_node()->value(); } -PackageSpec PackageCache::retrieve(DatabaseStorage &databaseStorage, const std::string &packageName) +template std::size_t StorageCacheEntries::clear(const Storage &storage) +{ + auto count = std::size_t(); + for (auto i = m_entries.begin(); i != m_entries.end();) { + if (i->ref.relatedStorage == &storage) { + i = m_entries.erase(i); + ++count; + } else { + ++i; + } + } + return count; +} + +template +auto StorageCache::retrieve(Storage &storage, const std::string &entryName) -> SpecType { // check for package in cache - const auto ref = PackageCacheRef(databaseStorage, packageName); + const auto ref = typename Entries::Ref(storage, entryName); const auto lock = std::unique_lock(m_mutex); - auto &cacheEntry = m_packages.findOrCreate(ref); - if (cacheEntry.package) { - return PackageSpec(cacheEntry.id, cacheEntry.package); + auto &cacheEntry = m_entries.findOrCreate(ref); + if (cacheEntry.entry) { + return PackageSpec(cacheEntry.id, cacheEntry.entry); } // check for package in storage, populate cache entry - cacheEntry.package = std::make_unique(); - auto txn = databaseStorage.packages.getROTransaction(); - if ((cacheEntry.id = txn.get<0>(packageName, *cacheEntry.package))) { - cacheEntry.ref.packageName = &cacheEntry.package->name; - return PackageSpec(cacheEntry.id, cacheEntry.package); + cacheEntry.entry = std::make_shared(); + auto txn = storage.packages.getROTransaction(); + if ((cacheEntry.id = txn.template get<0>(entryName, *cacheEntry.entry))) { + cacheEntry.ref.entryName = &cacheEntry.entry->name; + return PackageSpec(cacheEntry.id, cacheEntry.entry); } - m_packages.undo(); - return PackageSpec(0, std::shared_ptr()); + m_entries.undo(); + return PackageSpec(0, std::shared_ptr()); } -PackageCache::StoreResult PackageCache::store(DatabaseStorage &databaseStorage, const std::shared_ptr &package, bool force) +template +auto StorageCache::store(Storage &storage, const std::shared_ptr &entry, bool force) + -> StoreResult { // check for package in cache - const auto ref = PackageCacheRef(databaseStorage, package->name); - auto res = PackageCache::StoreResult(); + const auto ref = typename Entries::Ref(storage, entry->name); + auto res = StorageCache::StoreResult(); auto lock = std::unique_lock(m_mutex); - auto &cacheEntry = m_packages.findOrCreate(ref); - if (cacheEntry.package == package && !force) { + auto &cacheEntry = m_entries.findOrCreate(ref); + if (cacheEntry.entry == entry && !force) { // do nothing if cached package is the same as specified one res.id = cacheEntry.id; return res; - } else if (cacheEntry.package) { + } else if (cacheEntry.entry) { // retain certain information obtained from package contents if this is actually the same package as before - package->addDepsAndProvidesFromOtherPackage(*(res.oldPackage = cacheEntry.package)); + entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry)); } else { - cacheEntry.package = std::make_shared(); + cacheEntry.entry = std::make_shared(); } // check for package in storage - auto txn = databaseStorage.packages.getRWTransaction(); - if (!res.oldPackage && (cacheEntry.id = txn.get<0>(package->name, *cacheEntry.package))) { - package->addDepsAndProvidesFromOtherPackage(*(res.oldPackage = cacheEntry.package)); + auto txn = storage.packages.getRWTransaction(); + if (!res.oldEntry && (cacheEntry.id = txn.template get<0>(entry->name, *cacheEntry.entry))) { + entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry)); } // update cache entry - cacheEntry.ref.packageName = &package->name; - cacheEntry.package = package; + cacheEntry.ref.entryName = &entry->name; + cacheEntry.entry = entry; // update package in storage - cacheEntry.id = txn.put(*package, cacheEntry.id); + cacheEntry.id = txn.put(*entry, cacheEntry.id); txn.commit(); res.id = cacheEntry.id; res.updated = true; return res; } -PackageCache::StoreResult PackageCache::store( - DatabaseStorage &databaseStorage, PackageStorage::RWTransaction &txn, const std::shared_ptr &package) +template +auto StorageCache::store(Storage &storage, Txn &txn, const std::shared_ptr &entry) + -> StoreResult { // check for package in cache - const auto ref = PackageCacheRef(databaseStorage, package->name); - auto res = PackageCache::StoreResult(); + const auto ref = typename Entries::Ref(storage, entry->name); + auto res = StorageCache::StoreResult(); auto lock = std::unique_lock(m_mutex); - auto &cacheEntry = m_packages.findOrCreate(ref); - if (cacheEntry.package) { + auto &cacheEntry = m_entries.findOrCreate(ref); + if (cacheEntry.entry) { // retain certain information obtained from package contents if this is actually the same package as before res.id = cacheEntry.id; - package->addDepsAndProvidesFromOtherPackage(*(res.oldPackage = cacheEntry.package)); + entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry)); } else { // check for package in storage - cacheEntry.package = std::make_shared(); - if ((cacheEntry.id = txn.get<0>(package->name, *cacheEntry.package))) { - package->addDepsAndProvidesFromOtherPackage(*(res.oldPackage = cacheEntry.package)); + cacheEntry.entry = std::make_shared(); + if ((cacheEntry.id = txn.template get<0>(entry->name, *cacheEntry.entry))) { + entry->addDepsAndProvidesFromOtherPackage(*(res.oldEntry = cacheEntry.entry)); } } // update cache entry - cacheEntry.ref.packageName = &package->name; - cacheEntry.package = package; + cacheEntry.ref.entryName = &entry->name; + cacheEntry.entry = entry; // update package in storage - res.id = cacheEntry.id = txn.put(*package, cacheEntry.id); + res.id = cacheEntry.id = txn.put(*entry, cacheEntry.id); res.updated = true; return res; } -bool PackageCache::invalidate(DatabaseStorage &databaseStorage, const std::string &packageName) +template +bool StorageCache::invalidate(Storage &storage, const std::string &entryName) { // remove package from cache - const auto ref = PackageCacheRef(databaseStorage, packageName); + const auto ref = typename Entries::Ref(storage, entryName); auto lock = std::unique_lock(m_mutex); - m_packages.erase(ref); + m_entries.erase(ref); lock.unlock(); // remove package from storage - auto txn = databaseStorage.packages.getRWTransaction(); - if (auto i = txn.find<0>(packageName); i != txn.end()) { + auto txn = storage.packages.getRWTransaction(); + if (auto i = txn.template find<0>(entryName); i != txn.end()) { i.del(); txn.commit(); return true; @@ -109,30 +139,42 @@ bool PackageCache::invalidate(DatabaseStorage &databaseStorage, const std::strin return false; } -void PackageCache::clear(DatabaseStorage &databaseStorage) +template +void StorageCache::clear(Storage &storage) { - clearCacheOnly(databaseStorage); - auto packagesTxn = databaseStorage.packages.getRWTransaction(); + clearCacheOnly(storage); + auto packagesTxn = storage.packages.getRWTransaction(); packagesTxn.clear(); packagesTxn.commit(); - auto providedDepsTxn = databaseStorage.providedDeps.getRWTransaction(); + auto providedDepsTxn = storage.providedDeps.getRWTransaction(); providedDepsTxn.clear(); providedDepsTxn.commit(); - auto requiredDepsTxn = databaseStorage.requiredDeps.getRWTransaction(); + auto requiredDepsTxn = storage.requiredDeps.getRWTransaction(); requiredDepsTxn.clear(); requiredDepsTxn.commit(); - auto providedLibsTxn = databaseStorage.providedLibs.getRWTransaction(); + auto providedLibsTxn = storage.providedLibs.getRWTransaction(); providedLibsTxn.clear(); providedLibsTxn.commit(); - auto requiredLibsTxn = databaseStorage.requiredLibs.getRWTransaction(); + auto requiredLibsTxn = storage.requiredLibs.getRWTransaction(); requiredLibsTxn.clear(); requiredLibsTxn.commit(); } -void PackageCache::clearCacheOnly(DatabaseStorage &databaseStorage) +template +void StorageCache::clearCacheOnly(Storage &storage) { const auto lock = std::unique_lock(m_mutex); - m_packages.clear(databaseStorage); + m_entries.clear(storage); +} + +template struct StorageCacheRef; +template struct StorageCacheEntry; +template class StorageCacheEntries; +template struct StorageCache; + +StorageDistribution::StorageDistribution(const char *path, std::uint32_t maxDbs) +{ + m_env = LMDBSafe::getMDBEnv(path, MDB_NOSUBDIR, 0600, maxDbs); } DatabaseStorage::DatabaseStorage(const std::shared_ptr &env, PackageCache &packageCache, std::string_view uniqueDatabaseName) @@ -150,37 +192,7 @@ std::size_t hash_value(const PackageCacheRef &ref) { const auto hasher1 = boost::hash(); const auto hasher2 = boost::hash(); - return ((hasher1(ref.databaseStorage) ^ (hasher2(*ref.packageName) << 1)) >> 1); -} - -PackageCacheEntry &RecentlyUsedPackages::findOrCreate(const PackageCacheRef &ref) -{ - const auto &index = m_packages.get(); - if (auto i = index.find(ref); i != index.end()) { - m_packages.relocate(m_packages.begin(), m_packages.project<0>(i)); - return i.get_node()->value(); - } - const auto [i, newItem] = m_packages.emplace_front(ref); - if (!newItem) { - m_packages.relocate(m_packages.begin(), i); - } else if (m_packages.size() > m_limit) { - m_packages.pop_back(); - } - return i.get_node()->value(); -} - -std::size_t RecentlyUsedPackages::clear(const DatabaseStorage &databaseStorage) -{ - auto count = std::size_t(); - for (auto i = m_packages.begin(); i != m_packages.end();) { - if (i->ref.databaseStorage == &databaseStorage) { - i = m_packages.erase(i); - ++count; - } else { - ++i; - } - } - return count; + return ((hasher1(ref.relatedStorage) ^ (hasher2(*ref.entryName) << 1)) >> 1); } } // namespace LibPkg diff --git a/libpkg/data/storageprivate.h b/libpkg/data/storageprivate.h index d15823c..96cec3f 100644 --- a/libpkg/data/storageprivate.h +++ b/libpkg/data/storageprivate.h @@ -18,132 +18,141 @@ namespace LibPkg { using StorageID = std::uint32_t; -using PackageStorage = LMDBSafe::TypedDBI>; -using DependencyStorage = LMDBSafe::TypedDBI>; -using LibraryDependencyStorage - = LMDBSafe::TypedDBI>; -struct PackageCache; - -struct DatabaseStorage { - explicit DatabaseStorage(const std::shared_ptr &env, PackageCache &packageCache, std::string_view uniqueDatabaseName); - PackageCache &packageCache; - PackageStorage packages; - DependencyStorage providedDeps; - DependencyStorage requiredDeps; - LibraryDependencyStorage providedLibs; - LibraryDependencyStorage requiredLibs; - -private: - std::shared_ptr m_env; +template struct StorageCacheRef { + using Storage = StorageType; + explicit StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr &entry); + explicit StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName); + bool operator==(const StorageCacheRef &other) const; + const StorageType *relatedStorage = nullptr; + const std::string *entryName; }; -struct PackageCacheRef { - explicit PackageCacheRef(const DatabaseStorage &databaseStorage, const std::shared_ptr &package); - explicit PackageCacheRef(const DatabaseStorage &databaseStorage, const std::string &packageName); - bool operator==(const PackageCacheRef &other) const; - const DatabaseStorage *databaseStorage = nullptr; - const std::string *packageName; -}; - -inline PackageCacheRef::PackageCacheRef(const DatabaseStorage &databaseStorage, const std::shared_ptr &package) - : databaseStorage(&databaseStorage) - , packageName(&package->name) +template +inline StorageCacheRef::StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr &entry) + : relatedStorage(&relatedStorage) + , entryName(&entry->name) { } -inline PackageCacheRef::PackageCacheRef(const DatabaseStorage &databaseStorage, const std::string &packageName) - : databaseStorage(&databaseStorage) - , packageName(&packageName) +template +inline StorageCacheRef::StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName) + : relatedStorage(&relatedStorage) + , entryName(&entryName) { } -inline bool PackageCacheRef::operator==(const PackageCacheRef &other) const +template +inline bool StorageCacheRef::operator==(const StorageCacheRef &other) const { - return databaseStorage == other.databaseStorage && *packageName == *other.packageName; + return relatedStorage == other.relatedStorage && *entryName == *other.entryName; } -std::size_t hash_value(const PackageCacheRef &ref); - -struct PackageCacheEntry { - explicit PackageCacheEntry(const PackageCacheRef &ref); - PackageCacheRef ref; +template struct StorageCacheEntry { + using Ref = StorageRefType; + using Entry = EntryType; + using Storage = typename Ref::Storage; + explicit StorageCacheEntry(const StorageRefType &ref); + StorageRefType ref; StorageID id; - std::shared_ptr package; + std::shared_ptr entry; }; -inline PackageCacheEntry::PackageCacheEntry(const PackageCacheRef &ref) +template +inline StorageCacheEntry::StorageCacheEntry(const StorageRefType &ref) : ref(ref) , id(0) { } -class RecentlyUsedPackages { - using PackageList = boost::multi_index::multi_index_container, - boost::multi_index::hashed_unique, - BOOST_MULTI_INDEX_MEMBER(PackageCacheEntry, PackageCacheRef, ref)>>>; - using iterator = PackageList::iterator; - +template class StorageCacheEntries { public: - explicit RecentlyUsedPackages(std::size_t limit = 1000); + using Ref = typename StorageEntryType::Ref; + using Entry = typename StorageEntryType::Entry; + using Storage = typename StorageEntryType::Storage; + using StorageEntry = StorageEntryType; + using EntryList = boost::multi_index::multi_index_container, + boost::multi_index::hashed_unique, BOOST_MULTI_INDEX_MEMBER(StorageEntryType, Ref, ref)>>>; + using iterator = typename EntryList::iterator; - PackageCacheEntry &findOrCreate(const PackageCacheRef &ref); + explicit StorageCacheEntries(std::size_t limit = 1000); + + StorageEntry &findOrCreate(const Ref &ref); void undo(); - std::size_t erase(const PackageCacheRef &ref); - std::size_t clear(const DatabaseStorage &databaseStorage); + std::size_t erase(const Ref &ref); + std::size_t clear(const Storage &storage); iterator begin(); iterator end(); private: - PackageList m_packages; + EntryList m_entries; std::size_t m_limit; }; -inline RecentlyUsedPackages::RecentlyUsedPackages(std::size_t limit) +template +inline StorageCacheEntries::StorageCacheEntries(std::size_t limit) : m_limit(limit) { } -inline void RecentlyUsedPackages::undo() +template inline void StorageCacheEntries::undo() { - m_packages.pop_front(); + m_entries.pop_front(); } -inline std::size_t RecentlyUsedPackages::erase(const PackageCacheRef &ref) +template inline std::size_t StorageCacheEntries::erase(const Ref &ref) { - return m_packages.get().erase(ref); + return m_entries.template get().erase(ref); } -inline RecentlyUsedPackages::iterator RecentlyUsedPackages::begin() +template inline auto StorageCacheEntries::begin() -> iterator { - return m_packages.begin(); + return m_entries.begin(); } -inline RecentlyUsedPackages::iterator RecentlyUsedPackages::end() +template inline auto StorageCacheEntries::end() -> iterator { - return m_packages.end(); + return m_entries.end(); } -struct PackageCache { +template struct StorageCache { + using Entries = StorageEntriesType; + using Entry = typename Entries::Entry; + using Txn = TransactionType; + using Storage = typename Entries::Storage; struct StoreResult { StorageID id = 0; bool updated = false; - std::shared_ptr oldPackage; + std::shared_ptr oldEntry; }; - PackageSpec retrieve(DatabaseStorage &databaseStorage, const std::string &packageName); - StoreResult store(DatabaseStorage &databaseStorage, const std::shared_ptr &package, bool force); - StoreResult store(DatabaseStorage &databaseStorage, PackageStorage::RWTransaction &txn, const std::shared_ptr &package); - bool invalidate(DatabaseStorage &databaseStorage, const std::string &packageName); - void clear(DatabaseStorage &databaseStorage); - void clearCacheOnly(DatabaseStorage &databaseStorage); + SpecType retrieve(Storage &storage, const std::string &entryName); + StoreResult store(Storage &storage, const std::shared_ptr &entry, bool force); + StoreResult store(Storage &storage, Txn &txn, const std::shared_ptr &entry); + bool invalidate(Storage &storage, const std::string &entryName); + void clear(Storage &storage); + void clearCacheOnly(Storage &storage); private: - RecentlyUsedPackages m_packages; + Entries m_entries; std::mutex m_mutex; }; +using PackageStorage = LMDBSafe::TypedDBI>; +using DependencyStorage = LMDBSafe::TypedDBI>; +using LibraryDependencyStorage + = LMDBSafe::TypedDBI>; +using PackageCacheRef = StorageCacheRef; +using PackageCacheEntry = StorageCacheEntry; +using PackageCacheEntries = StorageCacheEntries; +using PackageCache = StorageCache; + +extern template struct StorageCacheRef; +extern template struct StorageCacheEntry; +extern template class StorageCacheEntries; +extern template struct StorageCache; + struct StorageDistribution { explicit StorageDistribution(const char *path, std::uint32_t maxDbs); @@ -159,6 +168,21 @@ inline std::unique_ptr StorageDistribution::forDatabase(std::st return std::make_unique(m_env, m_packageCache, uniqueDatabaseName); } +struct DatabaseStorage { + explicit DatabaseStorage(const std::shared_ptr &env, PackageCache &packageCache, std::string_view uniqueDatabaseName); + PackageCache &packageCache; + PackageStorage packages; + DependencyStorage providedDeps; + DependencyStorage requiredDeps; + LibraryDependencyStorage providedLibs; + LibraryDependencyStorage requiredLibs; + +private: + std::shared_ptr m_env; +}; + +std::size_t hash_value(const PackageCacheRef &ref); + } // namespace LibPkg #endif // LIBPKG_DATA_STORAGE_PRIVATE_H