From a7de520549ca2e9b14faa4a2ccd9c58f27cd689f Mon Sep 17 00:00:00 2001 From: Martchus Date: Sat, 19 Feb 2022 00:11:59 +0100 Subject: [PATCH] Use lazy-deserialization and package cache when searching in package name --- 3rdparty/lmdb-safe | 2 +- libpkg/algo/search.cpp | 23 +++++++++++++++++++++-- libpkg/data/config.h | 1 + libpkg/data/database.cpp | 16 +++++++++++++--- libpkg/data/database.h | 2 ++ librepomgr/webapi/routes.cpp | 3 +-- 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/3rdparty/lmdb-safe b/3rdparty/lmdb-safe index b302cdf..b87dd49 160000 --- a/3rdparty/lmdb-safe +++ b/3rdparty/lmdb-safe @@ -1 +1 @@ -Subproject commit b302cdfabe190ad4b1853a6e8b1837886c58dd4a +Subproject commit b87dd49bbaef54d71c3cd25e46db0ff86e73f469 diff --git a/libpkg/algo/search.cpp b/libpkg/algo/search.cpp index bc5c2b9..ebbbb2d 100644 --- a/libpkg/algo/search.cpp +++ b/libpkg/algo/search.cpp @@ -186,8 +186,27 @@ std::vector Config::findPackages(const std::regex ®ex) { auto pkgs = std::vector(); for (auto &db : databases) { - db.allPackages([&](StorageID packageID, const std::shared_ptr &package) { - if (std::regex_match(package->name, regex)) { + db.allPackagesByName([&](std::string_view packageName, const std::function &getPackage) { + if (std::regex_match(packageName.begin(), packageName.end(), regex)) { + auto [packageID, package] = getPackage(); + pkgs.emplace_back(db, package, packageID); + } + return false; + }); + } + return pkgs; +} + +std::vector Config::findPackages(const std::function &databasePred, std::string_view term) +{ + auto pkgs = std::vector(); + for (auto &db : databases) { + if (!databasePred(db)) { + continue; + } + db.allPackagesByName([&](std::string_view packageName, const std::function &getPackage) { + if (packageName.find(term) != std::string_view::npos) { + const auto [packageID, package] = getPackage(); pkgs.emplace_back(db, package, packageID); } return false; diff --git a/libpkg/data/config.h b/libpkg/data/config.h index cfd5b82..112a14f 100644 --- a/libpkg/data/config.h +++ b/libpkg/data/config.h @@ -147,6 +147,7 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar std::vector findPackages(const Dependency &dependency, bool reverse = false); std::vector findPackagesProvidingLibrary(const std::string &library, bool reverse = false); std::vector findPackages(const std::regex ®ex); + std::vector findPackages(const std::function &databasePred, std::string_view term); std::vector findPackages(const Package &package); std::vector findPackages( const std::function &databasePred, const std::function &packagePred); diff --git a/libpkg/data/database.cpp b/libpkg/data/database.cpp index c4a1e43..066df3c 100644 --- a/libpkg/data/database.cpp +++ b/libpkg/data/database.cpp @@ -104,12 +104,11 @@ void Database::clearPackages() std::vector> Database::findPackages(const std::function &pred) { // TODO: use cache here - // TODO: avoid std::move() auto pkgs = std::vector>(); auto txn = m_storage->packages.getROTransaction(); - for (auto i = txn.begin(); i != txn.end(); ++i) { + for (auto i = txn.begin(); i != txn.end(); ++i) { if (pred(*this, *i)) { - pkgs.emplace_back(std::make_shared(std::move(*i))); + pkgs.emplace_back(std::move(i.getPointer())); } } return pkgs; @@ -257,6 +256,17 @@ void Database::allPackages(const PackageVisitor &visitor) } } +void LibPkg::Database::allPackagesByName(const PackageVisitorByName &visitor) +{ + auto txn = m_storage->packages.getROTransaction(); + for (auto i = txn.begin_idx<0, std::shared_ptr>(); i != txn.end(); ++i) { + const auto packageName = i.getKey().get(); + if (visitor(packageName, [this, &txn, &i]() { return m_storage->packageCache.retrieve(*m_storage, &txn, i.value()); })) { + return; + } + } +} + std::size_t Database::packageCount() const { return m_storage->packages.getROTransaction().size(); diff --git a/libpkg/data/database.h b/libpkg/data/database.h index dd79161..348fe4b 100644 --- a/libpkg/data/database.h +++ b/libpkg/data/database.h @@ -113,6 +113,7 @@ private: struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable, public ReflectiveRapidJSON::BinarySerializable { using PackageVisitor = std::function &)>; + using PackageVisitorByName = std::function &)>; friend struct PackageUpdater; @@ -134,6 +135,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable &package); void addPackageDependencies(StorageID packageID, const std::shared_ptr &package); void allPackages(const PackageVisitor &visitor); + void allPackagesByName(const PackageVisitorByName &visitor); std::size_t packageCount() const; void providingPackages(const Dependency &dependency, bool reverse, const PackageVisitor &visitor); void providingPackages(const std::string &libraryName, bool reverse, const PackageVisitor &visitor); diff --git a/librepomgr/webapi/routes.cpp b/librepomgr/webapi/routes.cpp index 25ad144..7ca1d0c 100644 --- a/librepomgr/webapi/routes.cpp +++ b/librepomgr/webapi/routes.cpp @@ -265,8 +265,7 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) } case Mode::NameContains: pushPackages(params.setup.config.findPackages( - [&dbs, onlyFromAur](const LibPkg::Database &db) { return (dbs.empty() && !onlyFromAur) || dbs.find(db.name) != dbs.end(); }, - [&name](const LibPkg::Database &, const LibPkg::Package &pkg) { return pkg.name.find(name) != std::string::npos; })); + [&dbs, onlyFromAur](const LibPkg::Database &db) { return (dbs.empty() && !onlyFromAur) || dbs.find(db.name) != dbs.end(); }, name)); if (fromAur && !name.empty()) { neededAurPackages.emplace_back(std::move(name)); }