Limit results returned by the API for better scalability

3rdparty/lmdb-safe

@@ -1 +1 @@
-Subproject commit bf6dbf0e42c081c7b75bea9bb59ffbd0c0d6b529
+Subproject commit f6fd78155b0a4fd229b1b352fe87efed2460d982

libpkg/algo/search.cpp

@@ -97,7 +97,8 @@ Database *Config::findOrCreateDatabaseFromDenotation(std::string_view databaseDe
 /*!
  * \brief Returns all packages with the specified database name, database architecture and package name.
  */
-std::vector<PackageSearchResult> Config::findPackages(std::tuple<std::string_view, std::string_view, std::string_view> dbAndPackageName)
+std::vector<PackageSearchResult> Config::findPackages(
+    std::tuple<std::string_view, std::string_view, std::string_view> dbAndPackageName, std::size_t limit)
 {
     auto pkgs = std::vector<PackageSearchResult>();
     const auto &[dbName, dbArch, packageName] = dbAndPackageName;
@@ -115,6 +116,9 @@ std::vector<PackageSearchResult> Config::findPackages(std::tuple<std::string_vie
         if (const auto [id, package] = db.findPackageWithID(name); package) {
             pkgs.emplace_back(db, package, id);
         }
+        if (pkgs.size() >= limit) {
+            return pkgs;
+        }
     }
     return pkgs;
 }
@@ -146,7 +150,7 @@ PackageSearchResult Config::findPackage(const Dependency &dependency)
 /*!
  * \brief Returns all packages satisfying \a dependency or - if \a reverse is true - all packages requiring \a dependency.
  */
-std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependency, bool reverse)
+std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependency, bool reverse, std::size_t limit)
 {
     auto results = std::vector<PackageSearchResult>();
     for (auto &db : databases) {
@@ -155,7 +159,7 @@ std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependen
             if (visited.emplace(packageID).second) {
                 results.emplace_back(db, package, packageID);
             }
-            return false;
+            return results.size() >= limit;
         });
     }
     return results;
@@ -164,7 +168,7 @@ std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependen
 /*!
  * \brief Returns all packages providing \a library or - if \a reverse is true - all packages requiring \a library.
  */
-std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std::string &library, bool reverse)
+std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std::string &library, bool reverse, std::size_t limit)
 {
     auto results = std::vector<PackageSearchResult>();
     auto visited = std::unordered_set<StorageID>();
@@ -173,7 +177,7 @@ std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std:
             if (visited.emplace(packageID).second) {
                 results.emplace_back(db, package, packageID);
             }
-            return false;
+            return results.size() >= limit;
         });
     }
     return results;
@@ -182,7 +186,7 @@ std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std:
 /*!
  * \brief Returns all packages which names matches \a regex.
  */
-std::vector<PackageSearchResult> Config::findPackages(const std::regex &regex)
+std::vector<PackageSearchResult> Config::findPackages(const std::regex &regex, std::size_t limit)
 {
     auto pkgs = std::vector<PackageSearchResult>();
     for (auto &db : databases) {
@@ -191,13 +195,14 @@ std::vector<PackageSearchResult> Config::findPackages(const std::regex &regex)
                 auto [packageID, package] = getPackage();
                 pkgs.emplace_back(db, package, packageID);
             }
-            return false;
+            return pkgs.size() >= limit;
         });
     }
     return pkgs;
 }
 
-std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(const Database &)> &databasePred, std::string_view term)
+std::vector<PackageSearchResult> Config::findPackages(
+    const std::function<bool(const Database &)> &databasePred, std::string_view term, std::size_t limit)
 {
     auto pkgs = std::vector<PackageSearchResult>();
     for (auto &db : databases) {
@@ -209,7 +214,7 @@ std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(c
                 const auto [packageID, package] = getPackage();
                 pkgs.emplace_back(db, package, packageID);
             }
-            return false;
+            return pkgs.size() >= limit;
         });
     }
     return pkgs;
@@ -219,13 +224,16 @@ std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(c
  * \brief Returns all packages considered "the same" as \a package.
  * \remarks See Package::isSame().
  */
-std::vector<PackageSearchResult> Config::findPackages(const Package &package)
+std::vector<PackageSearchResult> Config::findPackages(const Package &package, std::size_t limit)
 {
     auto pkgs = std::vector<PackageSearchResult>();
     for (auto &db : databases) {
         if (const auto [id, pkg] = db.findPackageWithID(package.name); pkg && pkg->isSame(package)) {
             pkgs.emplace_back(db, pkg, id);
         }
+        if (pkgs.size() >= limit) {
+            return pkgs;
+        }
     }
     return pkgs;
 }
@@ -233,8 +241,8 @@ std::vector<PackageSearchResult> Config::findPackages(const Package &package)
 /*!
  * \brief Returns all packages \a packagePred returns true for from all databases \a databasePred returns true for.
  */
-std::vector<PackageSearchResult> Config::findPackages(
+std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(const Database &)> &databasePred,
-    const std::function<bool(const Database &)> &databasePred, const std::function<bool(const Database &, const Package &)> &packagePred)
+    const std::function<bool(const Database &, const Package &)> &packagePred, std::size_t limit)
 {
     auto pkgs = std::vector<PackageSearchResult>();
     for (auto &db : databases) {
@@ -245,7 +253,7 @@ std::vector<PackageSearchResult> Config::findPackages(
             if (packagePred(db, *package)) {
                 pkgs.emplace_back(db, std::move(package), packageID);
             }
-            return false;
+            return pkgs.size() >= limit;
         });
     }
     return pkgs;
@@ -254,7 +262,7 @@ std::vector<PackageSearchResult> Config::findPackages(
 /*!
  * \brief Returns all packages \a pred returns true for.
  */
-std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(const Database &, const Package &)> &pred)
+std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(const Database &, const Package &)> &pred, std::size_t limit)
 {
     auto pkgs = std::vector<PackageSearchResult>();
     for (auto &db : databases) {
@@ -262,7 +270,7 @@ std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(c
             if (pred(db, *package)) {
                 pkgs.emplace_back(db, std::move(package), packageID);
             }
-            return false;
+            return pkgs.size() >= limit;
         });
     }
     return pkgs;

libpkg/data/config.h

@@ -140,18 +140,24 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar
     Database *findOrCreateDatabase(std::string_view name, std::string_view architecture);
     Database *findOrCreateDatabaseFromDenotation(std::string_view databaseDenotation);
     static std::tuple<std::string_view, std::string_view, std::string_view> parsePackageDenotation(std::string_view packageDenotation);
-    std::vector<PackageSearchResult> findPackages(std::string_view packageDenotation);
+    std::vector<PackageSearchResult> findPackages(std::string_view packageDenotation, std::size_t limit = std::numeric_limits<std::size_t>::max());
-    std::vector<PackageSearchResult> findPackages(std::string_view dbName, std::string_view dbArch, std::string_view packageName);
-    std::vector<PackageSearchResult> findPackages(std::tuple<std::string_view, std::string_view, std::string_view> dbAndPackageName);
-    PackageSearchResult findPackage(const Dependency &dependency);
-    std::vector<PackageSearchResult> findPackages(const Dependency &dependency, bool reverse = false);
-    std::vector<PackageSearchResult> findPackagesProvidingLibrary(const std::string &library, bool reverse = false);
-    std::vector<PackageSearchResult> findPackages(const std::regex &regex);
-    std::vector<PackageSearchResult> findPackages(const std::function<bool(const Database &)> &databasePred, std::string_view term);
-    std::vector<PackageSearchResult> findPackages(const Package &package);
     std::vector<PackageSearchResult> findPackages(
-        const std::function<bool(const Database &)> &databasePred, const std::function<bool(const Database &, const Package &)> &packagePred);
+        std::string_view dbName, std::string_view dbArch, std::string_view packageName, std::size_t limit = std::numeric_limits<std::size_t>::max());
-    std::vector<PackageSearchResult> findPackages(const std::function<bool(const Database &, const Package &)> &pred);
+    std::vector<PackageSearchResult> findPackages(std::tuple<std::string_view, std::string_view, std::string_view> dbAndPackageName,
+        std::size_t limit = std::numeric_limits<std::size_t>::max());
+    PackageSearchResult findPackage(const Dependency &dependency);
+    std::vector<PackageSearchResult> findPackages(
+        const Dependency &dependency, bool reverse = false, std::size_t limit = std::numeric_limits<std::size_t>::max());
+    std::vector<PackageSearchResult> findPackagesProvidingLibrary(
+        const std::string &library, bool reverse = false, std::size_t limit = std::numeric_limits<std::size_t>::max());
+    std::vector<PackageSearchResult> findPackages(const std::regex &regex, std::size_t limit = std::numeric_limits<std::size_t>::max());
+    std::vector<PackageSearchResult> findPackages(const std::function<bool(const Database &)> &databasePred, std::string_view term,
+        std::size_t limit = std::numeric_limits<std::size_t>::max());
+    std::vector<PackageSearchResult> findPackages(const Package &package, std::size_t limit = std::numeric_limits<std::size_t>::max());
+    std::vector<PackageSearchResult> findPackages(const std::function<bool(const Database &)> &databasePred,
+        const std::function<bool(const Database &, const Package &)> &packagePred, std::size_t limit = std::numeric_limits<std::size_t>::max());
+    std::vector<PackageSearchResult> findPackages(
+        const std::function<bool(const Database &, const Package &)> &pred, std::size_t limit = std::numeric_limits<std::size_t>::max());
 
     std::vector<Database> databases;
     Database aur = Database("aur");
@@ -181,14 +187,15 @@ inline Status Config::computeStatus() const
     return Status(*this);
 }
 
-inline std::vector<PackageSearchResult> Config::findPackages(std::string_view dbName, std::string_view dbArch, std::string_view packageName)
+inline std::vector<PackageSearchResult> Config::findPackages(
+    std::string_view dbName, std::string_view dbArch, std::string_view packageName, std::size_t limit)
 {
-    return findPackages(std::make_tuple(dbName, dbArch, packageName));
+    return findPackages(std::make_tuple(dbName, dbArch, packageName), limit);
 }
 
-inline std::vector<PackageSearchResult> Config::findPackages(std::string_view packageDenotation)
+inline std::vector<PackageSearchResult> Config::findPackages(std::string_view packageDenotation, std::size_t limit)
 {
-    return findPackages(parsePackageDenotation(packageDenotation));
+    return findPackages(parsePackageDenotation(packageDenotation), limit);
 }
 
 } // namespace LibPkg

librepomgr/helper.h

@@ -39,7 +39,7 @@ inline std::optional<std::string_view> getLastValueSv(const std::multimap<std::s
     return std::nullopt;
 }
 
-template <typename TargetType, Traits::DisableIf<std::is_integral<TargetType>> * = nullptr>
+template <typename TargetType, Traits::DisableIfAny<std::is_integral<TargetType>, Traits::IsSpecializationOf<TargetType, std::atomic>> * = nullptr>
 void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result);
 
 template <>
@@ -60,7 +60,7 @@ inline void convertValue(const std::multimap<std::string, std::string> &multimap
     }
 }
 
-template <typename TargetType, Traits::EnableIf<std::is_integral<TargetType>> * = nullptr>
+template <typename TargetType, Traits::EnableIfAny<std::is_integral<TargetType>, Traits::IsSpecializationOf<TargetType, std::atomic>> * = nullptr>
 inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result)
 {
     using namespace std;
@@ -69,7 +69,11 @@ inline void convertValue(const std::multimap<std::string, std::string> &multimap
 
     if (const char *const value = getLastValue(multimap, key)) {
         try {
-            result = stringToNumber<TargetType>(value);
+            if constexpr (Traits::IsSpecializationOf<TargetType, std::atomic>::value) {
+                result = stringToNumber<typename TargetType::value_type>(value);
+            } else {
+                result = stringToNumber<TargetType>(value);
+            }
         } catch (const ConversionException &) {
             cerr << Phrases::ErrorMessage << "Specified number \"" << value << "\" for key \"" << key << "\" is invalid." << Phrases::End;
             return;

librepomgr/serversetup.cpp

@@ -90,6 +90,8 @@ void ServiceSetup::WebServerSetup::applyConfig(const std::multimap<std::string,
     convertValue(multimap, "port", port);
     convertValue(multimap, "threads", threadCount);
     convertValue(multimap, "static_files", staticFilesPath);
+    convertValue(multimap, "package_search_response_limit", packageSearchResponseLimit);
+    convertValue(multimap, "build_actions_response_limit", buildActionsResponseLimit);
     convertValue(multimap, "verify_ssl_certificates", verifySslCertificates);
     convertValue(multimap, "log_ssl_certificate_validation", logSslCertificateValidation);
 
@@ -338,11 +340,16 @@ std::size_t ServiceSetup::BuildSetup::buildActionCount()
 }
 
 void ServiceSetup::BuildSetup::forEachBuildAction(
-    std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func)
+    std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func, std::size_t limit, std::size_t start)
 {
     auto txn = m_storage->buildActions.getROTransaction();
-    count(txn.size());
+    const auto total = txn.size();
-    for (auto i = txn.begin(); i != txn.end(); ++i) {
+    count(std::min(limit, total));
+    const auto reverse = start == std::numeric_limits<std::size_t>::max();
+    for (auto i = reverse ? txn.rbegin()
+                          : txn.lower_bound(static_cast<LibPkg::StorageID>(
+                              start > std::numeric_limits<LibPkg::StorageID>::max() ? std::numeric_limits<LibPkg::StorageID>::max() : start));
+         i != txn.end() && limit; reverse ? --i : ++i, --limit) {
         if (func(i.getID(), std::move(i.value()))) {
             return;
         }

librepomgr/serversetup.h

@@ -68,6 +68,8 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
         unsigned short threadCount = 1;
         boost::asio::io_context ioContext;
         boost::asio::ssl::context sslContext{ boost::asio::ssl::context::sslv23_client };
+        std::atomic_size_t packageSearchResponseLimit = 20000; // sufficient to return a "full architecture"
+        std::atomic_size_t buildActionsResponseLimit = 200;
         bool verifySslCertificates = true;
         bool logSslCertificateValidation = false;
 
@@ -138,7 +140,8 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
         void deleteBuildAction(const std::vector<std::shared_ptr<BuildAction>> &actions);
         std::size_t buildActionCount();
         std::size_t runningBuildActionCount();
-        void forEachBuildAction(std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func);
+        void forEachBuildAction(std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func,
+            std::size_t limit, std::size_t start);
         void forEachBuildAction(std::function<bool(LibPkg::StorageID, BuildAction &, bool &)> &&func);
         std::vector<std::shared_ptr<BuildAction>> followUpBuildActions(BuildActionIdType forId);
 

librepomgr/webapi/params.h

@@ -6,6 +6,9 @@
 #include "./session.h"
 #include "./typedefs.h"
 
+#include <c++utilities/conversion/stringbuilder.h>
+#include <c++utilities/conversion/stringconversion.h>
+
 #include <unordered_map>
 
 namespace std {
@@ -51,10 +54,28 @@ struct LIBREPOMGR_EXPORT Url {
     bool hasPrettyFlag() const;
     std::string_view value(std::string_view paramName) const;
     std::vector<std::string> decodeValues(std::string_view paramName) const;
+    template <typename Number> Number asNumber(std::string_view paramName, Number def = Number()) const;
     static std::string decodeValue(std::string_view value);
     static std::string encodeValue(std::string_view value);
 };
 
+template <typename Number> Number Url::asNumber(std::string_view paramName, Number def) const
+{
+    using namespace CppUtilities;
+    const auto values = decodeValues(paramName);
+    if (values.size() > 1) {
+        throw BadRequest("more than one " % paramName + " specified");
+    }
+    if (!values.empty()) {
+        try {
+            return stringToNumber<Number>(values.front());
+        } catch (const ConversionException &) {
+            throw BadRequest(argsToString(paramName, " must be an integer"));
+        }
+    }
+    return def;
+}
+
 inline bool Url::hasPrettyFlag() const
 {
     return hasFlag("pretty");

librepomgr/webapi/routes.cpp

@@ -150,7 +150,7 @@ void getUnresolved(const Params &params, ResponseHandler &&handler)
 void getPackages(const Params &params, ResponseHandler &&handler)
 {
     // read mode
-    const auto modes(params.target.decodeValues("mode"));
+    const auto modes = params.target.decodeValues("mode");
     if (modes.size() > 1) {
         throw BadRequest("more than one mode specified");
     }
@@ -181,6 +181,13 @@ void getPackages(const Params &params, ResponseHandler &&handler)
         mode = modeIterator->second;
     }
 
+    // limit
+    auto limit = params.target.asNumber<std::size_t>("limit");
+    const auto serverLimit = params.setup.webServer.packageSearchResponseLimit.load();
+    if (!limit || limit > serverLimit) {
+        limit = serverLimit;
+    }
+
     // check for details flag
     const auto details = params.target.hasFlag("details");
     if (details && mode != Mode::Name) {
@@ -214,7 +221,8 @@ void getPackages(const Params &params, ResponseHandler &&handler)
     RAPIDJSON_NAMESPACE::Document document(RAPIDJSON_NAMESPACE::kArrayType);
     RAPIDJSON_NAMESPACE::Document::Array array(document.GetArray());
 
-    const auto pushPackages = [&dbs, &document, &array](auto &&packages) {
+    const auto pushPackages = [&dbs, &document, &array, &limit](auto &&packages) {
+        limit -= packages.size();
         for (const auto &package : packages) {
             if (!dbs.empty()) {
                 const auto *const db = std::get<Database *>(package.db);
@@ -248,15 +256,17 @@ void getPackages(const Params &params, ResponseHandler &&handler)
                 } else {
                     neededAurPackages.emplace_back(std::move(packageNameStr));
                 }
+                --limit;
             }
             if (!isDbAur && (!dbs.empty() || !onlyFromAur)) {
-                auto packages = params.setup.config.findPackages(packageDenotation);
+                auto packages = params.setup.config.findPackages(packageDenotation, limit);
                 if (details) {
                     for (const auto &package : packages) {
                         if (dbs.empty() || dbs.find(std::get<LibPkg::Database *>(package.db)->name) != dbs.end()) {
                             ReflectiveRapidJSON::JsonReflector::push(package.pkg, array, document.GetAllocator());
                         }
                     }
+                    limit -= packages.size();
                 } else {
                     pushPackages(std::move(packages));
                 }
@@ -265,7 +275,8 @@ void getPackages(const Params &params, ResponseHandler &&handler)
         }
         case Mode::NameContains:
             pushPackages(params.setup.config.findPackages(
-                [&dbs, onlyFromAur](const LibPkg::Database &db) { return (dbs.empty() && !onlyFromAur) || dbs.find(db.name) != dbs.end(); }, name));
+                [&dbs, onlyFromAur](const LibPkg::Database &db) { return (dbs.empty() && !onlyFromAur) || dbs.find(db.name) != dbs.end(); }, name,
+                limit));
             if (fromAur && !name.empty()) {
                 neededAurPackages.emplace_back(std::move(name));
             }
@@ -273,7 +284,7 @@ void getPackages(const Params &params, ResponseHandler &&handler)
         case Mode::Regex:
             // assume names are regexes
             try {
-                pushPackages(params.setup.config.findPackages(std::regex(name.data(), name.size())));
+                pushPackages(params.setup.config.findPackages(std::regex(name.data(), name.size()), limit));
             } catch (const std::regex_error &e) {
                 throw BadRequest(argsToString("regex is invalid: ", e.what()));
             }
@@ -281,12 +292,12 @@ void getPackages(const Params &params, ResponseHandler &&handler)
         case Mode::Provides:
         case Mode::Depends:
             // assume names are dependency notation
-            pushPackages(params.setup.config.findPackages(Dependency::fromString(name), mode == Mode::Depends));
+            pushPackages(params.setup.config.findPackages(Dependency::fromString(name), mode == Mode::Depends, limit));
             break;
         case Mode::LibProvides:
         case Mode::LibDepends:
             // assume names are "normalized" library names with platform prefix
-            pushPackages(params.setup.config.findPackagesProvidingLibrary(name, mode == Mode::LibDepends));
+            pushPackages(params.setup.config.findPackagesProvidingLibrary(name, mode == Mode::LibDepends, limit));
             break;
         default:;
         }

librepomgr/webapi/routes_buildaction.cpp

@@ -23,13 +23,21 @@ void getBuildActions(const Params &params, ResponseHandler &&handler)
     RAPIDJSON_NAMESPACE::Document jsonDoc(RAPIDJSON_NAMESPACE::kArrayType);
     RAPIDJSON_NAMESPACE::Value::Array array(jsonDoc.GetArray());
 
+    const auto start = params.target.asNumber<std::size_t>("start", std::numeric_limits<std::size_t>::max());
+    const auto serverLimit = params.setup.webServer.buildActionsResponseLimit.load();
+    auto limit = params.target.asNumber<std::size_t>("limit");
+    if (!limit || limit > serverLimit) {
+        limit = serverLimit;
+    }
+
     auto buildActionLock = params.setup.building.lockToRead();
     params.setup.building.forEachBuildAction(
         [&array, &jsonDoc](std::size_t count) { array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(count), jsonDoc.GetAllocator()); },
-        [&array, &jsonDoc](LibPkg::StorageID, BuildAction &&buildAction) {
+        [&array, &jsonDoc, limit](LibPkg::StorageID, BuildAction &&buildAction) {
             ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(buildAction), array, jsonDoc.GetAllocator());
-            return false;
+            return array.Size() >= limit;
-        });
+        },
+        limit, start);
     buildActionLock.unlock();
 
     handler(makeJson(params.request(), jsonDoc, params.target.hasPrettyFlag()));