diff --git a/3rdparty/lmdb-safe b/3rdparty/lmdb-safe index bf6dbf0..f6fd781 160000 --- a/3rdparty/lmdb-safe +++ b/3rdparty/lmdb-safe @@ -1 +1 @@ -Subproject commit bf6dbf0e42c081c7b75bea9bb59ffbd0c0d6b529 +Subproject commit f6fd78155b0a4fd229b1b352fe87efed2460d982 diff --git a/libpkg/algo/search.cpp b/libpkg/algo/search.cpp index 6f2708c..145dd41 100644 --- a/libpkg/algo/search.cpp +++ b/libpkg/algo/search.cpp @@ -97,7 +97,8 @@ Database *Config::findOrCreateDatabaseFromDenotation(std::string_view databaseDe /*! * \brief Returns all packages with the specified database name, database architecture and package name. */ -std::vector Config::findPackages(std::tuple dbAndPackageName) +std::vector Config::findPackages( + std::tuple dbAndPackageName, std::size_t limit) { auto pkgs = std::vector(); const auto &[dbName, dbArch, packageName] = dbAndPackageName; @@ -115,6 +116,9 @@ std::vector Config::findPackages(std::tuple= limit) { + return pkgs; + } } return pkgs; } @@ -146,7 +150,7 @@ PackageSearchResult Config::findPackage(const Dependency &dependency) /*! * \brief Returns all packages satisfying \a dependency or - if \a reverse is true - all packages requiring \a dependency. */ -std::vector Config::findPackages(const Dependency &dependency, bool reverse) +std::vector Config::findPackages(const Dependency &dependency, bool reverse, std::size_t limit) { auto results = std::vector(); for (auto &db : databases) { @@ -155,7 +159,7 @@ std::vector Config::findPackages(const Dependency &dependen if (visited.emplace(packageID).second) { results.emplace_back(db, package, packageID); } - return false; + return results.size() >= limit; }); } return results; @@ -164,7 +168,7 @@ std::vector Config::findPackages(const Dependency &dependen /*! * \brief Returns all packages providing \a library or - if \a reverse is true - all packages requiring \a library. */ -std::vector Config::findPackagesProvidingLibrary(const std::string &library, bool reverse) +std::vector Config::findPackagesProvidingLibrary(const std::string &library, bool reverse, std::size_t limit) { auto results = std::vector(); auto visited = std::unordered_set(); @@ -173,7 +177,7 @@ std::vector Config::findPackagesProvidingLibrary(const std: if (visited.emplace(packageID).second) { results.emplace_back(db, package, packageID); } - return false; + return results.size() >= limit; }); } return results; @@ -182,7 +186,7 @@ std::vector Config::findPackagesProvidingLibrary(const std: /*! * \brief Returns all packages which names matches \a regex. */ -std::vector Config::findPackages(const std::regex ®ex) +std::vector Config::findPackages(const std::regex ®ex, std::size_t limit) { auto pkgs = std::vector(); for (auto &db : databases) { @@ -191,13 +195,14 @@ std::vector Config::findPackages(const std::regex ®ex) auto [packageID, package] = getPackage(); pkgs.emplace_back(db, package, packageID); } - return false; + return pkgs.size() >= limit; }); } return pkgs; } -std::vector Config::findPackages(const std::function &databasePred, std::string_view term) +std::vector Config::findPackages( + const std::function &databasePred, std::string_view term, std::size_t limit) { auto pkgs = std::vector(); for (auto &db : databases) { @@ -209,7 +214,7 @@ std::vector Config::findPackages(const std::function= limit; }); } return pkgs; @@ -219,13 +224,16 @@ std::vector Config::findPackages(const std::function Config::findPackages(const Package &package) +std::vector Config::findPackages(const Package &package, std::size_t limit) { auto pkgs = std::vector(); for (auto &db : databases) { if (const auto [id, pkg] = db.findPackageWithID(package.name); pkg && pkg->isSame(package)) { pkgs.emplace_back(db, pkg, id); } + if (pkgs.size() >= limit) { + return pkgs; + } } return pkgs; } @@ -233,8 +241,8 @@ std::vector Config::findPackages(const Package &package) /*! * \brief Returns all packages \a packagePred returns true for from all databases \a databasePred returns true for. */ -std::vector Config::findPackages( - const std::function &databasePred, const std::function &packagePred) +std::vector Config::findPackages(const std::function &databasePred, + const std::function &packagePred, std::size_t limit) { auto pkgs = std::vector(); for (auto &db : databases) { @@ -245,7 +253,7 @@ std::vector Config::findPackages( if (packagePred(db, *package)) { pkgs.emplace_back(db, std::move(package), packageID); } - return false; + return pkgs.size() >= limit; }); } return pkgs; @@ -254,7 +262,7 @@ std::vector Config::findPackages( /*! * \brief Returns all packages \a pred returns true for. */ -std::vector Config::findPackages(const std::function &pred) +std::vector Config::findPackages(const std::function &pred, std::size_t limit) { auto pkgs = std::vector(); for (auto &db : databases) { @@ -262,7 +270,7 @@ std::vector Config::findPackages(const std::function= limit; }); } return pkgs; diff --git a/libpkg/data/config.h b/libpkg/data/config.h index 112a14f..fe057ec 100644 --- a/libpkg/data/config.h +++ b/libpkg/data/config.h @@ -140,18 +140,24 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar Database *findOrCreateDatabase(std::string_view name, std::string_view architecture); Database *findOrCreateDatabaseFromDenotation(std::string_view databaseDenotation); static std::tuple parsePackageDenotation(std::string_view packageDenotation); - std::vector findPackages(std::string_view packageDenotation); - std::vector findPackages(std::string_view dbName, std::string_view dbArch, std::string_view packageName); - std::vector findPackages(std::tuple dbAndPackageName); - PackageSearchResult findPackage(const Dependency &dependency); - std::vector findPackages(const Dependency &dependency, bool reverse = false); - std::vector findPackagesProvidingLibrary(const std::string &library, bool reverse = false); - std::vector findPackages(const std::regex ®ex); - std::vector findPackages(const std::function &databasePred, std::string_view term); - std::vector findPackages(const Package &package); + std::vector findPackages(std::string_view packageDenotation, std::size_t limit = std::numeric_limits::max()); std::vector findPackages( - const std::function &databasePred, const std::function &packagePred); - std::vector findPackages(const std::function &pred); + std::string_view dbName, std::string_view dbArch, std::string_view packageName, std::size_t limit = std::numeric_limits::max()); + std::vector findPackages(std::tuple dbAndPackageName, + std::size_t limit = std::numeric_limits::max()); + PackageSearchResult findPackage(const Dependency &dependency); + std::vector findPackages( + const Dependency &dependency, bool reverse = false, std::size_t limit = std::numeric_limits::max()); + std::vector findPackagesProvidingLibrary( + const std::string &library, bool reverse = false, std::size_t limit = std::numeric_limits::max()); + std::vector findPackages(const std::regex ®ex, std::size_t limit = std::numeric_limits::max()); + std::vector findPackages(const std::function &databasePred, std::string_view term, + std::size_t limit = std::numeric_limits::max()); + std::vector findPackages(const Package &package, std::size_t limit = std::numeric_limits::max()); + std::vector findPackages(const std::function &databasePred, + const std::function &packagePred, std::size_t limit = std::numeric_limits::max()); + std::vector findPackages( + const std::function &pred, std::size_t limit = std::numeric_limits::max()); std::vector databases; Database aur = Database("aur"); @@ -181,14 +187,15 @@ inline Status Config::computeStatus() const return Status(*this); } -inline std::vector Config::findPackages(std::string_view dbName, std::string_view dbArch, std::string_view packageName) +inline std::vector Config::findPackages( + std::string_view dbName, std::string_view dbArch, std::string_view packageName, std::size_t limit) { - return findPackages(std::make_tuple(dbName, dbArch, packageName)); + return findPackages(std::make_tuple(dbName, dbArch, packageName), limit); } -inline std::vector Config::findPackages(std::string_view packageDenotation) +inline std::vector Config::findPackages(std::string_view packageDenotation, std::size_t limit) { - return findPackages(parsePackageDenotation(packageDenotation)); + return findPackages(parsePackageDenotation(packageDenotation), limit); } } // namespace LibPkg diff --git a/librepomgr/helper.h b/librepomgr/helper.h index fa58c9b..31042e0 100644 --- a/librepomgr/helper.h +++ b/librepomgr/helper.h @@ -39,7 +39,7 @@ inline std::optional getLastValueSv(const std::multimap> * = nullptr> +template , Traits::IsSpecializationOf> * = nullptr> void convertValue(const std::multimap &multimap, const std::string &key, TargetType &result); template <> @@ -60,7 +60,7 @@ inline void convertValue(const std::multimap &multimap } } -template > * = nullptr> +template , Traits::IsSpecializationOf> * = nullptr> inline void convertValue(const std::multimap &multimap, const std::string &key, TargetType &result) { using namespace std; @@ -69,7 +69,11 @@ inline void convertValue(const std::multimap &multimap if (const char *const value = getLastValue(multimap, key)) { try { - result = stringToNumber(value); + if constexpr (Traits::IsSpecializationOf::value) { + result = stringToNumber(value); + } else { + result = stringToNumber(value); + } } catch (const ConversionException &) { cerr << Phrases::ErrorMessage << "Specified number \"" << value << "\" for key \"" << key << "\" is invalid." << Phrases::End; return; diff --git a/librepomgr/serversetup.cpp b/librepomgr/serversetup.cpp index d78dd50..62a28f9 100644 --- a/librepomgr/serversetup.cpp +++ b/librepomgr/serversetup.cpp @@ -90,6 +90,8 @@ void ServiceSetup::WebServerSetup::applyConfig(const std::multimap count, std::function &&func) + std::function count, std::function &&func, std::size_t limit, std::size_t start) { auto txn = m_storage->buildActions.getROTransaction(); - count(txn.size()); - for (auto i = txn.begin(); i != txn.end(); ++i) { + const auto total = txn.size(); + count(std::min(limit, total)); + const auto reverse = start == std::numeric_limits::max(); + for (auto i = reverse ? txn.rbegin() + : txn.lower_bound(static_cast( + start > std::numeric_limits::max() ? std::numeric_limits::max() : start)); + i != txn.end() && limit; reverse ? --i : ++i, --limit) { if (func(i.getID(), std::move(i.value()))) { return; } diff --git a/librepomgr/serversetup.h b/librepomgr/serversetup.h index 0b5983f..192942f 100644 --- a/librepomgr/serversetup.h +++ b/librepomgr/serversetup.h @@ -68,6 +68,8 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { unsigned short threadCount = 1; boost::asio::io_context ioContext; boost::asio::ssl::context sslContext{ boost::asio::ssl::context::sslv23_client }; + std::atomic_size_t packageSearchResponseLimit = 20000; // sufficient to return a "full architecture" + std::atomic_size_t buildActionsResponseLimit = 200; bool verifySslCertificates = true; bool logSslCertificateValidation = false; @@ -138,7 +140,8 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { void deleteBuildAction(const std::vector> &actions); std::size_t buildActionCount(); std::size_t runningBuildActionCount(); - void forEachBuildAction(std::function count, std::function &&func); + void forEachBuildAction(std::function count, std::function &&func, + std::size_t limit, std::size_t start); void forEachBuildAction(std::function &&func); std::vector> followUpBuildActions(BuildActionIdType forId); diff --git a/librepomgr/webapi/params.h b/librepomgr/webapi/params.h index b8f6a22..a440f4d 100644 --- a/librepomgr/webapi/params.h +++ b/librepomgr/webapi/params.h @@ -6,6 +6,9 @@ #include "./session.h" #include "./typedefs.h" +#include +#include + #include namespace std { @@ -51,10 +54,28 @@ struct LIBREPOMGR_EXPORT Url { bool hasPrettyFlag() const; std::string_view value(std::string_view paramName) const; std::vector decodeValues(std::string_view paramName) const; + template Number asNumber(std::string_view paramName, Number def = Number()) const; static std::string decodeValue(std::string_view value); static std::string encodeValue(std::string_view value); }; +template Number Url::asNumber(std::string_view paramName, Number def) const +{ + using namespace CppUtilities; + const auto values = decodeValues(paramName); + if (values.size() > 1) { + throw BadRequest("more than one " % paramName + " specified"); + } + if (!values.empty()) { + try { + return stringToNumber(values.front()); + } catch (const ConversionException &) { + throw BadRequest(argsToString(paramName, " must be an integer")); + } + } + return def; +} + inline bool Url::hasPrettyFlag() const { return hasFlag("pretty"); diff --git a/librepomgr/webapi/routes.cpp b/librepomgr/webapi/routes.cpp index 7ca1d0c..3a11a93 100644 --- a/librepomgr/webapi/routes.cpp +++ b/librepomgr/webapi/routes.cpp @@ -150,7 +150,7 @@ void getUnresolved(const Params ¶ms, ResponseHandler &&handler) void getPackages(const Params ¶ms, ResponseHandler &&handler) { // read mode - const auto modes(params.target.decodeValues("mode")); + const auto modes = params.target.decodeValues("mode"); if (modes.size() > 1) { throw BadRequest("more than one mode specified"); } @@ -181,6 +181,13 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) mode = modeIterator->second; } + // limit + auto limit = params.target.asNumber("limit"); + const auto serverLimit = params.setup.webServer.packageSearchResponseLimit.load(); + if (!limit || limit > serverLimit) { + limit = serverLimit; + } + // check for details flag const auto details = params.target.hasFlag("details"); if (details && mode != Mode::Name) { @@ -214,7 +221,8 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) RAPIDJSON_NAMESPACE::Document document(RAPIDJSON_NAMESPACE::kArrayType); RAPIDJSON_NAMESPACE::Document::Array array(document.GetArray()); - const auto pushPackages = [&dbs, &document, &array](auto &&packages) { + const auto pushPackages = [&dbs, &document, &array, &limit](auto &&packages) { + limit -= packages.size(); for (const auto &package : packages) { if (!dbs.empty()) { const auto *const db = std::get(package.db); @@ -248,15 +256,17 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) } else { neededAurPackages.emplace_back(std::move(packageNameStr)); } + --limit; } if (!isDbAur && (!dbs.empty() || !onlyFromAur)) { - auto packages = params.setup.config.findPackages(packageDenotation); + auto packages = params.setup.config.findPackages(packageDenotation, limit); if (details) { for (const auto &package : packages) { if (dbs.empty() || dbs.find(std::get(package.db)->name) != dbs.end()) { ReflectiveRapidJSON::JsonReflector::push(package.pkg, array, document.GetAllocator()); } } + limit -= packages.size(); } else { pushPackages(std::move(packages)); } @@ -265,7 +275,8 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) } case Mode::NameContains: pushPackages(params.setup.config.findPackages( - [&dbs, onlyFromAur](const LibPkg::Database &db) { return (dbs.empty() && !onlyFromAur) || dbs.find(db.name) != dbs.end(); }, name)); + [&dbs, onlyFromAur](const LibPkg::Database &db) { return (dbs.empty() && !onlyFromAur) || dbs.find(db.name) != dbs.end(); }, name, + limit)); if (fromAur && !name.empty()) { neededAurPackages.emplace_back(std::move(name)); } @@ -273,7 +284,7 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) case Mode::Regex: // assume names are regexes try { - pushPackages(params.setup.config.findPackages(std::regex(name.data(), name.size()))); + pushPackages(params.setup.config.findPackages(std::regex(name.data(), name.size()), limit)); } catch (const std::regex_error &e) { throw BadRequest(argsToString("regex is invalid: ", e.what())); } @@ -281,12 +292,12 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) case Mode::Provides: case Mode::Depends: // assume names are dependency notation - pushPackages(params.setup.config.findPackages(Dependency::fromString(name), mode == Mode::Depends)); + pushPackages(params.setup.config.findPackages(Dependency::fromString(name), mode == Mode::Depends, limit)); break; case Mode::LibProvides: case Mode::LibDepends: // assume names are "normalized" library names with platform prefix - pushPackages(params.setup.config.findPackagesProvidingLibrary(name, mode == Mode::LibDepends)); + pushPackages(params.setup.config.findPackagesProvidingLibrary(name, mode == Mode::LibDepends, limit)); break; default:; } diff --git a/librepomgr/webapi/routes_buildaction.cpp b/librepomgr/webapi/routes_buildaction.cpp index e544c22..5af8a52 100644 --- a/librepomgr/webapi/routes_buildaction.cpp +++ b/librepomgr/webapi/routes_buildaction.cpp @@ -23,13 +23,21 @@ void getBuildActions(const Params ¶ms, ResponseHandler &&handler) RAPIDJSON_NAMESPACE::Document jsonDoc(RAPIDJSON_NAMESPACE::kArrayType); RAPIDJSON_NAMESPACE::Value::Array array(jsonDoc.GetArray()); + const auto start = params.target.asNumber("start", std::numeric_limits::max()); + const auto serverLimit = params.setup.webServer.buildActionsResponseLimit.load(); + auto limit = params.target.asNumber("limit"); + if (!limit || limit > serverLimit) { + limit = serverLimit; + } + auto buildActionLock = params.setup.building.lockToRead(); params.setup.building.forEachBuildAction( [&array, &jsonDoc](std::size_t count) { array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(count), jsonDoc.GetAllocator()); }, - [&array, &jsonDoc](LibPkg::StorageID, BuildAction &&buildAction) { + [&array, &jsonDoc, limit](LibPkg::StorageID, BuildAction &&buildAction) { ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(buildAction), array, jsonDoc.GetAllocator()); - return false; - }); + return array.Size() >= limit; + }, + limit, start); buildActionLock.unlock(); handler(makeJson(params.request(), jsonDoc, params.target.hasPrettyFlag()));