Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Limit results returned by the API for better scalability

This commit is contained in:
Martchus 2022-02-25 00:29:43 +01:00
parent ff7f039519
commit 90ff9678fe
9 changed files with 117 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
3rdparty/lmdb-safe vendored

@ -1 +1 @@
Subproject commit bf6dbf0e42c081c7b75bea9bb59ffbd0c0d6b529
Subproject commit f6fd78155b0a4fd229b1b352fe87efed2460d982

38
libpkg/algo/search.cpp
View File

@ -97,7 +97,8 @@ Database *Config::findOrCreateDatabaseFromDenotation(std::string_view databaseDe
/*!
* \brief Returns all packages with the specified database name, database architecture and package name.
*/
std::vector<PackageSearchResult> Config::findPackages(std::tuple<std::string_view, std::string_view, std::string_view> dbAndPackageName)
std::vector<PackageSearchResult> Config::findPackages(
std::tuple<std::string_view, std::string_view, std::string_view> dbAndPackageName, std::size_t limit)
{
auto pkgs = std::vector<PackageSearchResult>();
const auto &[dbName, dbArch, packageName] = dbAndPackageName;
@ -115,6 +116,9 @@ std::vector<PackageSearchResult> Config::findPackages(std::tuple<std::string_vie
if (const auto [id, package] = db.findPackageWithID(name); package) {
pkgs.emplace_back(db, package, id);
}
if (pkgs.size() >= limit) {
return pkgs;
}
}
return pkgs;
}
@ -146,7 +150,7 @@ PackageSearchResult Config::findPackage(const Dependency &dependency)
/*!
* \brief Returns all packages satisfying \a dependency or - if \a reverse is true - all packages requiring \a dependency.
*/
std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependency, bool reverse)
std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependency, bool reverse, std::size_t limit)
{
auto results = std::vector<PackageSearchResult>();
for (auto &db : databases) {
@ -155,7 +159,7 @@ std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependen
if (visited.emplace(packageID).second) {
results.emplace_back(db, package, packageID);
}
return false;
return results.size() >= limit;
});
}
return results;
@ -164,7 +168,7 @@ std::vector<PackageSearchResult> Config::findPackages(const Dependency &dependen
/*!
* \brief Returns all packages providing \a library or - if \a reverse is true - all packages requiring \a library.
*/
std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std::string &library, bool reverse)
std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std::string &library, bool reverse, std::size_t limit)
{
auto results = std::vector<PackageSearchResult>();
auto visited = std::unordered_set<StorageID>();
@ -173,7 +177,7 @@ std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std:
if (visited.emplace(packageID).second) {
results.emplace_back(db, package, packageID);
}
return false;
return results.size() >= limit;
});
}
return results;
@ -182,7 +186,7 @@ std::vector<PackageSearchResult> Config::findPackagesProvidingLibrary(const std:
/*!
* \brief Returns all packages which names matches \a regex.
*/
std::vector<PackageSearchResult> Config::findPackages(const std::regex &regex)
std::vector<PackageSearchResult> Config::findPackages(const std::regex &regex, std::size_t limit)
{
auto pkgs = std::vector<PackageSearchResult>();
for (auto &db : databases) {
@ -191,13 +195,14 @@ std::vector<PackageSearchResult> Config::findPackages(const std::regex &regex)
auto [packageID, package] = getPackage();
pkgs.emplace_back(db, package, packageID);
}
return false;
return pkgs.size() >= limit;
});
}
return pkgs;
}
std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(const Database &)> &databasePred, std::string_view term)
std::vector<PackageSearchResult> Config::findPackages(
const std::function<bool(const Database &)> &databasePred, std::string_view term, std::size_t limit)
{
auto pkgs = std::vector<PackageSearchResult>();
for (auto &db : databases) {
@ -209,7 +214,7 @@ std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(c
const auto [packageID, package] = getPackage();
pkgs.emplace_back(db, package, packageID);
}
return false;
return pkgs.size() >= limit;
});
}
return pkgs;
@ -219,13 +224,16 @@ std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(c
* \brief Returns all packages considered "the same" as \a package.
* \remarks See Package::isSame().
*/
std::vector<PackageSearchResult> Config::findPackages(const Package &package)
std::vector<PackageSearchResult> Config::findPackages(const Package &package, std::size_t limit)
{
auto pkgs = std::vector<PackageSearchResult>();
for (auto &db : databases) {
if (const auto [id, pkg] = db.findPackageWithID(package.name); pkg && pkg->isSame(package)) {
pkgs.emplace_back(db, pkg, id);
}
if (pkgs.size() >= limit) {
return pkgs;
}
}
return pkgs;
}
@ -233,8 +241,8 @@ std::vector<PackageSearchResult> Config::findPackages(const Package &package)
/*!
* \brief Returns all packages \a packagePred returns true for from all databases \a databasePred returns true for.
*/
std::vector<PackageSearchResult> Config::findPackages(
const std::function<bool(const Database &)> &databasePred, const std::function<bool(const Database &, const Package &)> &packagePred)
std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(const Database &)> &databasePred,
const std::function<bool(const Database &, const Package &)> &packagePred, std::size_t limit)
{
auto pkgs = std::vector<PackageSearchResult>();
for (auto &db : databases) {
@ -245,7 +253,7 @@ std::vector<PackageSearchResult> Config::findPackages(
if (packagePred(db, *package)) {
pkgs.emplace_back(db, std::move(package), packageID);
}
return false;
return pkgs.size() >= limit;
});
}
return pkgs;
@ -254,7 +262,7 @@ std::vector<PackageSearchResult> Config::findPackages(
/*!
* \brief Returns all packages \a pred returns true for.
*/
std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(const Database &, const Package &)> &pred)
std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(const Database &, const Package &)> &pred, std::size_t limit)
{
auto pkgs = std::vector<PackageSearchResult>();
for (auto &db : databases) {
@ -262,7 +270,7 @@ std::vector<PackageSearchResult> Config::findPackages(const std::function<bool(c
if (pred(db, *package)) {
pkgs.emplace_back(db, std::move(package), packageID);
}
return false;
return pkgs.size() >= limit;
});
}
return pkgs;

37
libpkg/data/config.h
View File

@ -140,18 +140,24 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar
Database *findOrCreateDatabase(std::string_view name, std::string_view architecture);
Database *findOrCreateDatabaseFromDenotation(std::string_view databaseDenotation);
static std::tuple<std::string_view, std::string_view, std::string_view> parsePackageDenotation(std::string_view packageDenotation);
std::vector<PackageSearchResult> findPackages(std::string_view packageDenotation);
std::vector<PackageSearchResult> findPackages(std::string_view dbName, std::string_view dbArch, std::string_view packageName);
std::vector<PackageSearchResult> findPackages(std::tuple<std::string_view, std::string_view, std::string_view> dbAndPackageName);
PackageSearchResult findPackage(const Dependency &dependency);
std::vector<PackageSearchResult> findPackages(const Dependency &dependency, bool reverse = false);
std::vector<PackageSearchResult> findPackagesProvidingLibrary(const std::string &library, bool reverse = false);
std::vector<PackageSearchResult> findPackages(const std::regex &regex);
std::vector<PackageSearchResult> findPackages(const std::function<bool(const Database &)> &databasePred, std::string_view term);
std::vector<PackageSearchResult> findPackages(const Package &package);
std::vector<PackageSearchResult> findPackages(std::string_view packageDenotation, std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<PackageSearchResult> findPackages(
const std::function<bool(const Database &)> &databasePred, const std::function<bool(const Database &, const Package &)> &packagePred);
std::vector<PackageSearchResult> findPackages(const std::function<bool(const Database &, const Package &)> &pred);
std::string_view dbName, std::string_view dbArch, std::string_view packageName, std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<PackageSearchResult> findPackages(std::tuple<std::string_view, std::string_view, std::string_view> dbAndPackageName,
std::size_t limit = std::numeric_limits<std::size_t>::max());
PackageSearchResult findPackage(const Dependency &dependency);
std::vector<PackageSearchResult> findPackages(
const Dependency &dependency, bool reverse = false, std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<PackageSearchResult> findPackagesProvidingLibrary(
const std::string &library, bool reverse = false, std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<PackageSearchResult> findPackages(const std::regex &regex, std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<PackageSearchResult> findPackages(const std::function<bool(const Database &)> &databasePred, std::string_view term,
std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<PackageSearchResult> findPackages(const Package &package, std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<PackageSearchResult> findPackages(const std::function<bool(const Database &)> &databasePred,
const std::function<bool(const Database &, const Package &)> &packagePred, std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<PackageSearchResult> findPackages(
const std::function<bool(const Database &, const Package &)> &pred, std::size_t limit = std::numeric_limits<std::size_t>::max());
std::vector<Database> databases;
Database aur = Database("aur");
@ -181,14 +187,15 @@ inline Status Config::computeStatus() const
return Status(*this);
}
inline std::vector<PackageSearchResult> Config::findPackages(std::string_view dbName, std::string_view dbArch, std::string_view packageName)
inline std::vector<PackageSearchResult> Config::findPackages(
std::string_view dbName, std::string_view dbArch, std::string_view packageName, std::size_t limit)
{
return findPackages(std::make_tuple(dbName, dbArch, packageName));
return findPackages(std::make_tuple(dbName, dbArch, packageName), limit);
}
inline std::vector<PackageSearchResult> Config::findPackages(std::string_view packageDenotation)
inline std::vector<PackageSearchResult> Config::findPackages(std::string_view packageDenotation, std::size_t limit)
{
return findPackages(parsePackageDenotation(packageDenotation));
return findPackages(parsePackageDenotation(packageDenotation), limit);
}
} // namespace LibPkg

10
librepomgr/helper.h
View File

@ -39,7 +39,7 @@ inline std::optional<std::string_view> getLastValueSv(const std::multimap<std::s
return std::nullopt;
}
template <typename TargetType, Traits::DisableIf<std::is_integral<TargetType>> * = nullptr>
template <typename TargetType, Traits::DisableIfAny<std::is_integral<TargetType>, Traits::IsSpecializationOf<TargetType, std::atomic>> * = nullptr>
void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result);
template <>
@ -60,7 +60,7 @@ inline void convertValue(const std::multimap<std::string, std::string> &multimap
}
}
template <typename TargetType, Traits::EnableIf<std::is_integral<TargetType>> * = nullptr>
template <typename TargetType, Traits::EnableIfAny<std::is_integral<TargetType>, Traits::IsSpecializationOf<TargetType, std::atomic>> * = nullptr>
inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result)
{
using namespace std;
@ -69,7 +69,11 @@ inline void convertValue(const std::multimap<std::string, std::string> &multimap
if (const char *const value = getLastValue(multimap, key)) {
try {
result = stringToNumber<TargetType>(value);
if constexpr (Traits::IsSpecializationOf<TargetType, std::atomic>::value) {
result = stringToNumber<typename TargetType::value_type>(value);
} else {
result = stringToNumber<TargetType>(value);
}
} catch (const ConversionException &) {
cerr << Phrases::ErrorMessage << "Specified number \"" << value << "\" for key \"" << key << "\" is invalid." << Phrases::End;
return;

13
librepomgr/serversetup.cpp
View File

@ -90,6 +90,8 @@ void ServiceSetup::WebServerSetup::applyConfig(const std::multimap<std::string,
convertValue(multimap, "port", port);
convertValue(multimap, "threads", threadCount);
convertValue(multimap, "static_files", staticFilesPath);
convertValue(multimap, "package_search_response_limit", packageSearchResponseLimit);
convertValue(multimap, "build_actions_response_limit", buildActionsResponseLimit);
convertValue(multimap, "verify_ssl_certificates", verifySslCertificates);
convertValue(multimap, "log_ssl_certificate_validation", logSslCertificateValidation);
@ -338,11 +340,16 @@ std::size_t ServiceSetup::BuildSetup::buildActionCount()
}
void ServiceSetup::BuildSetup::forEachBuildAction(
std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func)
std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func, std::size_t limit, std::size_t start)
{
auto txn = m_storage->buildActions.getROTransaction();
count(txn.size());
for (auto i = txn.begin(); i != txn.end(); ++i) {
const auto total = txn.size();
count(std::min(limit, total));
const auto reverse = start == std::numeric_limits<std::size_t>::max();
for (auto i = reverse ? txn.rbegin()
: txn.lower_bound(static_cast<LibPkg::StorageID>(
start > std::numeric_limits<LibPkg::StorageID>::max() ? std::numeric_limits<LibPkg::StorageID>::max() : start));
i != txn.end() && limit; reverse ? --i : ++i, --limit) {
if (func(i.getID(), std::move(i.value()))) {
return;
}

5
librepomgr/serversetup.h
View File

@ -68,6 +68,8 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
unsigned short threadCount = 1;
boost::asio::io_context ioContext;
boost::asio::ssl::context sslContext{ boost::asio::ssl::context::sslv23_client };
std::atomic_size_t packageSearchResponseLimit = 20000; // sufficient to return a "full architecture"
std::atomic_size_t buildActionsResponseLimit = 200;
bool verifySslCertificates = true;
bool logSslCertificateValidation = false;
@ -138,7 +140,8 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
void deleteBuildAction(const std::vector<std::shared_ptr<BuildAction>> &actions);
std::size_t buildActionCount();
std::size_t runningBuildActionCount();
void forEachBuildAction(std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func);
void forEachBuildAction(std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func,
std::size_t limit, std::size_t start);
void forEachBuildAction(std::function<bool(LibPkg::StorageID, BuildAction &, bool &)> &&func);
std::vector<std::shared_ptr<BuildAction>> followUpBuildActions(BuildActionIdType forId);

21
librepomgr/webapi/params.h
View File

@ -6,6 +6,9 @@
#include "./session.h"
#include "./typedefs.h"
#include <c++utilities/conversion/stringbuilder.h>
#include <c++utilities/conversion/stringconversion.h>
#include <unordered_map>
namespace std {
@ -51,10 +54,28 @@ struct LIBREPOMGR_EXPORT Url {
bool hasPrettyFlag() const;
std::string_view value(std::string_view paramName) const;
std::vector<std::string> decodeValues(std::string_view paramName) const;
template <typename Number> Number asNumber(std::string_view paramName, Number def = Number()) const;
static std::string decodeValue(std::string_view value);
static std::string encodeValue(std::string_view value);
};
template <typename Number> Number Url::asNumber(std::string_view paramName, Number def) const
{
using namespace CppUtilities;
const auto values = decodeValues(paramName);
if (values.size() > 1) {
throw BadRequest("more than one " % paramName + " specified");
}
if (!values.empty()) {
try {
return stringToNumber<Number>(values.front());
} catch (const ConversionException &) {
throw BadRequest(argsToString(paramName, " must be an integer"));
}
}
return def;
}
inline bool Url::hasPrettyFlag() const
{
return hasFlag("pretty");

25
librepomgr/webapi/routes.cpp
View File

@ -150,7 +150,7 @@ void getUnresolved(const Params &params, ResponseHandler &&handler)
void getPackages(const Params &params, ResponseHandler &&handler)
{
// read mode
const auto modes(params.target.decodeValues("mode"));
const auto modes = params.target.decodeValues("mode");
if (modes.size() > 1) {
throw BadRequest("more than one mode specified");
}
@ -181,6 +181,13 @@ void getPackages(const Params &params, ResponseHandler &&handler)
mode = modeIterator->second;
}
// limit
auto limit = params.target.asNumber<std::size_t>("limit");
const auto serverLimit = params.setup.webServer.packageSearchResponseLimit.load();
if (!limit || limit > serverLimit) {
limit = serverLimit;
}
// check for details flag
const auto details = params.target.hasFlag("details");
if (details && mode != Mode::Name) {
@ -214,7 +221,8 @@ void getPackages(const Params &params, ResponseHandler &&handler)
RAPIDJSON_NAMESPACE::Document document(RAPIDJSON_NAMESPACE::kArrayType);
RAPIDJSON_NAMESPACE::Document::Array array(document.GetArray());
const auto pushPackages = [&dbs, &document, &array](auto &&packages) {
const auto pushPackages = [&dbs, &document, &array, &limit](auto &&packages) {
limit -= packages.size();
for (const auto &package : packages) {
if (!dbs.empty()) {
const auto *const db = std::get<Database *>(package.db);
@ -248,15 +256,17 @@ void getPackages(const Params &params, ResponseHandler &&handler)
} else {
neededAurPackages.emplace_back(std::move(packageNameStr));
}
--limit;
}
if (!isDbAur && (!dbs.empty() || !onlyFromAur)) {
auto packages = params.setup.config.findPackages(packageDenotation);
auto packages = params.setup.config.findPackages(packageDenotation, limit);
if (details) {
for (const auto &package : packages) {
if (dbs.empty() || dbs.find(std::get<LibPkg::Database *>(package.db)->name) != dbs.end()) {
ReflectiveRapidJSON::JsonReflector::push(package.pkg, array, document.GetAllocator());
}
}
limit -= packages.size();
} else {
pushPackages(std::move(packages));
}
@ -265,7 +275,8 @@ void getPackages(const Params &params, ResponseHandler &&handler)
}
case Mode::NameContains:
pushPackages(params.setup.config.findPackages(
[&dbs, onlyFromAur](const LibPkg::Database &db) { return (dbs.empty() && !onlyFromAur) || dbs.find(db.name) != dbs.end(); }, name));
[&dbs, onlyFromAur](const LibPkg::Database &db) { return (dbs.empty() && !onlyFromAur) || dbs.find(db.name) != dbs.end(); }, name,
limit));
if (fromAur && !name.empty()) {
neededAurPackages.emplace_back(std::move(name));
}
@ -273,7 +284,7 @@ void getPackages(const Params &params, ResponseHandler &&handler)
case Mode::Regex:
// assume names are regexes
try {
pushPackages(params.setup.config.findPackages(std::regex(name.data(), name.size())));
pushPackages(params.setup.config.findPackages(std::regex(name.data(), name.size()), limit));
} catch (const std::regex_error &e) {
throw BadRequest(argsToString("regex is invalid: ", e.what()));
}
@ -281,12 +292,12 @@ void getPackages(const Params &params, ResponseHandler &&handler)
case Mode::Provides:
case Mode::Depends:
// assume names are dependency notation
pushPackages(params.setup.config.findPackages(Dependency::fromString(name), mode == Mode::Depends));
pushPackages(params.setup.config.findPackages(Dependency::fromString(name), mode == Mode::Depends, limit));
break;
case Mode::LibProvides:
case Mode::LibDepends:
// assume names are "normalized" library names with platform prefix
pushPackages(params.setup.config.findPackagesProvidingLibrary(name, mode == Mode::LibDepends));
pushPackages(params.setup.config.findPackagesProvidingLibrary(name, mode == Mode::LibDepends, limit));
break;
default:;
}

14
librepomgr/webapi/routes_buildaction.cpp
View File

@ -23,13 +23,21 @@ void getBuildActions(const Params &params, ResponseHandler &&handler)
RAPIDJSON_NAMESPACE::Document jsonDoc(RAPIDJSON_NAMESPACE::kArrayType);
RAPIDJSON_NAMESPACE::Value::Array array(jsonDoc.GetArray());
const auto start = params.target.asNumber<std::size_t>("start", std::numeric_limits<std::size_t>::max());
const auto serverLimit = params.setup.webServer.buildActionsResponseLimit.load();
auto limit = params.target.asNumber<std::size_t>("limit");
if (!limit || limit > serverLimit) {
limit = serverLimit;
}
auto buildActionLock = params.setup.building.lockToRead();
params.setup.building.forEachBuildAction(
[&array, &jsonDoc](std::size_t count) { array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(count), jsonDoc.GetAllocator()); },
[&array, &jsonDoc](LibPkg::StorageID, BuildAction &&buildAction) {
[&array, &jsonDoc, limit](LibPkg::StorageID, BuildAction &&buildAction) {
ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(buildAction), array, jsonDoc.GetAllocator());
return false;
});
return array.Size() >= limit;
},
limit, start);
buildActionLock.unlock();
handler(makeJson(params.request(), jsonDoc, params.target.hasPrettyFlag()));