Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Avoid loading databases when modification date is <= last update 

* Do HTTP head request first when loading database from mirror to avoid
  downloading the full database all the time
* Use the last modification date of the local database file because with
  the persistent storage even local database reloads became a bit expensive
This commit is contained in:
Martchus 2022-01-25 00:13:10 +01:00
parent fe09463b0a
commit 218dfecf56
19 changed files with 255 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libpkg/data/config.h
View File

@ -111,7 +111,7 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar
// load config and packages
void loadPacmanConfig(const char *pacmanConfigPath);
void loadAllPackages(bool withFiles);
void loadAllPackages(bool withFiles, bool force);
// storage and caching
void initStorage(const char *path = "libpkg.db", std::uint32_t maxDbs = 0);

2
libpkg/data/database.h
View File

@ -125,7 +125,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
void deducePathsFromLocalDirs();
void resetConfiguration();
void clearPackages();
void loadPackages(bool withFiles = false);
void loadPackagesFromConfiguredPaths(bool withFiles = false, bool force = false);
void loadPackages(const std::string &databaseData, CppUtilities::DateTime lastModified);
void loadPackages(FileMap &&databaseFiles, CppUtilities::DateTime lastModified);
static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);

4
libpkg/parser/config.cpp
View File

@ -143,11 +143,11 @@ void Config::loadPacmanConfig(const char *pacmanConfigPath)
}
}
void Config::loadAllPackages(bool withFiles)
void Config::loadAllPackages(bool withFiles, bool force)
{
for (auto &db : databases) {
try {
db.loadPackages(withFiles);
db.loadPackagesFromConfiguredPaths(withFiles, force);
} catch (const runtime_error &e) {
cerr << Phrases::ErrorMessage << "Unable to load database \"" << db.name << "\": " << e.what() << Phrases::EndFlush;
}

7
libpkg/parser/database.cpp
View File

@ -21,13 +21,16 @@ bool Database::isFileRelevant(const char *filePath, const char *fileName, mode_t
return !std::strcmp(fileName, "desc") || !std::strcmp(fileName, "depends") || !std::strcmp(fileName, "files");
}
void Database::loadPackages(bool withFiles)
void Database::loadPackagesFromConfiguredPaths(bool withFiles, bool force)
{
const auto &dbPath = withFiles && !filesPath.empty() ? filesPath : path;
if (dbPath.empty()) {
throw runtime_error("local path not configured");
}
loadPackages(extractFiles(dbPath, &isFileRelevant), lastModified(dbPath));
const auto lastFileUpdate = lastModified(dbPath);
if (force || lastFileUpdate > lastUpdate) {
loadPackages(extractFiles(dbPath, &isFileRelevant), lastFileUpdate);
}
}
void LibPkg::Database::loadPackages(const string &databaseData, DateTime lastModified)

2
libpkg/tests/parser.cpp
View File

@ -305,7 +305,7 @@ void ParserTests::testParsingDatabase()
db->filesPath = testFilePath("core.files");
// load packages
config.loadAllPackages(true);
config.loadAllPackages(true, false);
CPPUNIT_ASSERT_EQUAL_MESSAGE("all 215 packages present"s, 215_st, db->packageCount());
const auto autoreconf = db->findPackage("autoconf");
CPPUNIT_ASSERT_MESSAGE("autoreconf exists", autoreconf != nullptr);

9
librepomgr/buildactions/buildactionmeta.cpp
View File

@ -78,7 +78,14 @@ BuildActionMetaInfo::BuildActionMetaInfo()
.category = "Repo management",
.name = "Reload databases",
.type = "reload-database",
.flags = {},
.flags = {
BuildActionFlagMetaInfo{
.id = static_cast<BuildActionFlagType>(ReloadDatabaseFlags::ForceReload),
.name = "Force reload",
.desc = "Load the database even if its last modification data isn't newer than the last time the database was loaded",
.param = "force-reload",
},
},
.settings = {},
.directory = false,
.sourceDb = false,

5
librepomgr/buildactions/buildactionmeta.h
View File

@ -55,6 +55,10 @@ enum class CheckForUpdatesFlags : BuildActionFlagType {
None,
ConsiderRegularPackage = (1 << 0), // be consistent with LibPkg::UpdateCheckOptions here
};
enum class ReloadDatabaseFlags : BuildActionFlagType {
None,
ForceReload = (1 << 0),
};
enum class ReloadLibraryDependenciesFlags : BuildActionFlagType {
None,
ForceReload = (1 << 0),
@ -179,6 +183,7 @@ inline const BuildActionTypeMetaMapping &BuildActionMetaInfo::mappingForId(Build
} // namespace LibRepoMgr
CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ReloadDatabaseFlags)
CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ReloadLibraryDependenciesFlags)
CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::PrepareBuildFlags)
CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ConductBuildFlags)

21
librepomgr/buildactions/reloaddatabase.cpp
View File

@ -28,6 +28,8 @@ ReloadDatabase::ReloadDatabase(ServiceSetup &setup, const std::shared_ptr<BuildA
void ReloadDatabase::run()
{
const auto flags = static_cast<ReloadDatabaseFlags>(m_buildAction->flags);
const auto force = flags & ReloadDatabaseFlags::ForceReload;
const auto withFiles = m_setup.building.loadFilesDbs;
vector<LibPkg::Database *> dbsToLoadFromMirror;
@ -70,15 +72,26 @@ void ReloadDatabase::run()
continue;
}
boost::asio::post(
m_setup.building.ioContext.get_executor(), [this, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
m_buildAction->appendOutput(
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
m_setup.building.ioContext.get_executor(), [this, force, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
try {
auto dbFileLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(dbName, dbArch));
const auto lastModified = LibPkg::lastModified(dbPath);
if (!force) {
auto configReadLock2 = m_setup.config.lockToRead();
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) {
configReadLock2.unlock();
m_buildAction->appendOutput(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
"\" from local file \"", dbPath, "\"; last modification time <= last update (", lastModified.toString(), '<', '=',
lastUpdate.toString(), ')', '\n');
return;
}
}
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile));
dbFileLock.lock().unlock();
m_buildAction->appendOutput(
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
const auto configLock = m_setup.config.lockToWrite();
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
if (!destinationDb) {
@ -99,7 +112,7 @@ void ReloadDatabase::run()
// query databases
auto query = WebClient::prepareDatabaseQuery(m_buildAction->log(), dbsToLoadFromMirror, withFiles);
std::get<std::shared_lock<std::shared_mutex>>(configReadLock).unlock();
WebClient::queryDatabases(m_buildAction->log(), m_setup, std::move(query.queryParamsForDbs), session);
WebClient::queryDatabases(m_buildAction->log(), m_setup, std::move(query.queryParamsForDbs), session, force);
m_preparationFailures = std::move(query.failedDbs);
// clear AUR cache

2
librepomgr/buildactions/repomanagement.cpp
View File

@ -653,7 +653,7 @@ void CleanRepository::run()
std::make_unique<LibPkg::Database>(argsToString("clean-repository-", dbFilePaths.front().stem()), dbFilePaths.front()));
db->arch = dirInfo.canonicalPath.stem();
db->initStorage(*m_setup.config.storage());
db->loadPackages();
db->loadPackagesFromConfiguredPaths();
dirInfo.relevantDbs.emplace(db.get());
// acquire lock for db directory
dirInfo.lock.emplace<SharedLoggingLock>(m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(*db)));

2
librepomgr/serversetup.cpp
View File

@ -636,7 +636,7 @@ void ServiceSetup::run()
#endif
loadConfigFiles(true);
config.discardDatabases();
config.loadAllPackages(building.loadFilesDbs);
config.loadAllPackages(building.loadFilesDbs, building.forceLoadingDbs);
#ifndef CPP_UTILITIES_DEBUG_BUILD
} catch (const std::exception &e) {
cerr << Phrases::SubError << e.what() << endl;

3
librepomgr/serversetup.h
View File

@ -107,6 +107,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
std::string testFilesDir;
BuildPresets presets;
bool loadFilesDbs = false;
bool forceLoadingDbs = false;
// never changed after startup
unsigned short threadCount = 4;
@ -151,7 +152,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
void restoreLibraryDependenciesFromJson(const std::string &json, ReflectiveRapidJSON::JsonDeserializationErrors *errors);
std::size_t restoreState();
std::size_t saveState();
void initStorage();
void initStorage();
void run();
ServiceStatus computeStatus() const;
};

4
librepomgr/tests/buildactions.cpp
View File

@ -550,7 +550,7 @@ void BuildActionsTests::testConductingBuild()
readFile("repos/boost/os/x86_64/boost-libs-1.73.0-1-x86_64.pkg.tar.zst.sig"));
// add packages needing a rebuild to trigger auto-staging
m_setup.config.loadAllPackages(false);
m_setup.config.loadAllPackages(false, true);
auto *const boostDb = m_setup.config.findDatabase("boost"sv, "x86_64"sv);
auto *const miscDb = m_setup.config.findDatabase("misc"sv, "x86_64"sv);
CPPUNIT_ASSERT_MESSAGE("boost database present", boostDb);
@ -670,7 +670,7 @@ void BuildActionsTests::testCleanup()
auto *const miscDb = m_setup.config.findOrCreateDatabase("misc"sv, std::string_view());
miscDb->path = repoDir64 / "misc.db";
miscDb->localDbDir = miscDb->localPkgDir = repoDir64;
miscDb->loadPackages();
miscDb->loadPackagesFromConfiguredPaths();
// create and run build action
m_buildAction = std::make_shared<BuildAction>(0, &m_setup);

4
librepomgr/webapi/routes.cpp
View File

@ -345,8 +345,10 @@ void getPackages(const Params &params, ResponseHandler &&handler)
void postLoadPackages(const Params &params, ResponseHandler &&handler)
{
const auto withFiles = params.target.hasFlag("with-files");
const auto force = params.target.hasFlag("force");
auto lock = params.setup.config.lockToWrite();
params.setup.config.loadAllPackages(params.target.hasFlag("with-files"));
params.setup.config.loadAllPackages(withFiles, force);
lock.unlock();
handler(makeText(params.request(), "packages loaded"));
}

187
librepomgr/webclient/database.cpp
View File

@ -56,103 +56,138 @@ static int matchToInt(const std::sub_match<const char *> &match)
return stringToNumber<int>(sv);
}
void queryDatabases(
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&dbQueries, std::shared_ptr<DatabaseQuerySession> &dbQuerySession)
static CppUtilities::DateTime parseLastModified(LogContext &log, const auto &message, std::string_view dbName, std::string_view dbArch)
{
auto lastModified = DateTime();
const auto lastModifiedHeader = message.find(boost::beast::http::field::last_modified);
if (lastModifiedHeader != message.cend()) {
// parse "Last-Modified" header which should be something like "<day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT"
const auto lastModifiedStr = lastModifiedHeader->value();
static const auto lastModifiedPattern = std::regex("..., (\\d\\d) (...) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT");
static const auto months = unordered_map<string, int>{ { "Jan", 1 }, { "Feb", 2 }, { "Mar", 3 }, { "Apr", 4 }, { "May", 5 }, { "Jun", 6 },
{ "Jul", 7 }, { "Aug", 8 }, { "Sep", 9 }, { "Oct", 10 }, { "Nov", 11 }, { "Dec", 12 } };
try {
auto match = std::cmatch();
if (!std::regex_search(lastModifiedStr.cbegin(), lastModifiedStr.cend(), match, lastModifiedPattern)) {
throw ConversionException("date/time denotation not in expected format");
}
const auto month = months.find(match[2]);
if (month == months.cend()) {
throw ConversionException("month is invalid");
}
lastModified = DateTime::fromDateAndTime(
matchToInt(match[3]), month->second, matchToInt(match[1]), matchToInt(match[4]), matchToInt(match[5]), matchToInt(match[6]));
} catch (const ConversionException &e) {
log(Phrases::WarningMessage, "Unable to parse \"Last-Modified\" header for database ", dbName, '@', dbArch, ": ", e.what(),
" (last modification time was \"", string_view(lastModifiedStr.data(), lastModifiedStr.size()), "\")\n");
}
}
return lastModified;
}
void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&dbQueries,
std::shared_ptr<DatabaseQuerySession> &dbQuerySession, bool force)
{
for (auto &query : dbQueries) {
log(Phrases::InfoMessage, "Retrieving \"", query.databaseName, "\" from mirror: ", query.url, '\n');
auto session = runSessionFromUrl(
setup.building.ioContext, setup.webServer.sslContext, query.url,
[&log, &setup, dbName = std::move(query.databaseName), dbArch = std::move(query.databaseArch), dbQuerySession](
Session &session2, const WebClient::HttpClientError &error) mutable {
if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) {
log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": ",
error.what(), '\n');
dbQuerySession->addResponse(std::move(dbName));
return;
}
auto headHandler
= force ? Session::HeadHandler() : [&log, &setup, dbName = query.databaseName, dbArch = query.databaseArch](Session &session3) mutable {
auto lastModified = parseLastModified(log, session3.headResponse.get(), dbName, dbArch);
auto configReadLock = setup.config.lockToRead();
auto *const destinationDb = setup.config.findDatabase(dbName, dbArch);
if (!destinationDb) {
log(Phrases::InfoMessage, "Skip requesting database \"", dbName, '@', dbArch, "\" as it no longer exists\n");
session3.skip = true;
return;
}
const auto lastUpdate = destinationDb->lastUpdate;
configReadLock.unlock();
if (lastModified > lastUpdate) {
return;
}
log(Phrases::InfoMessage, "Skip requesting database \"", dbName, '@', dbArch,
"\" from mirror; last modification time <= last update (", lastModified.toString(), " <= ", lastUpdate.toString(), ')', '\n');
session3.skip = true;
};
auto handler = [&log, &setup, dbName = std::move(query.databaseName), dbArch = std::move(query.databaseArch), dbQuerySession, force](
Session &session2, const WebClient::HttpClientError &error) mutable {
if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) {
log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": ", error.what(),
'\n');
dbQuerySession->addResponse(std::move(dbName));
return;
}
if (session2.skip) {
return;
}
const auto &response = get<FileResponse>(session2.response);
const auto &message = response.get();
if (message.result() != boost::beast::http::status::ok) {
log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName,
": mirror returned ", message.result_int(), " response\n");
dbQuerySession->addResponse(std::move(dbName));
return;
}
const auto &response = std::get<FileResponse>(session2.response);
const auto &message = response.get();
if (message.result() != boost::beast::http::status::ok) {
log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": mirror returned ",
message.result_int(), " response\n");
dbQuerySession->addResponse(std::move(dbName));
return;
}
// find last modification time
auto lastModified = DateTime();
const auto lastModifiedHeader = message.find(boost::beast::http::field::last_modified);
if (lastModifiedHeader != message.cend()) {
// parse "Last-Modified" header which should be something like "<day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT"
const auto lastModifiedStr = lastModifiedHeader->value();
static const auto lastModifiedPattern = regex("..., (\\d\\d) (...) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT");
static const auto months = unordered_map<string, int>{ { "Jan", 1 }, { "Feb", 2 }, { "Mar", 3 }, { "Apr", 4 }, { "May", 5 },
{ "Jun", 6 }, { "Jul", 7 }, { "Aug", 8 }, { "Sep", 9 }, { "Oct", 10 }, { "Nov", 11 }, { "Dec", 12 } };
try {
cmatch match;
if (!regex_search(lastModifiedStr.cbegin(), lastModifiedStr.cend(), match, lastModifiedPattern)) {
throw ConversionException("date/time denotation not in expected format");
}
const auto month = months.find(match[2]);
if (month == months.cend()) {
throw ConversionException("month is invalid");
}
lastModified = DateTime::fromDateAndTime(matchToInt(match[3]), month->second, matchToInt(match[1]), matchToInt(match[4]),
matchToInt(match[5]), matchToInt(match[6]));
} catch (const ConversionException &e) {
log(Phrases::WarningMessage, "Unable to parse \"Last-Modified\" header for database ", dbName, ": ", e.what(),
" (last modification time was \"", string_view(lastModifiedStr.data(), lastModifiedStr.size()), "\")\n");
}
}
// log
if (lastModified.isNull()) {
log(Phrases::InfoMessage, "Loading database \"", dbName, "\" from mirror response\n");
lastModified = DateTime::gmtNow();
} else {
log(Phrases::InfoMessage, "Loading database \"", dbName,
"\" from mirror response; last modification time: ", lastModified.toString(), '\n');
}
try {
// load packages
auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant);
auto packages = Package::fromDatabaseFile(move(files));
// insert packages
auto lock = setup.config.lockToWrite();
auto db = setup.config.findDatabase(dbName, dbArch);
if (!db) {
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
// log/skip
auto lastModified = parseLastModified(log, message, dbName, dbArch);
if (lastModified.isNull()) {
log(Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from mirror response\n");
lastModified = DateTime::gmtNow();
} else if (!force) {
auto configReadLock = setup.config.lockToRead();
if (auto *const destinationDb = setup.config.findDatabase(dbName, dbArch)) {
if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) {
configReadLock.unlock();
log(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
"\" from mirror response; last modification time <= last update (", lastModified.toString(),
" <= ", lastUpdate.toString(), ')', '\n');
return;
}
db->replacePackages(packages, lastModified);
} catch (const std::runtime_error &e) {
log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
dbQuerySession->addResponse(std::move(dbName));
}
},
move(query.destinationFilePath));
} else {
log(Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch,
"\" from mirror response; last modification time: ", lastModified.toString(), '\n');
}
try {
// load packages
auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant);
auto packages = Package::fromDatabaseFile(std::move(files));
// insert packages
auto lock = setup.config.lockToWrite();
auto db = setup.config.findDatabase(dbName, dbArch);
if (!db) {
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
return;
}
db->replacePackages(packages, lastModified);
} catch (const std::runtime_error &e) {
log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
dbQuerySession->addResponse(std::move(dbName));
}
};
auto session = runSessionFromUrl(setup.building.ioContext, setup.webServer.sslContext, query.url, std::move(handler), std::move(headHandler),
std::move(query.destinationFilePath));
}
}
std::shared_ptr<DatabaseQuerySession> queryDatabases(
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, DatabaseQuerySession::HandlerType &&handler)
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, bool force, DatabaseQuerySession::HandlerType &&handler)
{
auto dbQuerySession = DatabaseQuerySession::create(setup.building.ioContext, move(handler));
queryDatabases(log, setup, move(urls), dbQuerySession);
queryDatabases(log, setup, move(urls), dbQuerySession, force);
return dbQuerySession;
}
std::shared_ptr<DatabaseQuerySession> queryDatabases(LogContext &log, ServiceSetup &setup, std::shared_lock<shared_mutex> *configReadLock,
const std::vector<Database *> &dbs, DatabaseQuerySession::HandlerType &&handler)
const std::vector<Database *> &dbs, bool force, DatabaseQuerySession::HandlerType &&handler)
{
shared_lock<shared_mutex> ownConfigReadLock;
auto query = prepareDatabaseQuery(log, dbs, setup.building.loadFilesDbs);
configReadLock->unlock();
return queryDatabases(log, setup, std::move(query.queryParamsForDbs), move(handler));
return queryDatabases(log, setup, std::move(query.queryParamsForDbs), force, move(handler));
}
PackageCachingSession::PackageCachingSession(PackageCachingDataForSession &data, boost::asio::io_context &ioContext,

8
librepomgr/webclient/database.h
View File

@ -30,11 +30,11 @@ struct DatabaseQuery {
[[nodiscard]] DatabaseQuery prepareDatabaseQuery(LogContext &log, const std::vector<LibPkg::Database *> &dbs, bool withFiles);
std::shared_ptr<DatabaseQuerySession> queryDatabases(
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, DatabaseQuerySession::HandlerType &&handler);
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, bool force, DatabaseQuerySession::HandlerType &&handler);
std::shared_ptr<DatabaseQuerySession> queryDatabases(LogContext &log, ServiceSetup &setup, std::shared_lock<std::shared_mutex> *configReadLock,
const std::vector<LibPkg::Database *> &dbs, DatabaseQuerySession::HandlerType &&handler);
void queryDatabases(
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, std::shared_ptr<DatabaseQuerySession> &dbQuerySession);
const std::vector<LibPkg::Database *> &dbs, bool force, DatabaseQuerySession::HandlerType &&handler);
void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls,
std::shared_ptr<DatabaseQuerySession> &dbQuerySession, bool force = false);
struct PackageCachingDataForPackage {
std::string_view url;

65
librepomgr/webclient/session.cpp
View File

@ -57,10 +57,10 @@ void Session::run(
// set up an HTTP request message
request.version(version);
request.method(verb);
request.target(target);
request.set(http::field::host, host);
request.set(http::field::user_agent, APP_NAME " " APP_VERSION);
method = verb;
// setup a file response
if (!destinationFilePath.empty()) {
@ -131,6 +131,17 @@ void Session::handshakeDone(boost::beast::error_code ec)
void Session::sendRequest()
{
// send the HTTP request to the remote host
if (m_headHandler) {
request.method(boost::beast::http::verb::head);
std::visit(
[this](auto &&stream) {
boost::beast::http::async_write(
stream, request, std::bind(&Session::headRequested, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
},
m_stream);
return;
}
request.method(method);
std::visit(
[this](auto &&stream) {
boost::beast::http::async_write(
@ -139,6 +150,24 @@ void Session::sendRequest()
m_stream);
}
void Session::headRequested(boost::beast::error_code ec, std::size_t bytesTransferred)
{
boost::ignore_unused(bytesTransferred);
if (ec) {
m_handler(*this, HttpClientError("sending HEAD request", ec));
return;
}
// receive the HTTP response
headResponse.skip(true);
std::visit(
[this](auto &stream) {
http::async_read(
stream, m_buffer, headResponse, std::bind(&Session::headReceived, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
},
m_stream);
}
void Session::requested(boost::beast::error_code ec, std::size_t bytesTransferred)
{
boost::ignore_unused(bytesTransferred);
@ -222,6 +251,22 @@ bool Session::continueReadingChunks()
return true;
}
void Session::headReceived(boost::beast::error_code ec, std::size_t bytesTransferred)
{
boost::ignore_unused(bytesTransferred);
if (ec) {
m_handler(*this, HttpClientError("receiving HEAD response", ec));
return;
}
m_headHandler(*this);
m_headHandler = HeadHandler();
if (skip) {
sendRequest();
} else {
closeGracefully();
}
}
void Session::received(boost::beast::error_code ec, std::size_t bytesTransferred)
{
boost::ignore_unused(bytesTransferred);
@ -229,8 +274,12 @@ void Session::received(boost::beast::error_code ec, std::size_t bytesTransferred
m_handler(*this, HttpClientError("receiving response", ec));
return;
}
closeGracefully();
}
// close the stream gracefully
void Session::closeGracefully()
{
auto ec = boost::beast::error_code();
if (auto *const sslStream = std::get_if<SslStream>(&m_stream)) {
// perform the SSL handshake
sslStream->async_shutdown(std::bind(&Session::closed, shared_from_this(), std::placeholders::_1));
@ -249,6 +298,14 @@ void Session::closed(boost::beast::error_code ec)
std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
std::string_view url, Session::Handler &&handler, std::string &&destinationPath, std::string_view userName, std::string_view password,
boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
{
return runSessionFromUrl(ioContext, sslContext, url, std::move(handler), Session::HeadHandler(), std::move(destinationPath), userName, password,
verb, bodyLimit, std::move(chunkHandler));
}
std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
std::string_view url, Session::Handler &&handler, Session::HeadHandler &&headHandler, std::string &&destinationPath, std::string_view userName,
std::string_view password, boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
{
std::string host, port, target;
auto ssl = false;
@ -284,8 +341,8 @@ std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asi
port = ssl ? "443" : "80";
}
auto session
= ssl ? std::make_shared<Session>(ioContext, sslContext, std::move(handler)) : std::make_shared<Session>(ioContext, std::move(handler));
auto session = ssl ? std::make_shared<Session>(ioContext, sslContext, std::move(handler), std::move(headHandler))
: std::make_shared<Session>(ioContext, std::move(handler), std::move(headHandler));
if (!userName.empty()) {
const auto authInfo = userName % ":" + password;
session->request.set(boost::beast::http::field::authorization,

27
librepomgr/webclient/session.h
View File

@ -45,6 +45,7 @@ inline LibRepoMgr::WebClient::HttpClientError::operator bool() const
using Response = WebAPI::Response;
using FileResponse = boost::beast::http::response_parser<boost::beast::http::file_body>;
using StringResponse = boost::beast::http::response_parser<boost::beast::http::string_body>;
using EmptyResponse = boost::beast::http::response_parser<boost::beast::http::empty_body>;
using MultiResponse = std::variant<Response, FileResponse, StringResponse>;
using Request = boost::beast::http::request<boost::beast::http::empty_body>;
struct ChunkProcessing;
@ -52,14 +53,17 @@ struct ChunkProcessing;
class LIBREPOMGR_EXPORT Session : public std::enable_shared_from_this<Session> {
public:
using Handler = std::function<void(Session &, const HttpClientError &error)>;
using HeadHandler = std::function<void(Session &)>;
using ChunkHandler = std::function<void(const boost::beast::http::chunk_extensions &chunkExtensions, std::string_view chunkData)>;
template <typename ResponseType = Response> explicit Session(boost::asio::io_context &ioContext, const Handler &handler = Handler());
template <typename ResponseType = Response> explicit Session(boost::asio::io_context &ioContext, Handler &&handler = Handler());
template <typename ResponseType = Response>
explicit Session(boost::asio::io_context &ioContext, Handler &&handler = Handler(), HeadHandler &&headHandler = HeadHandler());
template <typename ResponseType = Response>
explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, const Handler &handler = Handler());
template <typename ResponseType = Response>
explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler);
explicit Session(
boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler, HeadHandler &&headHandler = HeadHandler());
void setChunkHandler(ChunkHandler &&handler);
void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target,
@ -83,25 +87,33 @@ private:
void connected(boost::beast::error_code ec);
void handshakeDone(boost::beast::error_code ec);
void sendRequest();
void headRequested(boost::beast::error_code ec, std::size_t bytesTransferred);
void requested(boost::beast::error_code ec, std::size_t bytesTransferred);
void onChunkHeader(std::uint64_t chunkSize, boost::beast::string_view extensions, boost::beast::error_code &ec);
std::size_t onChunkBody(std::uint64_t bytesLeftInThisChunk, boost::beast::string_view chunkBodyData, boost::beast::error_code &ec);
void chunkReceived(boost::beast::error_code ec, std::size_t bytesTransferred);
bool continueReadingChunks();
void headReceived(boost::beast::error_code ec, std::size_t bytesTransferred);
void received(boost::beast::error_code ec, std::size_t bytesTransferred);
void closeGracefully();
void closed(boost::beast::error_code ec);
public:
Request request;
MultiResponse response;
EmptyResponse headResponse;
std::string destinationFilePath;
boost::beast::http::verb method = boost::beast::http::verb::get;
bool skip = false;
private:
boost::asio::ip::tcp::resolver m_resolver;
std::variant<RawSocket, SslStream> m_stream;
boost::beast::flat_buffer m_buffer;
std::unique_ptr<ChunkProcessing> m_chunkProcessing;
Request m_headRequest;
Handler m_handler;
HeadHandler m_headHandler;
};
template <typename ResponseType>
@ -114,11 +126,12 @@ inline Session::Session(boost::asio::io_context &ioContext, const Handler &handl
}
template <typename ResponseType>
inline Session::Session(boost::asio::io_context &ioContext, Handler &&handler)
inline Session::Session(boost::asio::io_context &ioContext, Handler &&handler, HeadHandler &&headHandler)
: response(ResponseType{})
, m_resolver(ioContext)
, m_stream(RawSocket{ ioContext })
, m_handler(std::move(handler))
, m_headHandler(std::move(headHandler))
{
}
@ -132,11 +145,12 @@ inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::co
}
template <typename ResponseType>
inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler)
inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler, HeadHandler &&headHandler)
: response(ResponseType{})
, m_resolver(ioContext)
, m_stream(SslStream{ ioContext, sslContext })
, m_handler(std::move(handler))
, m_headHandler(std::move(headHandler))
{
}
@ -145,6 +159,11 @@ LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSession
std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext,
boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, Session::HeadHandler &&headHandler,
std::string &&destinationPath = std::string(), std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
} // namespace WebClient
} // namespace LibRepoMgr

2
pacfind/main.cpp
View File

@ -92,7 +92,7 @@ int main(int argc, const char *argv[])
} else {
db.filesPath = db.filesPathFromRegularPath();
}
db.loadPackages(true);
db.loadPackagesFromConfiguredPaths(true, true);
} catch (const std::runtime_error &e) {
std::cerr << "Unable to load database \"" << db.name << "\": " << e.what() << '\n';
}

10
srv/main.cpp
View File

@ -30,12 +30,14 @@ int main(int argc, const char *argv[])
OperationArgument runArg("run", 'r', "runs the server");
ConfigValueArgument configFileArg("config-file", 'c', "specifies the path of the config file", { "path" });
configFileArg.setEnvironmentVariable(PROJECT_VARNAME_UPPER "_CONFIG_FILE");
runArg.setSubArguments({ &configFileArg });
ConfigValueArgument forceLoadingDBsArg("force-loading-dbs", 'f', "forces loading DBs, even if DB files have not been modified since last parse");
runArg.setSubArguments({ &configFileArg, &forceLoadingDBsArg });
runArg.setImplicit(true);
runArg.setCallback([&setup, &configFileArg](const ArgumentOccurrence &) {
if (configFileArg.firstValue()) {
setup.configFilePath = configFileArg.firstValue();
runArg.setCallback([&setup, &configFileArg, &forceLoadingDBsArg](const ArgumentOccurrence &) {
if (const auto configFilePath = configFileArg.firstValue()) {
setup.configFilePath = configFilePath;
}
setup.building.forceLoadingDbs = forceLoadingDBsArg.isPresent();
setup.run();
});
HelpArgument helpArg(parser);