Avoid loading databases when modification date is <= last update
* Do HTTP head request first when loading database from mirror to avoid downloading the full database all the time * Use the last modification date of the local database file because with the persistent storage even local database reloads became a bit expensive
This commit is contained in:
parent
fe09463b0a
commit
218dfecf56
|
@ -111,7 +111,7 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar
|
|||
|
||||
// load config and packages
|
||||
void loadPacmanConfig(const char *pacmanConfigPath);
|
||||
void loadAllPackages(bool withFiles);
|
||||
void loadAllPackages(bool withFiles, bool force);
|
||||
|
||||
// storage and caching
|
||||
void initStorage(const char *path = "libpkg.db", std::uint32_t maxDbs = 0);
|
||||
|
|
|
@ -125,7 +125,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
|
|||
void deducePathsFromLocalDirs();
|
||||
void resetConfiguration();
|
||||
void clearPackages();
|
||||
void loadPackages(bool withFiles = false);
|
||||
void loadPackagesFromConfiguredPaths(bool withFiles = false, bool force = false);
|
||||
void loadPackages(const std::string &databaseData, CppUtilities::DateTime lastModified);
|
||||
void loadPackages(FileMap &&databaseFiles, CppUtilities::DateTime lastModified);
|
||||
static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);
|
||||
|
|
|
@ -143,11 +143,11 @@ void Config::loadPacmanConfig(const char *pacmanConfigPath)
|
|||
}
|
||||
}
|
||||
|
||||
void Config::loadAllPackages(bool withFiles)
|
||||
void Config::loadAllPackages(bool withFiles, bool force)
|
||||
{
|
||||
for (auto &db : databases) {
|
||||
try {
|
||||
db.loadPackages(withFiles);
|
||||
db.loadPackagesFromConfiguredPaths(withFiles, force);
|
||||
} catch (const runtime_error &e) {
|
||||
cerr << Phrases::ErrorMessage << "Unable to load database \"" << db.name << "\": " << e.what() << Phrases::EndFlush;
|
||||
}
|
||||
|
|
|
@ -21,13 +21,16 @@ bool Database::isFileRelevant(const char *filePath, const char *fileName, mode_t
|
|||
return !std::strcmp(fileName, "desc") || !std::strcmp(fileName, "depends") || !std::strcmp(fileName, "files");
|
||||
}
|
||||
|
||||
void Database::loadPackages(bool withFiles)
|
||||
void Database::loadPackagesFromConfiguredPaths(bool withFiles, bool force)
|
||||
{
|
||||
const auto &dbPath = withFiles && !filesPath.empty() ? filesPath : path;
|
||||
if (dbPath.empty()) {
|
||||
throw runtime_error("local path not configured");
|
||||
}
|
||||
loadPackages(extractFiles(dbPath, &isFileRelevant), lastModified(dbPath));
|
||||
const auto lastFileUpdate = lastModified(dbPath);
|
||||
if (force || lastFileUpdate > lastUpdate) {
|
||||
loadPackages(extractFiles(dbPath, &isFileRelevant), lastFileUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
void LibPkg::Database::loadPackages(const string &databaseData, DateTime lastModified)
|
||||
|
|
|
@ -305,7 +305,7 @@ void ParserTests::testParsingDatabase()
|
|||
db->filesPath = testFilePath("core.files");
|
||||
|
||||
// load packages
|
||||
config.loadAllPackages(true);
|
||||
config.loadAllPackages(true, false);
|
||||
CPPUNIT_ASSERT_EQUAL_MESSAGE("all 215 packages present"s, 215_st, db->packageCount());
|
||||
const auto autoreconf = db->findPackage("autoconf");
|
||||
CPPUNIT_ASSERT_MESSAGE("autoreconf exists", autoreconf != nullptr);
|
||||
|
|
|
@ -78,7 +78,14 @@ BuildActionMetaInfo::BuildActionMetaInfo()
|
|||
.category = "Repo management",
|
||||
.name = "Reload databases",
|
||||
.type = "reload-database",
|
||||
.flags = {},
|
||||
.flags = {
|
||||
BuildActionFlagMetaInfo{
|
||||
.id = static_cast<BuildActionFlagType>(ReloadDatabaseFlags::ForceReload),
|
||||
.name = "Force reload",
|
||||
.desc = "Load the database even if its last modification data isn't newer than the last time the database was loaded",
|
||||
.param = "force-reload",
|
||||
},
|
||||
},
|
||||
.settings = {},
|
||||
.directory = false,
|
||||
.sourceDb = false,
|
||||
|
|
|
@ -55,6 +55,10 @@ enum class CheckForUpdatesFlags : BuildActionFlagType {
|
|||
None,
|
||||
ConsiderRegularPackage = (1 << 0), // be consistent with LibPkg::UpdateCheckOptions here
|
||||
};
|
||||
enum class ReloadDatabaseFlags : BuildActionFlagType {
|
||||
None,
|
||||
ForceReload = (1 << 0),
|
||||
};
|
||||
enum class ReloadLibraryDependenciesFlags : BuildActionFlagType {
|
||||
None,
|
||||
ForceReload = (1 << 0),
|
||||
|
@ -179,6 +183,7 @@ inline const BuildActionTypeMetaMapping &BuildActionMetaInfo::mappingForId(Build
|
|||
|
||||
} // namespace LibRepoMgr
|
||||
|
||||
CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ReloadDatabaseFlags)
|
||||
CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ReloadLibraryDependenciesFlags)
|
||||
CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::PrepareBuildFlags)
|
||||
CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ConductBuildFlags)
|
||||
|
|
|
@ -28,6 +28,8 @@ ReloadDatabase::ReloadDatabase(ServiceSetup &setup, const std::shared_ptr<BuildA
|
|||
|
||||
void ReloadDatabase::run()
|
||||
{
|
||||
const auto flags = static_cast<ReloadDatabaseFlags>(m_buildAction->flags);
|
||||
const auto force = flags & ReloadDatabaseFlags::ForceReload;
|
||||
const auto withFiles = m_setup.building.loadFilesDbs;
|
||||
vector<LibPkg::Database *> dbsToLoadFromMirror;
|
||||
|
||||
|
@ -70,15 +72,26 @@ void ReloadDatabase::run()
|
|||
continue;
|
||||
}
|
||||
boost::asio::post(
|
||||
m_setup.building.ioContext.get_executor(), [this, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
|
||||
m_buildAction->appendOutput(
|
||||
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
||||
m_setup.building.ioContext.get_executor(), [this, force, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
|
||||
try {
|
||||
auto dbFileLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(dbName, dbArch));
|
||||
const auto lastModified = LibPkg::lastModified(dbPath);
|
||||
if (!force) {
|
||||
auto configReadLock2 = m_setup.config.lockToRead();
|
||||
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
||||
if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) {
|
||||
configReadLock2.unlock();
|
||||
m_buildAction->appendOutput(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
|
||||
"\" from local file \"", dbPath, "\"; last modification time <= last update (", lastModified.toString(), '<', '=',
|
||||
lastUpdate.toString(), ')', '\n');
|
||||
return;
|
||||
}
|
||||
}
|
||||
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
|
||||
auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile));
|
||||
dbFileLock.lock().unlock();
|
||||
m_buildAction->appendOutput(
|
||||
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
||||
const auto configLock = m_setup.config.lockToWrite();
|
||||
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
||||
if (!destinationDb) {
|
||||
|
@ -99,7 +112,7 @@ void ReloadDatabase::run()
|
|||
// query databases
|
||||
auto query = WebClient::prepareDatabaseQuery(m_buildAction->log(), dbsToLoadFromMirror, withFiles);
|
||||
std::get<std::shared_lock<std::shared_mutex>>(configReadLock).unlock();
|
||||
WebClient::queryDatabases(m_buildAction->log(), m_setup, std::move(query.queryParamsForDbs), session);
|
||||
WebClient::queryDatabases(m_buildAction->log(), m_setup, std::move(query.queryParamsForDbs), session, force);
|
||||
m_preparationFailures = std::move(query.failedDbs);
|
||||
|
||||
// clear AUR cache
|
||||
|
|
|
@ -653,7 +653,7 @@ void CleanRepository::run()
|
|||
std::make_unique<LibPkg::Database>(argsToString("clean-repository-", dbFilePaths.front().stem()), dbFilePaths.front()));
|
||||
db->arch = dirInfo.canonicalPath.stem();
|
||||
db->initStorage(*m_setup.config.storage());
|
||||
db->loadPackages();
|
||||
db->loadPackagesFromConfiguredPaths();
|
||||
dirInfo.relevantDbs.emplace(db.get());
|
||||
// acquire lock for db directory
|
||||
dirInfo.lock.emplace<SharedLoggingLock>(m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(*db)));
|
||||
|
|
|
@ -636,7 +636,7 @@ void ServiceSetup::run()
|
|||
#endif
|
||||
loadConfigFiles(true);
|
||||
config.discardDatabases();
|
||||
config.loadAllPackages(building.loadFilesDbs);
|
||||
config.loadAllPackages(building.loadFilesDbs, building.forceLoadingDbs);
|
||||
#ifndef CPP_UTILITIES_DEBUG_BUILD
|
||||
} catch (const std::exception &e) {
|
||||
cerr << Phrases::SubError << e.what() << endl;
|
||||
|
|
|
@ -107,6 +107,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
|
|||
std::string testFilesDir;
|
||||
BuildPresets presets;
|
||||
bool loadFilesDbs = false;
|
||||
bool forceLoadingDbs = false;
|
||||
|
||||
// never changed after startup
|
||||
unsigned short threadCount = 4;
|
||||
|
@ -151,7 +152,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
|
|||
void restoreLibraryDependenciesFromJson(const std::string &json, ReflectiveRapidJSON::JsonDeserializationErrors *errors);
|
||||
std::size_t restoreState();
|
||||
std::size_t saveState();
|
||||
void initStorage();
|
||||
void initStorage();
|
||||
void run();
|
||||
ServiceStatus computeStatus() const;
|
||||
};
|
||||
|
|
|
@ -550,7 +550,7 @@ void BuildActionsTests::testConductingBuild()
|
|||
readFile("repos/boost/os/x86_64/boost-libs-1.73.0-1-x86_64.pkg.tar.zst.sig"));
|
||||
|
||||
// add packages needing a rebuild to trigger auto-staging
|
||||
m_setup.config.loadAllPackages(false);
|
||||
m_setup.config.loadAllPackages(false, true);
|
||||
auto *const boostDb = m_setup.config.findDatabase("boost"sv, "x86_64"sv);
|
||||
auto *const miscDb = m_setup.config.findDatabase("misc"sv, "x86_64"sv);
|
||||
CPPUNIT_ASSERT_MESSAGE("boost database present", boostDb);
|
||||
|
@ -670,7 +670,7 @@ void BuildActionsTests::testCleanup()
|
|||
auto *const miscDb = m_setup.config.findOrCreateDatabase("misc"sv, std::string_view());
|
||||
miscDb->path = repoDir64 / "misc.db";
|
||||
miscDb->localDbDir = miscDb->localPkgDir = repoDir64;
|
||||
miscDb->loadPackages();
|
||||
miscDb->loadPackagesFromConfiguredPaths();
|
||||
|
||||
// create and run build action
|
||||
m_buildAction = std::make_shared<BuildAction>(0, &m_setup);
|
||||
|
|
|
@ -345,8 +345,10 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler)
|
|||
|
||||
void postLoadPackages(const Params ¶ms, ResponseHandler &&handler)
|
||||
{
|
||||
const auto withFiles = params.target.hasFlag("with-files");
|
||||
const auto force = params.target.hasFlag("force");
|
||||
auto lock = params.setup.config.lockToWrite();
|
||||
params.setup.config.loadAllPackages(params.target.hasFlag("with-files"));
|
||||
params.setup.config.loadAllPackages(withFiles, force);
|
||||
lock.unlock();
|
||||
handler(makeText(params.request(), "packages loaded"));
|
||||
}
|
||||
|
|
|
@ -56,103 +56,138 @@ static int matchToInt(const std::sub_match<const char *> &match)
|
|||
return stringToNumber<int>(sv);
|
||||
}
|
||||
|
||||
void queryDatabases(
|
||||
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&dbQueries, std::shared_ptr<DatabaseQuerySession> &dbQuerySession)
|
||||
static CppUtilities::DateTime parseLastModified(LogContext &log, const auto &message, std::string_view dbName, std::string_view dbArch)
|
||||
{
|
||||
auto lastModified = DateTime();
|
||||
const auto lastModifiedHeader = message.find(boost::beast::http::field::last_modified);
|
||||
if (lastModifiedHeader != message.cend()) {
|
||||
// parse "Last-Modified" header which should be something like "<day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT"
|
||||
const auto lastModifiedStr = lastModifiedHeader->value();
|
||||
static const auto lastModifiedPattern = std::regex("..., (\\d\\d) (...) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT");
|
||||
static const auto months = unordered_map<string, int>{ { "Jan", 1 }, { "Feb", 2 }, { "Mar", 3 }, { "Apr", 4 }, { "May", 5 }, { "Jun", 6 },
|
||||
{ "Jul", 7 }, { "Aug", 8 }, { "Sep", 9 }, { "Oct", 10 }, { "Nov", 11 }, { "Dec", 12 } };
|
||||
try {
|
||||
auto match = std::cmatch();
|
||||
if (!std::regex_search(lastModifiedStr.cbegin(), lastModifiedStr.cend(), match, lastModifiedPattern)) {
|
||||
throw ConversionException("date/time denotation not in expected format");
|
||||
}
|
||||
const auto month = months.find(match[2]);
|
||||
if (month == months.cend()) {
|
||||
throw ConversionException("month is invalid");
|
||||
}
|
||||
lastModified = DateTime::fromDateAndTime(
|
||||
matchToInt(match[3]), month->second, matchToInt(match[1]), matchToInt(match[4]), matchToInt(match[5]), matchToInt(match[6]));
|
||||
} catch (const ConversionException &e) {
|
||||
log(Phrases::WarningMessage, "Unable to parse \"Last-Modified\" header for database ", dbName, '@', dbArch, ": ", e.what(),
|
||||
" (last modification time was \"", string_view(lastModifiedStr.data(), lastModifiedStr.size()), "\")\n");
|
||||
}
|
||||
}
|
||||
return lastModified;
|
||||
}
|
||||
|
||||
void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&dbQueries,
|
||||
std::shared_ptr<DatabaseQuerySession> &dbQuerySession, bool force)
|
||||
{
|
||||
for (auto &query : dbQueries) {
|
||||
log(Phrases::InfoMessage, "Retrieving \"", query.databaseName, "\" from mirror: ", query.url, '\n');
|
||||
auto session = runSessionFromUrl(
|
||||
setup.building.ioContext, setup.webServer.sslContext, query.url,
|
||||
[&log, &setup, dbName = std::move(query.databaseName), dbArch = std::move(query.databaseArch), dbQuerySession](
|
||||
Session &session2, const WebClient::HttpClientError &error) mutable {
|
||||
if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) {
|
||||
log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": ",
|
||||
error.what(), '\n');
|
||||
dbQuerySession->addResponse(std::move(dbName));
|
||||
return;
|
||||
}
|
||||
auto headHandler
|
||||
= force ? Session::HeadHandler() : [&log, &setup, dbName = query.databaseName, dbArch = query.databaseArch](Session &session3) mutable {
|
||||
auto lastModified = parseLastModified(log, session3.headResponse.get(), dbName, dbArch);
|
||||
auto configReadLock = setup.config.lockToRead();
|
||||
auto *const destinationDb = setup.config.findDatabase(dbName, dbArch);
|
||||
if (!destinationDb) {
|
||||
log(Phrases::InfoMessage, "Skip requesting database \"", dbName, '@', dbArch, "\" as it no longer exists\n");
|
||||
session3.skip = true;
|
||||
return;
|
||||
}
|
||||
const auto lastUpdate = destinationDb->lastUpdate;
|
||||
configReadLock.unlock();
|
||||
if (lastModified > lastUpdate) {
|
||||
return;
|
||||
}
|
||||
log(Phrases::InfoMessage, "Skip requesting database \"", dbName, '@', dbArch,
|
||||
"\" from mirror; last modification time <= last update (", lastModified.toString(), " <= ", lastUpdate.toString(), ')', '\n');
|
||||
session3.skip = true;
|
||||
};
|
||||
auto handler = [&log, &setup, dbName = std::move(query.databaseName), dbArch = std::move(query.databaseArch), dbQuerySession, force](
|
||||
Session &session2, const WebClient::HttpClientError &error) mutable {
|
||||
if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) {
|
||||
log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": ", error.what(),
|
||||
'\n');
|
||||
dbQuerySession->addResponse(std::move(dbName));
|
||||
return;
|
||||
}
|
||||
if (session2.skip) {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto &response = get<FileResponse>(session2.response);
|
||||
const auto &message = response.get();
|
||||
if (message.result() != boost::beast::http::status::ok) {
|
||||
log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName,
|
||||
": mirror returned ", message.result_int(), " response\n");
|
||||
dbQuerySession->addResponse(std::move(dbName));
|
||||
return;
|
||||
}
|
||||
const auto &response = std::get<FileResponse>(session2.response);
|
||||
const auto &message = response.get();
|
||||
if (message.result() != boost::beast::http::status::ok) {
|
||||
log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": mirror returned ",
|
||||
message.result_int(), " response\n");
|
||||
dbQuerySession->addResponse(std::move(dbName));
|
||||
return;
|
||||
}
|
||||
|
||||
// find last modification time
|
||||
auto lastModified = DateTime();
|
||||
const auto lastModifiedHeader = message.find(boost::beast::http::field::last_modified);
|
||||
if (lastModifiedHeader != message.cend()) {
|
||||
// parse "Last-Modified" header which should be something like "<day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT"
|
||||
const auto lastModifiedStr = lastModifiedHeader->value();
|
||||
static const auto lastModifiedPattern = regex("..., (\\d\\d) (...) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT");
|
||||
static const auto months = unordered_map<string, int>{ { "Jan", 1 }, { "Feb", 2 }, { "Mar", 3 }, { "Apr", 4 }, { "May", 5 },
|
||||
{ "Jun", 6 }, { "Jul", 7 }, { "Aug", 8 }, { "Sep", 9 }, { "Oct", 10 }, { "Nov", 11 }, { "Dec", 12 } };
|
||||
try {
|
||||
cmatch match;
|
||||
if (!regex_search(lastModifiedStr.cbegin(), lastModifiedStr.cend(), match, lastModifiedPattern)) {
|
||||
throw ConversionException("date/time denotation not in expected format");
|
||||
}
|
||||
const auto month = months.find(match[2]);
|
||||
if (month == months.cend()) {
|
||||
throw ConversionException("month is invalid");
|
||||
}
|
||||
lastModified = DateTime::fromDateAndTime(matchToInt(match[3]), month->second, matchToInt(match[1]), matchToInt(match[4]),
|
||||
matchToInt(match[5]), matchToInt(match[6]));
|
||||
} catch (const ConversionException &e) {
|
||||
log(Phrases::WarningMessage, "Unable to parse \"Last-Modified\" header for database ", dbName, ": ", e.what(),
|
||||
" (last modification time was \"", string_view(lastModifiedStr.data(), lastModifiedStr.size()), "\")\n");
|
||||
}
|
||||
}
|
||||
|
||||
// log
|
||||
if (lastModified.isNull()) {
|
||||
log(Phrases::InfoMessage, "Loading database \"", dbName, "\" from mirror response\n");
|
||||
lastModified = DateTime::gmtNow();
|
||||
} else {
|
||||
log(Phrases::InfoMessage, "Loading database \"", dbName,
|
||||
"\" from mirror response; last modification time: ", lastModified.toString(), '\n');
|
||||
}
|
||||
|
||||
try {
|
||||
// load packages
|
||||
auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant);
|
||||
auto packages = Package::fromDatabaseFile(move(files));
|
||||
|
||||
// insert packages
|
||||
auto lock = setup.config.lockToWrite();
|
||||
auto db = setup.config.findDatabase(dbName, dbArch);
|
||||
if (!db) {
|
||||
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
||||
// log/skip
|
||||
auto lastModified = parseLastModified(log, message, dbName, dbArch);
|
||||
if (lastModified.isNull()) {
|
||||
log(Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from mirror response\n");
|
||||
lastModified = DateTime::gmtNow();
|
||||
} else if (!force) {
|
||||
auto configReadLock = setup.config.lockToRead();
|
||||
if (auto *const destinationDb = setup.config.findDatabase(dbName, dbArch)) {
|
||||
if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) {
|
||||
configReadLock.unlock();
|
||||
log(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
|
||||
"\" from mirror response; last modification time <= last update (", lastModified.toString(),
|
||||
" <= ", lastUpdate.toString(), ')', '\n');
|
||||
return;
|
||||
}
|
||||
db->replacePackages(packages, lastModified);
|
||||
} catch (const std::runtime_error &e) {
|
||||
log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
|
||||
dbQuerySession->addResponse(std::move(dbName));
|
||||
}
|
||||
},
|
||||
move(query.destinationFilePath));
|
||||
} else {
|
||||
log(Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch,
|
||||
"\" from mirror response; last modification time: ", lastModified.toString(), '\n');
|
||||
}
|
||||
|
||||
try {
|
||||
// load packages
|
||||
auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant);
|
||||
auto packages = Package::fromDatabaseFile(std::move(files));
|
||||
|
||||
// insert packages
|
||||
auto lock = setup.config.lockToWrite();
|
||||
auto db = setup.config.findDatabase(dbName, dbArch);
|
||||
if (!db) {
|
||||
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
||||
return;
|
||||
}
|
||||
db->replacePackages(packages, lastModified);
|
||||
} catch (const std::runtime_error &e) {
|
||||
log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
|
||||
dbQuerySession->addResponse(std::move(dbName));
|
||||
}
|
||||
};
|
||||
auto session = runSessionFromUrl(setup.building.ioContext, setup.webServer.sslContext, query.url, std::move(handler), std::move(headHandler),
|
||||
std::move(query.destinationFilePath));
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<DatabaseQuerySession> queryDatabases(
|
||||
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, DatabaseQuerySession::HandlerType &&handler)
|
||||
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, bool force, DatabaseQuerySession::HandlerType &&handler)
|
||||
{
|
||||
auto dbQuerySession = DatabaseQuerySession::create(setup.building.ioContext, move(handler));
|
||||
queryDatabases(log, setup, move(urls), dbQuerySession);
|
||||
queryDatabases(log, setup, move(urls), dbQuerySession, force);
|
||||
return dbQuerySession;
|
||||
}
|
||||
|
||||
std::shared_ptr<DatabaseQuerySession> queryDatabases(LogContext &log, ServiceSetup &setup, std::shared_lock<shared_mutex> *configReadLock,
|
||||
const std::vector<Database *> &dbs, DatabaseQuerySession::HandlerType &&handler)
|
||||
const std::vector<Database *> &dbs, bool force, DatabaseQuerySession::HandlerType &&handler)
|
||||
{
|
||||
shared_lock<shared_mutex> ownConfigReadLock;
|
||||
auto query = prepareDatabaseQuery(log, dbs, setup.building.loadFilesDbs);
|
||||
configReadLock->unlock();
|
||||
return queryDatabases(log, setup, std::move(query.queryParamsForDbs), move(handler));
|
||||
return queryDatabases(log, setup, std::move(query.queryParamsForDbs), force, move(handler));
|
||||
}
|
||||
|
||||
PackageCachingSession::PackageCachingSession(PackageCachingDataForSession &data, boost::asio::io_context &ioContext,
|
||||
|
|
|
@ -30,11 +30,11 @@ struct DatabaseQuery {
|
|||
|
||||
[[nodiscard]] DatabaseQuery prepareDatabaseQuery(LogContext &log, const std::vector<LibPkg::Database *> &dbs, bool withFiles);
|
||||
std::shared_ptr<DatabaseQuerySession> queryDatabases(
|
||||
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, DatabaseQuerySession::HandlerType &&handler);
|
||||
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, bool force, DatabaseQuerySession::HandlerType &&handler);
|
||||
std::shared_ptr<DatabaseQuerySession> queryDatabases(LogContext &log, ServiceSetup &setup, std::shared_lock<std::shared_mutex> *configReadLock,
|
||||
const std::vector<LibPkg::Database *> &dbs, DatabaseQuerySession::HandlerType &&handler);
|
||||
void queryDatabases(
|
||||
LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, std::shared_ptr<DatabaseQuerySession> &dbQuerySession);
|
||||
const std::vector<LibPkg::Database *> &dbs, bool force, DatabaseQuerySession::HandlerType &&handler);
|
||||
void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls,
|
||||
std::shared_ptr<DatabaseQuerySession> &dbQuerySession, bool force = false);
|
||||
|
||||
struct PackageCachingDataForPackage {
|
||||
std::string_view url;
|
||||
|
|
|
@ -57,10 +57,10 @@ void Session::run(
|
|||
|
||||
// set up an HTTP request message
|
||||
request.version(version);
|
||||
request.method(verb);
|
||||
request.target(target);
|
||||
request.set(http::field::host, host);
|
||||
request.set(http::field::user_agent, APP_NAME " " APP_VERSION);
|
||||
method = verb;
|
||||
|
||||
// setup a file response
|
||||
if (!destinationFilePath.empty()) {
|
||||
|
@ -131,6 +131,17 @@ void Session::handshakeDone(boost::beast::error_code ec)
|
|||
void Session::sendRequest()
|
||||
{
|
||||
// send the HTTP request to the remote host
|
||||
if (m_headHandler) {
|
||||
request.method(boost::beast::http::verb::head);
|
||||
std::visit(
|
||||
[this](auto &&stream) {
|
||||
boost::beast::http::async_write(
|
||||
stream, request, std::bind(&Session::headRequested, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
||||
},
|
||||
m_stream);
|
||||
return;
|
||||
}
|
||||
request.method(method);
|
||||
std::visit(
|
||||
[this](auto &&stream) {
|
||||
boost::beast::http::async_write(
|
||||
|
@ -139,6 +150,24 @@ void Session::sendRequest()
|
|||
m_stream);
|
||||
}
|
||||
|
||||
void Session::headRequested(boost::beast::error_code ec, std::size_t bytesTransferred)
|
||||
{
|
||||
boost::ignore_unused(bytesTransferred);
|
||||
if (ec) {
|
||||
m_handler(*this, HttpClientError("sending HEAD request", ec));
|
||||
return;
|
||||
}
|
||||
|
||||
// receive the HTTP response
|
||||
headResponse.skip(true);
|
||||
std::visit(
|
||||
[this](auto &stream) {
|
||||
http::async_read(
|
||||
stream, m_buffer, headResponse, std::bind(&Session::headReceived, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
||||
},
|
||||
m_stream);
|
||||
}
|
||||
|
||||
void Session::requested(boost::beast::error_code ec, std::size_t bytesTransferred)
|
||||
{
|
||||
boost::ignore_unused(bytesTransferred);
|
||||
|
@ -222,6 +251,22 @@ bool Session::continueReadingChunks()
|
|||
return true;
|
||||
}
|
||||
|
||||
void Session::headReceived(boost::beast::error_code ec, std::size_t bytesTransferred)
|
||||
{
|
||||
boost::ignore_unused(bytesTransferred);
|
||||
if (ec) {
|
||||
m_handler(*this, HttpClientError("receiving HEAD response", ec));
|
||||
return;
|
||||
}
|
||||
m_headHandler(*this);
|
||||
m_headHandler = HeadHandler();
|
||||
if (skip) {
|
||||
sendRequest();
|
||||
} else {
|
||||
closeGracefully();
|
||||
}
|
||||
}
|
||||
|
||||
void Session::received(boost::beast::error_code ec, std::size_t bytesTransferred)
|
||||
{
|
||||
boost::ignore_unused(bytesTransferred);
|
||||
|
@ -229,8 +274,12 @@ void Session::received(boost::beast::error_code ec, std::size_t bytesTransferred
|
|||
m_handler(*this, HttpClientError("receiving response", ec));
|
||||
return;
|
||||
}
|
||||
closeGracefully();
|
||||
}
|
||||
|
||||
// close the stream gracefully
|
||||
void Session::closeGracefully()
|
||||
{
|
||||
auto ec = boost::beast::error_code();
|
||||
if (auto *const sslStream = std::get_if<SslStream>(&m_stream)) {
|
||||
// perform the SSL handshake
|
||||
sslStream->async_shutdown(std::bind(&Session::closed, shared_from_this(), std::placeholders::_1));
|
||||
|
@ -249,6 +298,14 @@ void Session::closed(boost::beast::error_code ec)
|
|||
std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
|
||||
std::string_view url, Session::Handler &&handler, std::string &&destinationPath, std::string_view userName, std::string_view password,
|
||||
boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
|
||||
{
|
||||
return runSessionFromUrl(ioContext, sslContext, url, std::move(handler), Session::HeadHandler(), std::move(destinationPath), userName, password,
|
||||
verb, bodyLimit, std::move(chunkHandler));
|
||||
}
|
||||
|
||||
std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
|
||||
std::string_view url, Session::Handler &&handler, Session::HeadHandler &&headHandler, std::string &&destinationPath, std::string_view userName,
|
||||
std::string_view password, boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
|
||||
{
|
||||
std::string host, port, target;
|
||||
auto ssl = false;
|
||||
|
@ -284,8 +341,8 @@ std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asi
|
|||
port = ssl ? "443" : "80";
|
||||
}
|
||||
|
||||
auto session
|
||||
= ssl ? std::make_shared<Session>(ioContext, sslContext, std::move(handler)) : std::make_shared<Session>(ioContext, std::move(handler));
|
||||
auto session = ssl ? std::make_shared<Session>(ioContext, sslContext, std::move(handler), std::move(headHandler))
|
||||
: std::make_shared<Session>(ioContext, std::move(handler), std::move(headHandler));
|
||||
if (!userName.empty()) {
|
||||
const auto authInfo = userName % ":" + password;
|
||||
session->request.set(boost::beast::http::field::authorization,
|
||||
|
|
|
@ -45,6 +45,7 @@ inline LibRepoMgr::WebClient::HttpClientError::operator bool() const
|
|||
using Response = WebAPI::Response;
|
||||
using FileResponse = boost::beast::http::response_parser<boost::beast::http::file_body>;
|
||||
using StringResponse = boost::beast::http::response_parser<boost::beast::http::string_body>;
|
||||
using EmptyResponse = boost::beast::http::response_parser<boost::beast::http::empty_body>;
|
||||
using MultiResponse = std::variant<Response, FileResponse, StringResponse>;
|
||||
using Request = boost::beast::http::request<boost::beast::http::empty_body>;
|
||||
struct ChunkProcessing;
|
||||
|
@ -52,14 +53,17 @@ struct ChunkProcessing;
|
|||
class LIBREPOMGR_EXPORT Session : public std::enable_shared_from_this<Session> {
|
||||
public:
|
||||
using Handler = std::function<void(Session &, const HttpClientError &error)>;
|
||||
using HeadHandler = std::function<void(Session &)>;
|
||||
using ChunkHandler = std::function<void(const boost::beast::http::chunk_extensions &chunkExtensions, std::string_view chunkData)>;
|
||||
|
||||
template <typename ResponseType = Response> explicit Session(boost::asio::io_context &ioContext, const Handler &handler = Handler());
|
||||
template <typename ResponseType = Response> explicit Session(boost::asio::io_context &ioContext, Handler &&handler = Handler());
|
||||
template <typename ResponseType = Response>
|
||||
explicit Session(boost::asio::io_context &ioContext, Handler &&handler = Handler(), HeadHandler &&headHandler = HeadHandler());
|
||||
template <typename ResponseType = Response>
|
||||
explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, const Handler &handler = Handler());
|
||||
template <typename ResponseType = Response>
|
||||
explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler);
|
||||
explicit Session(
|
||||
boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler, HeadHandler &&headHandler = HeadHandler());
|
||||
|
||||
void setChunkHandler(ChunkHandler &&handler);
|
||||
void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target,
|
||||
|
@ -83,25 +87,33 @@ private:
|
|||
void connected(boost::beast::error_code ec);
|
||||
void handshakeDone(boost::beast::error_code ec);
|
||||
void sendRequest();
|
||||
void headRequested(boost::beast::error_code ec, std::size_t bytesTransferred);
|
||||
void requested(boost::beast::error_code ec, std::size_t bytesTransferred);
|
||||
void onChunkHeader(std::uint64_t chunkSize, boost::beast::string_view extensions, boost::beast::error_code &ec);
|
||||
std::size_t onChunkBody(std::uint64_t bytesLeftInThisChunk, boost::beast::string_view chunkBodyData, boost::beast::error_code &ec);
|
||||
void chunkReceived(boost::beast::error_code ec, std::size_t bytesTransferred);
|
||||
bool continueReadingChunks();
|
||||
void headReceived(boost::beast::error_code ec, std::size_t bytesTransferred);
|
||||
void received(boost::beast::error_code ec, std::size_t bytesTransferred);
|
||||
void closeGracefully();
|
||||
void closed(boost::beast::error_code ec);
|
||||
|
||||
public:
|
||||
Request request;
|
||||
MultiResponse response;
|
||||
EmptyResponse headResponse;
|
||||
std::string destinationFilePath;
|
||||
boost::beast::http::verb method = boost::beast::http::verb::get;
|
||||
bool skip = false;
|
||||
|
||||
private:
|
||||
boost::asio::ip::tcp::resolver m_resolver;
|
||||
std::variant<RawSocket, SslStream> m_stream;
|
||||
boost::beast::flat_buffer m_buffer;
|
||||
std::unique_ptr<ChunkProcessing> m_chunkProcessing;
|
||||
Request m_headRequest;
|
||||
Handler m_handler;
|
||||
HeadHandler m_headHandler;
|
||||
};
|
||||
|
||||
template <typename ResponseType>
|
||||
|
@ -114,11 +126,12 @@ inline Session::Session(boost::asio::io_context &ioContext, const Handler &handl
|
|||
}
|
||||
|
||||
template <typename ResponseType>
|
||||
inline Session::Session(boost::asio::io_context &ioContext, Handler &&handler)
|
||||
inline Session::Session(boost::asio::io_context &ioContext, Handler &&handler, HeadHandler &&headHandler)
|
||||
: response(ResponseType{})
|
||||
, m_resolver(ioContext)
|
||||
, m_stream(RawSocket{ ioContext })
|
||||
, m_handler(std::move(handler))
|
||||
, m_headHandler(std::move(headHandler))
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -132,11 +145,12 @@ inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::co
|
|||
}
|
||||
|
||||
template <typename ResponseType>
|
||||
inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler)
|
||||
inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler, HeadHandler &&headHandler)
|
||||
: response(ResponseType{})
|
||||
, m_resolver(ioContext)
|
||||
, m_stream(SslStream{ ioContext, sslContext })
|
||||
, m_handler(std::move(handler))
|
||||
, m_headHandler(std::move(headHandler))
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -145,6 +159,11 @@ LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSession
|
|||
std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
|
||||
boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
|
||||
Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
|
||||
LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext,
|
||||
boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, Session::HeadHandler &&headHandler,
|
||||
std::string &&destinationPath = std::string(), std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
|
||||
boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
|
||||
Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
|
||||
|
||||
} // namespace WebClient
|
||||
} // namespace LibRepoMgr
|
||||
|
|
|
@ -92,7 +92,7 @@ int main(int argc, const char *argv[])
|
|||
} else {
|
||||
db.filesPath = db.filesPathFromRegularPath();
|
||||
}
|
||||
db.loadPackages(true);
|
||||
db.loadPackagesFromConfiguredPaths(true, true);
|
||||
} catch (const std::runtime_error &e) {
|
||||
std::cerr << "Unable to load database \"" << db.name << "\": " << e.what() << '\n';
|
||||
}
|
||||
|
|
10
srv/main.cpp
10
srv/main.cpp
|
@ -30,12 +30,14 @@ int main(int argc, const char *argv[])
|
|||
OperationArgument runArg("run", 'r', "runs the server");
|
||||
ConfigValueArgument configFileArg("config-file", 'c', "specifies the path of the config file", { "path" });
|
||||
configFileArg.setEnvironmentVariable(PROJECT_VARNAME_UPPER "_CONFIG_FILE");
|
||||
runArg.setSubArguments({ &configFileArg });
|
||||
ConfigValueArgument forceLoadingDBsArg("force-loading-dbs", 'f', "forces loading DBs, even if DB files have not been modified since last parse");
|
||||
runArg.setSubArguments({ &configFileArg, &forceLoadingDBsArg });
|
||||
runArg.setImplicit(true);
|
||||
runArg.setCallback([&setup, &configFileArg](const ArgumentOccurrence &) {
|
||||
if (configFileArg.firstValue()) {
|
||||
setup.configFilePath = configFileArg.firstValue();
|
||||
runArg.setCallback([&setup, &configFileArg, &forceLoadingDBsArg](const ArgumentOccurrence &) {
|
||||
if (const auto configFilePath = configFileArg.firstValue()) {
|
||||
setup.configFilePath = configFilePath;
|
||||
}
|
||||
setup.building.forceLoadingDbs = forceLoadingDBsArg.isPresent();
|
||||
setup.run();
|
||||
});
|
||||
HelpArgument helpArg(parser);
|
||||
|
|
Loading…
Reference in New Issue