diff --git a/libpkg/data/config.h b/libpkg/data/config.h index c8c0fba..a72755f 100644 --- a/libpkg/data/config.h +++ b/libpkg/data/config.h @@ -111,7 +111,7 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar // load config and packages void loadPacmanConfig(const char *pacmanConfigPath); - void loadAllPackages(bool withFiles); + void loadAllPackages(bool withFiles, bool force); // storage and caching void initStorage(const char *path = "libpkg.db", std::uint32_t maxDbs = 0); diff --git a/libpkg/data/database.h b/libpkg/data/database.h index cdc5ff2..6217c44 100644 --- a/libpkg/data/database.h +++ b/libpkg/data/database.h @@ -125,7 +125,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable lastUpdate) { + loadPackages(extractFiles(dbPath, &isFileRelevant), lastFileUpdate); + } } void LibPkg::Database::loadPackages(const string &databaseData, DateTime lastModified) diff --git a/libpkg/tests/parser.cpp b/libpkg/tests/parser.cpp index 3869489..428f85f 100644 --- a/libpkg/tests/parser.cpp +++ b/libpkg/tests/parser.cpp @@ -305,7 +305,7 @@ void ParserTests::testParsingDatabase() db->filesPath = testFilePath("core.files"); // load packages - config.loadAllPackages(true); + config.loadAllPackages(true, false); CPPUNIT_ASSERT_EQUAL_MESSAGE("all 215 packages present"s, 215_st, db->packageCount()); const auto autoreconf = db->findPackage("autoconf"); CPPUNIT_ASSERT_MESSAGE("autoreconf exists", autoreconf != nullptr); diff --git a/librepomgr/buildactions/buildactionmeta.cpp b/librepomgr/buildactions/buildactionmeta.cpp index ff88b55..be9b4ce 100644 --- a/librepomgr/buildactions/buildactionmeta.cpp +++ b/librepomgr/buildactions/buildactionmeta.cpp @@ -78,7 +78,14 @@ BuildActionMetaInfo::BuildActionMetaInfo() .category = "Repo management", .name = "Reload databases", .type = "reload-database", - .flags = {}, + .flags = { + BuildActionFlagMetaInfo{ + .id = static_cast(ReloadDatabaseFlags::ForceReload), + .name = "Force reload", + .desc = "Load the database even if its last modification data isn't newer than the last time the database was loaded", + .param = "force-reload", + }, + }, .settings = {}, .directory = false, .sourceDb = false, diff --git a/librepomgr/buildactions/buildactionmeta.h b/librepomgr/buildactions/buildactionmeta.h index bfb7ecf..c2400f0 100644 --- a/librepomgr/buildactions/buildactionmeta.h +++ b/librepomgr/buildactions/buildactionmeta.h @@ -55,6 +55,10 @@ enum class CheckForUpdatesFlags : BuildActionFlagType { None, ConsiderRegularPackage = (1 << 0), // be consistent with LibPkg::UpdateCheckOptions here }; +enum class ReloadDatabaseFlags : BuildActionFlagType { + None, + ForceReload = (1 << 0), +}; enum class ReloadLibraryDependenciesFlags : BuildActionFlagType { None, ForceReload = (1 << 0), @@ -179,6 +183,7 @@ inline const BuildActionTypeMetaMapping &BuildActionMetaInfo::mappingForId(Build } // namespace LibRepoMgr +CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ReloadDatabaseFlags) CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ReloadLibraryDependenciesFlags) CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::PrepareBuildFlags) CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ConductBuildFlags) diff --git a/librepomgr/buildactions/reloaddatabase.cpp b/librepomgr/buildactions/reloaddatabase.cpp index 68b0587..ab99a8f 100644 --- a/librepomgr/buildactions/reloaddatabase.cpp +++ b/librepomgr/buildactions/reloaddatabase.cpp @@ -28,6 +28,8 @@ ReloadDatabase::ReloadDatabase(ServiceSetup &setup, const std::shared_ptr(m_buildAction->flags); + const auto force = flags & ReloadDatabaseFlags::ForceReload; const auto withFiles = m_setup.building.loadFilesDbs; vector dbsToLoadFromMirror; @@ -70,15 +72,26 @@ void ReloadDatabase::run() continue; } boost::asio::post( - m_setup.building.ioContext.get_executor(), [this, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable { - m_buildAction->appendOutput( - Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n"); + m_setup.building.ioContext.get_executor(), [this, force, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable { try { auto dbFileLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(dbName, dbArch)); const auto lastModified = LibPkg::lastModified(dbPath); + if (!force) { + auto configReadLock2 = m_setup.config.lockToRead(); + auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch); + if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) { + configReadLock2.unlock(); + m_buildAction->appendOutput(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch, + "\" from local file \"", dbPath, "\"; last modification time <= last update (", lastModified.toString(), '<', '=', + lastUpdate.toString(), ')', '\n'); + return; + } + } auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant); auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile)); dbFileLock.lock().unlock(); + m_buildAction->appendOutput( + Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n"); const auto configLock = m_setup.config.lockToWrite(); auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch); if (!destinationDb) { @@ -99,7 +112,7 @@ void ReloadDatabase::run() // query databases auto query = WebClient::prepareDatabaseQuery(m_buildAction->log(), dbsToLoadFromMirror, withFiles); std::get>(configReadLock).unlock(); - WebClient::queryDatabases(m_buildAction->log(), m_setup, std::move(query.queryParamsForDbs), session); + WebClient::queryDatabases(m_buildAction->log(), m_setup, std::move(query.queryParamsForDbs), session, force); m_preparationFailures = std::move(query.failedDbs); // clear AUR cache diff --git a/librepomgr/buildactions/repomanagement.cpp b/librepomgr/buildactions/repomanagement.cpp index fe4012c..ad4bd37 100644 --- a/librepomgr/buildactions/repomanagement.cpp +++ b/librepomgr/buildactions/repomanagement.cpp @@ -653,7 +653,7 @@ void CleanRepository::run() std::make_unique(argsToString("clean-repository-", dbFilePaths.front().stem()), dbFilePaths.front())); db->arch = dirInfo.canonicalPath.stem(); db->initStorage(*m_setup.config.storage()); - db->loadPackages(); + db->loadPackagesFromConfiguredPaths(); dirInfo.relevantDbs.emplace(db.get()); // acquire lock for db directory dirInfo.lock.emplace(m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(*db))); diff --git a/librepomgr/serversetup.cpp b/librepomgr/serversetup.cpp index 50c763d..c6c26f3 100644 --- a/librepomgr/serversetup.cpp +++ b/librepomgr/serversetup.cpp @@ -636,7 +636,7 @@ void ServiceSetup::run() #endif loadConfigFiles(true); config.discardDatabases(); - config.loadAllPackages(building.loadFilesDbs); + config.loadAllPackages(building.loadFilesDbs, building.forceLoadingDbs); #ifndef CPP_UTILITIES_DEBUG_BUILD } catch (const std::exception &e) { cerr << Phrases::SubError << e.what() << endl; diff --git a/librepomgr/serversetup.h b/librepomgr/serversetup.h index 4d4a7a3..c0fa37f 100644 --- a/librepomgr/serversetup.h +++ b/librepomgr/serversetup.h @@ -107,6 +107,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { std::string testFilesDir; BuildPresets presets; bool loadFilesDbs = false; + bool forceLoadingDbs = false; // never changed after startup unsigned short threadCount = 4; @@ -151,7 +152,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { void restoreLibraryDependenciesFromJson(const std::string &json, ReflectiveRapidJSON::JsonDeserializationErrors *errors); std::size_t restoreState(); std::size_t saveState(); - void initStorage(); + void initStorage(); void run(); ServiceStatus computeStatus() const; }; diff --git a/librepomgr/tests/buildactions.cpp b/librepomgr/tests/buildactions.cpp index 483c507..e3a6d6d 100644 --- a/librepomgr/tests/buildactions.cpp +++ b/librepomgr/tests/buildactions.cpp @@ -550,7 +550,7 @@ void BuildActionsTests::testConductingBuild() readFile("repos/boost/os/x86_64/boost-libs-1.73.0-1-x86_64.pkg.tar.zst.sig")); // add packages needing a rebuild to trigger auto-staging - m_setup.config.loadAllPackages(false); + m_setup.config.loadAllPackages(false, true); auto *const boostDb = m_setup.config.findDatabase("boost"sv, "x86_64"sv); auto *const miscDb = m_setup.config.findDatabase("misc"sv, "x86_64"sv); CPPUNIT_ASSERT_MESSAGE("boost database present", boostDb); @@ -670,7 +670,7 @@ void BuildActionsTests::testCleanup() auto *const miscDb = m_setup.config.findOrCreateDatabase("misc"sv, std::string_view()); miscDb->path = repoDir64 / "misc.db"; miscDb->localDbDir = miscDb->localPkgDir = repoDir64; - miscDb->loadPackages(); + miscDb->loadPackagesFromConfiguredPaths(); // create and run build action m_buildAction = std::make_shared(0, &m_setup); diff --git a/librepomgr/webapi/routes.cpp b/librepomgr/webapi/routes.cpp index 270fde6..48c9167 100644 --- a/librepomgr/webapi/routes.cpp +++ b/librepomgr/webapi/routes.cpp @@ -345,8 +345,10 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) void postLoadPackages(const Params ¶ms, ResponseHandler &&handler) { + const auto withFiles = params.target.hasFlag("with-files"); + const auto force = params.target.hasFlag("force"); auto lock = params.setup.config.lockToWrite(); - params.setup.config.loadAllPackages(params.target.hasFlag("with-files")); + params.setup.config.loadAllPackages(withFiles, force); lock.unlock(); handler(makeText(params.request(), "packages loaded")); } diff --git a/librepomgr/webclient/database.cpp b/librepomgr/webclient/database.cpp index 9a525ed..904c150 100644 --- a/librepomgr/webclient/database.cpp +++ b/librepomgr/webclient/database.cpp @@ -56,103 +56,138 @@ static int matchToInt(const std::sub_match &match) return stringToNumber(sv); } -void queryDatabases( - LogContext &log, ServiceSetup &setup, std::vector &&dbQueries, std::shared_ptr &dbQuerySession) +static CppUtilities::DateTime parseLastModified(LogContext &log, const auto &message, std::string_view dbName, std::string_view dbArch) +{ + auto lastModified = DateTime(); + const auto lastModifiedHeader = message.find(boost::beast::http::field::last_modified); + if (lastModifiedHeader != message.cend()) { + // parse "Last-Modified" header which should be something like ", :: GMT" + const auto lastModifiedStr = lastModifiedHeader->value(); + static const auto lastModifiedPattern = std::regex("..., (\\d\\d) (...) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT"); + static const auto months = unordered_map{ { "Jan", 1 }, { "Feb", 2 }, { "Mar", 3 }, { "Apr", 4 }, { "May", 5 }, { "Jun", 6 }, + { "Jul", 7 }, { "Aug", 8 }, { "Sep", 9 }, { "Oct", 10 }, { "Nov", 11 }, { "Dec", 12 } }; + try { + auto match = std::cmatch(); + if (!std::regex_search(lastModifiedStr.cbegin(), lastModifiedStr.cend(), match, lastModifiedPattern)) { + throw ConversionException("date/time denotation not in expected format"); + } + const auto month = months.find(match[2]); + if (month == months.cend()) { + throw ConversionException("month is invalid"); + } + lastModified = DateTime::fromDateAndTime( + matchToInt(match[3]), month->second, matchToInt(match[1]), matchToInt(match[4]), matchToInt(match[5]), matchToInt(match[6])); + } catch (const ConversionException &e) { + log(Phrases::WarningMessage, "Unable to parse \"Last-Modified\" header for database ", dbName, '@', dbArch, ": ", e.what(), + " (last modification time was \"", string_view(lastModifiedStr.data(), lastModifiedStr.size()), "\")\n"); + } + } + return lastModified; +} + +void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector &&dbQueries, + std::shared_ptr &dbQuerySession, bool force) { for (auto &query : dbQueries) { log(Phrases::InfoMessage, "Retrieving \"", query.databaseName, "\" from mirror: ", query.url, '\n'); - auto session = runSessionFromUrl( - setup.building.ioContext, setup.webServer.sslContext, query.url, - [&log, &setup, dbName = std::move(query.databaseName), dbArch = std::move(query.databaseArch), dbQuerySession]( - Session &session2, const WebClient::HttpClientError &error) mutable { - if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) { - log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": ", - error.what(), '\n'); - dbQuerySession->addResponse(std::move(dbName)); - return; - } + auto headHandler + = force ? Session::HeadHandler() : [&log, &setup, dbName = query.databaseName, dbArch = query.databaseArch](Session &session3) mutable { + auto lastModified = parseLastModified(log, session3.headResponse.get(), dbName, dbArch); + auto configReadLock = setup.config.lockToRead(); + auto *const destinationDb = setup.config.findDatabase(dbName, dbArch); + if (!destinationDb) { + log(Phrases::InfoMessage, "Skip requesting database \"", dbName, '@', dbArch, "\" as it no longer exists\n"); + session3.skip = true; + return; + } + const auto lastUpdate = destinationDb->lastUpdate; + configReadLock.unlock(); + if (lastModified > lastUpdate) { + return; + } + log(Phrases::InfoMessage, "Skip requesting database \"", dbName, '@', dbArch, + "\" from mirror; last modification time <= last update (", lastModified.toString(), " <= ", lastUpdate.toString(), ')', '\n'); + session3.skip = true; + }; + auto handler = [&log, &setup, dbName = std::move(query.databaseName), dbArch = std::move(query.databaseArch), dbQuerySession, force]( + Session &session2, const WebClient::HttpClientError &error) mutable { + if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) { + log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": ", error.what(), + '\n'); + dbQuerySession->addResponse(std::move(dbName)); + return; + } + if (session2.skip) { + return; + } - const auto &response = get(session2.response); - const auto &message = response.get(); - if (message.result() != boost::beast::http::status::ok) { - log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, - ": mirror returned ", message.result_int(), " response\n"); - dbQuerySession->addResponse(std::move(dbName)); - return; - } + const auto &response = std::get(session2.response); + const auto &message = response.get(); + if (message.result() != boost::beast::http::status::ok) { + log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": mirror returned ", + message.result_int(), " response\n"); + dbQuerySession->addResponse(std::move(dbName)); + return; + } - // find last modification time - auto lastModified = DateTime(); - const auto lastModifiedHeader = message.find(boost::beast::http::field::last_modified); - if (lastModifiedHeader != message.cend()) { - // parse "Last-Modified" header which should be something like ", :: GMT" - const auto lastModifiedStr = lastModifiedHeader->value(); - static const auto lastModifiedPattern = regex("..., (\\d\\d) (...) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT"); - static const auto months = unordered_map{ { "Jan", 1 }, { "Feb", 2 }, { "Mar", 3 }, { "Apr", 4 }, { "May", 5 }, - { "Jun", 6 }, { "Jul", 7 }, { "Aug", 8 }, { "Sep", 9 }, { "Oct", 10 }, { "Nov", 11 }, { "Dec", 12 } }; - try { - cmatch match; - if (!regex_search(lastModifiedStr.cbegin(), lastModifiedStr.cend(), match, lastModifiedPattern)) { - throw ConversionException("date/time denotation not in expected format"); - } - const auto month = months.find(match[2]); - if (month == months.cend()) { - throw ConversionException("month is invalid"); - } - lastModified = DateTime::fromDateAndTime(matchToInt(match[3]), month->second, matchToInt(match[1]), matchToInt(match[4]), - matchToInt(match[5]), matchToInt(match[6])); - } catch (const ConversionException &e) { - log(Phrases::WarningMessage, "Unable to parse \"Last-Modified\" header for database ", dbName, ": ", e.what(), - " (last modification time was \"", string_view(lastModifiedStr.data(), lastModifiedStr.size()), "\")\n"); - } - } - - // log - if (lastModified.isNull()) { - log(Phrases::InfoMessage, "Loading database \"", dbName, "\" from mirror response\n"); - lastModified = DateTime::gmtNow(); - } else { - log(Phrases::InfoMessage, "Loading database \"", dbName, - "\" from mirror response; last modification time: ", lastModified.toString(), '\n'); - } - - try { - // load packages - auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant); - auto packages = Package::fromDatabaseFile(move(files)); - - // insert packages - auto lock = setup.config.lockToWrite(); - auto db = setup.config.findDatabase(dbName, dbArch); - if (!db) { - log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n"); + // log/skip + auto lastModified = parseLastModified(log, message, dbName, dbArch); + if (lastModified.isNull()) { + log(Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from mirror response\n"); + lastModified = DateTime::gmtNow(); + } else if (!force) { + auto configReadLock = setup.config.lockToRead(); + if (auto *const destinationDb = setup.config.findDatabase(dbName, dbArch)) { + if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) { + configReadLock.unlock(); + log(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch, + "\" from mirror response; last modification time <= last update (", lastModified.toString(), + " <= ", lastUpdate.toString(), ')', '\n'); return; } - db->replacePackages(packages, lastModified); - } catch (const std::runtime_error &e) { - log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n'); - dbQuerySession->addResponse(std::move(dbName)); } - }, - move(query.destinationFilePath)); + } else { + log(Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, + "\" from mirror response; last modification time: ", lastModified.toString(), '\n'); + } + + try { + // load packages + auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant); + auto packages = Package::fromDatabaseFile(std::move(files)); + + // insert packages + auto lock = setup.config.lockToWrite(); + auto db = setup.config.findDatabase(dbName, dbArch); + if (!db) { + log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n"); + return; + } + db->replacePackages(packages, lastModified); + } catch (const std::runtime_error &e) { + log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n'); + dbQuerySession->addResponse(std::move(dbName)); + } + }; + auto session = runSessionFromUrl(setup.building.ioContext, setup.webServer.sslContext, query.url, std::move(handler), std::move(headHandler), + std::move(query.destinationFilePath)); } } std::shared_ptr queryDatabases( - LogContext &log, ServiceSetup &setup, std::vector &&urls, DatabaseQuerySession::HandlerType &&handler) + LogContext &log, ServiceSetup &setup, std::vector &&urls, bool force, DatabaseQuerySession::HandlerType &&handler) { auto dbQuerySession = DatabaseQuerySession::create(setup.building.ioContext, move(handler)); - queryDatabases(log, setup, move(urls), dbQuerySession); + queryDatabases(log, setup, move(urls), dbQuerySession, force); return dbQuerySession; } std::shared_ptr queryDatabases(LogContext &log, ServiceSetup &setup, std::shared_lock *configReadLock, - const std::vector &dbs, DatabaseQuerySession::HandlerType &&handler) + const std::vector &dbs, bool force, DatabaseQuerySession::HandlerType &&handler) { - shared_lock ownConfigReadLock; auto query = prepareDatabaseQuery(log, dbs, setup.building.loadFilesDbs); configReadLock->unlock(); - return queryDatabases(log, setup, std::move(query.queryParamsForDbs), move(handler)); + return queryDatabases(log, setup, std::move(query.queryParamsForDbs), force, move(handler)); } PackageCachingSession::PackageCachingSession(PackageCachingDataForSession &data, boost::asio::io_context &ioContext, diff --git a/librepomgr/webclient/database.h b/librepomgr/webclient/database.h index bd150b5..51fa345 100644 --- a/librepomgr/webclient/database.h +++ b/librepomgr/webclient/database.h @@ -30,11 +30,11 @@ struct DatabaseQuery { [[nodiscard]] DatabaseQuery prepareDatabaseQuery(LogContext &log, const std::vector &dbs, bool withFiles); std::shared_ptr queryDatabases( - LogContext &log, ServiceSetup &setup, std::vector &&urls, DatabaseQuerySession::HandlerType &&handler); + LogContext &log, ServiceSetup &setup, std::vector &&urls, bool force, DatabaseQuerySession::HandlerType &&handler); std::shared_ptr queryDatabases(LogContext &log, ServiceSetup &setup, std::shared_lock *configReadLock, - const std::vector &dbs, DatabaseQuerySession::HandlerType &&handler); -void queryDatabases( - LogContext &log, ServiceSetup &setup, std::vector &&urls, std::shared_ptr &dbQuerySession); + const std::vector &dbs, bool force, DatabaseQuerySession::HandlerType &&handler); +void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector &&urls, + std::shared_ptr &dbQuerySession, bool force = false); struct PackageCachingDataForPackage { std::string_view url; diff --git a/librepomgr/webclient/session.cpp b/librepomgr/webclient/session.cpp index 629937b..576cadc 100644 --- a/librepomgr/webclient/session.cpp +++ b/librepomgr/webclient/session.cpp @@ -57,10 +57,10 @@ void Session::run( // set up an HTTP request message request.version(version); - request.method(verb); request.target(target); request.set(http::field::host, host); request.set(http::field::user_agent, APP_NAME " " APP_VERSION); + method = verb; // setup a file response if (!destinationFilePath.empty()) { @@ -131,6 +131,17 @@ void Session::handshakeDone(boost::beast::error_code ec) void Session::sendRequest() { // send the HTTP request to the remote host + if (m_headHandler) { + request.method(boost::beast::http::verb::head); + std::visit( + [this](auto &&stream) { + boost::beast::http::async_write( + stream, request, std::bind(&Session::headRequested, shared_from_this(), std::placeholders::_1, std::placeholders::_2)); + }, + m_stream); + return; + } + request.method(method); std::visit( [this](auto &&stream) { boost::beast::http::async_write( @@ -139,6 +150,24 @@ void Session::sendRequest() m_stream); } +void Session::headRequested(boost::beast::error_code ec, std::size_t bytesTransferred) +{ + boost::ignore_unused(bytesTransferred); + if (ec) { + m_handler(*this, HttpClientError("sending HEAD request", ec)); + return; + } + + // receive the HTTP response + headResponse.skip(true); + std::visit( + [this](auto &stream) { + http::async_read( + stream, m_buffer, headResponse, std::bind(&Session::headReceived, shared_from_this(), std::placeholders::_1, std::placeholders::_2)); + }, + m_stream); +} + void Session::requested(boost::beast::error_code ec, std::size_t bytesTransferred) { boost::ignore_unused(bytesTransferred); @@ -222,6 +251,22 @@ bool Session::continueReadingChunks() return true; } +void Session::headReceived(boost::beast::error_code ec, std::size_t bytesTransferred) +{ + boost::ignore_unused(bytesTransferred); + if (ec) { + m_handler(*this, HttpClientError("receiving HEAD response", ec)); + return; + } + m_headHandler(*this); + m_headHandler = HeadHandler(); + if (skip) { + sendRequest(); + } else { + closeGracefully(); + } +} + void Session::received(boost::beast::error_code ec, std::size_t bytesTransferred) { boost::ignore_unused(bytesTransferred); @@ -229,8 +274,12 @@ void Session::received(boost::beast::error_code ec, std::size_t bytesTransferred m_handler(*this, HttpClientError("receiving response", ec)); return; } + closeGracefully(); +} - // close the stream gracefully +void Session::closeGracefully() +{ + auto ec = boost::beast::error_code(); if (auto *const sslStream = std::get_if(&m_stream)) { // perform the SSL handshake sslStream->async_shutdown(std::bind(&Session::closed, shared_from_this(), std::placeholders::_1)); @@ -249,6 +298,14 @@ void Session::closed(boost::beast::error_code ec) std::variant> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, std::string &&destinationPath, std::string_view userName, std::string_view password, boost::beast::http::verb verb, std::optional bodyLimit, Session::ChunkHandler &&chunkHandler) +{ + return runSessionFromUrl(ioContext, sslContext, url, std::move(handler), Session::HeadHandler(), std::move(destinationPath), userName, password, + verb, bodyLimit, std::move(chunkHandler)); +} + +std::variant> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, + std::string_view url, Session::Handler &&handler, Session::HeadHandler &&headHandler, std::string &&destinationPath, std::string_view userName, + std::string_view password, boost::beast::http::verb verb, std::optional bodyLimit, Session::ChunkHandler &&chunkHandler) { std::string host, port, target; auto ssl = false; @@ -284,8 +341,8 @@ std::variant> runSessionFromUrl(boost::asi port = ssl ? "443" : "80"; } - auto session - = ssl ? std::make_shared(ioContext, sslContext, std::move(handler)) : std::make_shared(ioContext, std::move(handler)); + auto session = ssl ? std::make_shared(ioContext, sslContext, std::move(handler), std::move(headHandler)) + : std::make_shared(ioContext, std::move(handler), std::move(headHandler)); if (!userName.empty()) { const auto authInfo = userName % ":" + password; session->request.set(boost::beast::http::field::authorization, diff --git a/librepomgr/webclient/session.h b/librepomgr/webclient/session.h index a43bdb0..95008ed 100644 --- a/librepomgr/webclient/session.h +++ b/librepomgr/webclient/session.h @@ -45,6 +45,7 @@ inline LibRepoMgr::WebClient::HttpClientError::operator bool() const using Response = WebAPI::Response; using FileResponse = boost::beast::http::response_parser; using StringResponse = boost::beast::http::response_parser; +using EmptyResponse = boost::beast::http::response_parser; using MultiResponse = std::variant; using Request = boost::beast::http::request; struct ChunkProcessing; @@ -52,14 +53,17 @@ struct ChunkProcessing; class LIBREPOMGR_EXPORT Session : public std::enable_shared_from_this { public: using Handler = std::function; + using HeadHandler = std::function; using ChunkHandler = std::function; template explicit Session(boost::asio::io_context &ioContext, const Handler &handler = Handler()); - template explicit Session(boost::asio::io_context &ioContext, Handler &&handler = Handler()); + template + explicit Session(boost::asio::io_context &ioContext, Handler &&handler = Handler(), HeadHandler &&headHandler = HeadHandler()); template explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, const Handler &handler = Handler()); template - explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler); + explicit Session( + boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler, HeadHandler &&headHandler = HeadHandler()); void setChunkHandler(ChunkHandler &&handler); void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, @@ -83,25 +87,33 @@ private: void connected(boost::beast::error_code ec); void handshakeDone(boost::beast::error_code ec); void sendRequest(); + void headRequested(boost::beast::error_code ec, std::size_t bytesTransferred); void requested(boost::beast::error_code ec, std::size_t bytesTransferred); void onChunkHeader(std::uint64_t chunkSize, boost::beast::string_view extensions, boost::beast::error_code &ec); std::size_t onChunkBody(std::uint64_t bytesLeftInThisChunk, boost::beast::string_view chunkBodyData, boost::beast::error_code &ec); void chunkReceived(boost::beast::error_code ec, std::size_t bytesTransferred); bool continueReadingChunks(); + void headReceived(boost::beast::error_code ec, std::size_t bytesTransferred); void received(boost::beast::error_code ec, std::size_t bytesTransferred); + void closeGracefully(); void closed(boost::beast::error_code ec); public: Request request; MultiResponse response; + EmptyResponse headResponse; std::string destinationFilePath; + boost::beast::http::verb method = boost::beast::http::verb::get; + bool skip = false; private: boost::asio::ip::tcp::resolver m_resolver; std::variant m_stream; boost::beast::flat_buffer m_buffer; std::unique_ptr m_chunkProcessing; + Request m_headRequest; Handler m_handler; + HeadHandler m_headHandler; }; template @@ -114,11 +126,12 @@ inline Session::Session(boost::asio::io_context &ioContext, const Handler &handl } template -inline Session::Session(boost::asio::io_context &ioContext, Handler &&handler) +inline Session::Session(boost::asio::io_context &ioContext, Handler &&handler, HeadHandler &&headHandler) : response(ResponseType{}) , m_resolver(ioContext) , m_stream(RawSocket{ ioContext }) , m_handler(std::move(handler)) + , m_headHandler(std::move(headHandler)) { } @@ -132,11 +145,12 @@ inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::co } template -inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler) +inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler, HeadHandler &&headHandler) : response(ResponseType{}) , m_resolver(ioContext) , m_stream(SslStream{ ioContext, sslContext }) , m_handler(std::move(handler)) + , m_headHandler(std::move(headHandler)) { } @@ -145,6 +159,11 @@ LIBREPOMGR_EXPORT std::variant> runSession std::string_view userName = std::string_view(), std::string_view password = std::string_view(), boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional bodyLimit = std::nullopt, Session::ChunkHandler &&chunkHandler = Session::ChunkHandler()); +LIBREPOMGR_EXPORT std::variant> runSessionFromUrl(boost::asio::io_context &ioContext, + boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, Session::HeadHandler &&headHandler, + std::string &&destinationPath = std::string(), std::string_view userName = std::string_view(), std::string_view password = std::string_view(), + boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional bodyLimit = std::nullopt, + Session::ChunkHandler &&chunkHandler = Session::ChunkHandler()); } // namespace WebClient } // namespace LibRepoMgr diff --git a/pacfind/main.cpp b/pacfind/main.cpp index 2d3dbd1..a6d4905 100644 --- a/pacfind/main.cpp +++ b/pacfind/main.cpp @@ -92,7 +92,7 @@ int main(int argc, const char *argv[]) } else { db.filesPath = db.filesPathFromRegularPath(); } - db.loadPackages(true); + db.loadPackagesFromConfiguredPaths(true, true); } catch (const std::runtime_error &e) { std::cerr << "Unable to load database \"" << db.name << "\": " << e.what() << '\n'; } diff --git a/srv/main.cpp b/srv/main.cpp index 8b179c7..1ac2d52 100644 --- a/srv/main.cpp +++ b/srv/main.cpp @@ -30,12 +30,14 @@ int main(int argc, const char *argv[]) OperationArgument runArg("run", 'r', "runs the server"); ConfigValueArgument configFileArg("config-file", 'c', "specifies the path of the config file", { "path" }); configFileArg.setEnvironmentVariable(PROJECT_VARNAME_UPPER "_CONFIG_FILE"); - runArg.setSubArguments({ &configFileArg }); + ConfigValueArgument forceLoadingDBsArg("force-loading-dbs", 'f', "forces loading DBs, even if DB files have not been modified since last parse"); + runArg.setSubArguments({ &configFileArg, &forceLoadingDBsArg }); runArg.setImplicit(true); - runArg.setCallback([&setup, &configFileArg](const ArgumentOccurrence &) { - if (configFileArg.firstValue()) { - setup.configFilePath = configFileArg.firstValue(); + runArg.setCallback([&setup, &configFileArg, &forceLoadingDBsArg](const ArgumentOccurrence &) { + if (const auto configFilePath = configFileArg.firstValue()) { + setup.configFilePath = configFilePath; } + setup.building.forceLoadingDbs = forceLoadingDBsArg.isPresent(); setup.run(); }); HelpArgument helpArg(parser);