Avoid loading databases when modification date is <= last update

* Do HTTP head request first when loading database from mirror to avoid
  downloading the full database all the time
* Use the last modification date of the local database file because with
  the persistent storage even local database reloads became a bit expensive

libpkg/data/config.h

@@ -111,7 +111,7 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar
 
     // load config and packages
     void loadPacmanConfig(const char *pacmanConfigPath);
-    void loadAllPackages(bool withFiles);
+    void loadAllPackages(bool withFiles, bool force);
 
     // storage and caching
     void initStorage(const char *path = "libpkg.db", std::uint32_t maxDbs = 0);

libpkg/data/database.h

@@ -125,7 +125,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
     void deducePathsFromLocalDirs();
     void resetConfiguration();
     void clearPackages();
-    void loadPackages(bool withFiles = false);
+    void loadPackagesFromConfiguredPaths(bool withFiles = false, bool force = false);
     void loadPackages(const std::string &databaseData, CppUtilities::DateTime lastModified);
     void loadPackages(FileMap &&databaseFiles, CppUtilities::DateTime lastModified);
     static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);

libpkg/parser/config.cpp

@@ -143,11 +143,11 @@ void Config::loadPacmanConfig(const char *pacmanConfigPath)
     }
 }
 
-void Config::loadAllPackages(bool withFiles)
+void Config::loadAllPackages(bool withFiles, bool force)
 {
     for (auto &db : databases) {
         try {
-            db.loadPackages(withFiles);
+            db.loadPackagesFromConfiguredPaths(withFiles, force);
         } catch (const runtime_error &e) {
             cerr << Phrases::ErrorMessage << "Unable to load database \"" << db.name << "\": " << e.what() << Phrases::EndFlush;
         }

libpkg/parser/database.cpp

@@ -21,13 +21,16 @@ bool Database::isFileRelevant(const char *filePath, const char *fileName, mode_t
     return !std::strcmp(fileName, "desc") || !std::strcmp(fileName, "depends") || !std::strcmp(fileName, "files");
 }
 
-void Database::loadPackages(bool withFiles)
+void Database::loadPackagesFromConfiguredPaths(bool withFiles, bool force)
 {
     const auto &dbPath = withFiles && !filesPath.empty() ? filesPath : path;
     if (dbPath.empty()) {
         throw runtime_error("local path not configured");
     }
-    loadPackages(extractFiles(dbPath, &isFileRelevant), lastModified(dbPath));
+    const auto lastFileUpdate = lastModified(dbPath);
+    if (force || lastFileUpdate > lastUpdate) {
+        loadPackages(extractFiles(dbPath, &isFileRelevant), lastFileUpdate);
+    }
 }
 
 void LibPkg::Database::loadPackages(const string &databaseData, DateTime lastModified)

libpkg/tests/parser.cpp

@@ -305,7 +305,7 @@ void ParserTests::testParsingDatabase()
         db->filesPath = testFilePath("core.files");
 
         // load packages
-        config.loadAllPackages(true);
+        config.loadAllPackages(true, false);
         CPPUNIT_ASSERT_EQUAL_MESSAGE("all 215 packages present"s, 215_st, db->packageCount());
         const auto autoreconf = db->findPackage("autoconf");
         CPPUNIT_ASSERT_MESSAGE("autoreconf exists", autoreconf != nullptr);

librepomgr/buildactions/buildactionmeta.cpp

@@ -78,7 +78,14 @@ BuildActionMetaInfo::BuildActionMetaInfo()
             .category = "Repo management",
             .name = "Reload databases",
             .type = "reload-database",
-            .flags = {},
+            .flags = {
+                BuildActionFlagMetaInfo{
+                    .id = static_cast<BuildActionFlagType>(ReloadDatabaseFlags::ForceReload),
+                    .name = "Force reload",
+                    .desc = "Load the database even if its last modification data isn't newer than the last time the database was loaded",
+                    .param = "force-reload",
+                },
+            },
             .settings = {},
             .directory = false,
             .sourceDb = false,

librepomgr/buildactions/buildactionmeta.h

@@ -55,6 +55,10 @@ enum class CheckForUpdatesFlags : BuildActionFlagType {
     None,
     ConsiderRegularPackage = (1 << 0), // be consistent with LibPkg::UpdateCheckOptions here
 };
+enum class ReloadDatabaseFlags : BuildActionFlagType {
+    None,
+    ForceReload = (1 << 0),
+};
 enum class ReloadLibraryDependenciesFlags : BuildActionFlagType {
     None,
     ForceReload = (1 << 0),
@@ -179,6 +183,7 @@ inline const BuildActionTypeMetaMapping &BuildActionMetaInfo::mappingForId(Build
 
 } // namespace LibRepoMgr
 
+CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ReloadDatabaseFlags)
 CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ReloadLibraryDependenciesFlags)
 CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::PrepareBuildFlags)
 CPP_UTILITIES_MARK_FLAG_ENUM_CLASS(LibRepoMgr, LibRepoMgr::ConductBuildFlags)

librepomgr/buildactions/reloaddatabase.cpp

@@ -28,6 +28,8 @@ ReloadDatabase::ReloadDatabase(ServiceSetup &setup, const std::shared_ptr<BuildA
 
 void ReloadDatabase::run()
 {
+    const auto flags = static_cast<ReloadDatabaseFlags>(m_buildAction->flags);
+    const auto force = flags & ReloadDatabaseFlags::ForceReload;
     const auto withFiles = m_setup.building.loadFilesDbs;
     vector<LibPkg::Database *> dbsToLoadFromMirror;
 
@@ -70,15 +72,26 @@ void ReloadDatabase::run()
             continue;
         }
         boost::asio::post(
-            m_setup.building.ioContext.get_executor(), [this, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
+            m_setup.building.ioContext.get_executor(), [this, force, session, dbName = db->name, dbArch = db->arch, dbPath = move(dbPath)]() mutable {
-                m_buildAction->appendOutput(
-                    Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
                 try {
                     auto dbFileLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(dbName, dbArch));
                     const auto lastModified = LibPkg::lastModified(dbPath);
+                    if (!force) {
+                        auto configReadLock2 = m_setup.config.lockToRead();
+                        auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
+                        if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) {
+                            configReadLock2.unlock();
+                            m_buildAction->appendOutput(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
+                                "\" from local file \"", dbPath, "\"; last modification time <= last update (", lastModified.toString(), '<', '=',
+                                lastUpdate.toString(), ')', '\n');
+                            return;
+                        }
+                    }
                     auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
                     auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile));
                     dbFileLock.lock().unlock();
+                    m_buildAction->appendOutput(
+                        Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
                     const auto configLock = m_setup.config.lockToWrite();
                     auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
                     if (!destinationDb) {
@@ -99,7 +112,7 @@ void ReloadDatabase::run()
     // query databases
     auto query = WebClient::prepareDatabaseQuery(m_buildAction->log(), dbsToLoadFromMirror, withFiles);
     std::get<std::shared_lock<std::shared_mutex>>(configReadLock).unlock();
-    WebClient::queryDatabases(m_buildAction->log(), m_setup, std::move(query.queryParamsForDbs), session);
+    WebClient::queryDatabases(m_buildAction->log(), m_setup, std::move(query.queryParamsForDbs), session, force);
     m_preparationFailures = std::move(query.failedDbs);
 
     // clear AUR cache

librepomgr/buildactions/repomanagement.cpp

@@ -653,7 +653,7 @@ void CleanRepository::run()
                 std::make_unique<LibPkg::Database>(argsToString("clean-repository-", dbFilePaths.front().stem()), dbFilePaths.front()));
             db->arch = dirInfo.canonicalPath.stem();
             db->initStorage(*m_setup.config.storage());
-            db->loadPackages();
+            db->loadPackagesFromConfiguredPaths();
             dirInfo.relevantDbs.emplace(db.get());
             // acquire lock for db directory
             dirInfo.lock.emplace<SharedLoggingLock>(m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(*db)));

librepomgr/serversetup.cpp

@@ -636,7 +636,7 @@ void ServiceSetup::run()
 #endif
         loadConfigFiles(true);
         config.discardDatabases();
-        config.loadAllPackages(building.loadFilesDbs);
+        config.loadAllPackages(building.loadFilesDbs, building.forceLoadingDbs);
 #ifndef CPP_UTILITIES_DEBUG_BUILD
     } catch (const std::exception &e) {
         cerr << Phrases::SubError << e.what() << endl;

librepomgr/serversetup.h

@@ -107,6 +107,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
         std::string testFilesDir;
         BuildPresets presets;
         bool loadFilesDbs = false;
+        bool forceLoadingDbs = false;
 
         // never changed after startup
         unsigned short threadCount = 4;
@@ -151,7 +152,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
     void restoreLibraryDependenciesFromJson(const std::string &json, ReflectiveRapidJSON::JsonDeserializationErrors *errors);
     std::size_t restoreState();
     std::size_t saveState();
-    void  initStorage();
+    void initStorage();
     void run();
     ServiceStatus computeStatus() const;
 };

librepomgr/tests/buildactions.cpp

@@ -550,7 +550,7 @@ void BuildActionsTests::testConductingBuild()
         readFile("repos/boost/os/x86_64/boost-libs-1.73.0-1-x86_64.pkg.tar.zst.sig"));
 
     // add packages needing a rebuild to trigger auto-staging
-    m_setup.config.loadAllPackages(false);
+    m_setup.config.loadAllPackages(false, true);
     auto *const boostDb = m_setup.config.findDatabase("boost"sv, "x86_64"sv);
     auto *const miscDb = m_setup.config.findDatabase("misc"sv, "x86_64"sv);
     CPPUNIT_ASSERT_MESSAGE("boost database present", boostDb);
@@ -670,7 +670,7 @@ void BuildActionsTests::testCleanup()
     auto *const miscDb = m_setup.config.findOrCreateDatabase("misc"sv, std::string_view());
     miscDb->path = repoDir64 / "misc.db";
     miscDb->localDbDir = miscDb->localPkgDir = repoDir64;
-    miscDb->loadPackages();
+    miscDb->loadPackagesFromConfiguredPaths();
 
     // create and run build action
     m_buildAction = std::make_shared<BuildAction>(0, &m_setup);

librepomgr/webapi/routes.cpp

@@ -345,8 +345,10 @@ void getPackages(const Params &params, ResponseHandler &&handler)
 
 void postLoadPackages(const Params &params, ResponseHandler &&handler)
 {
+    const auto withFiles = params.target.hasFlag("with-files");
+    const auto force = params.target.hasFlag("force");
     auto lock = params.setup.config.lockToWrite();
-    params.setup.config.loadAllPackages(params.target.hasFlag("with-files"));
+    params.setup.config.loadAllPackages(withFiles, force);
     lock.unlock();
     handler(makeText(params.request(), "packages loaded"));
 }

librepomgr/webclient/database.cpp

@@ -56,103 +56,138 @@ static int matchToInt(const std::sub_match<const char *> &match)
     return stringToNumber<int>(sv);
 }
 
-void queryDatabases(
+static CppUtilities::DateTime parseLastModified(LogContext &log, const auto &message, std::string_view dbName, std::string_view dbArch)
-    LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&dbQueries, std::shared_ptr<DatabaseQuerySession> &dbQuerySession)
+{
+    auto lastModified = DateTime();
+    const auto lastModifiedHeader = message.find(boost::beast::http::field::last_modified);
+    if (lastModifiedHeader != message.cend()) {
+        // parse "Last-Modified" header which should be something like "<day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT"
+        const auto lastModifiedStr = lastModifiedHeader->value();
+        static const auto lastModifiedPattern = std::regex("..., (\\d\\d) (...) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT");
+        static const auto months = unordered_map<string, int>{ { "Jan", 1 }, { "Feb", 2 }, { "Mar", 3 }, { "Apr", 4 }, { "May", 5 }, { "Jun", 6 },
+            { "Jul", 7 }, { "Aug", 8 }, { "Sep", 9 }, { "Oct", 10 }, { "Nov", 11 }, { "Dec", 12 } };
+        try {
+            auto match = std::cmatch();
+            if (!std::regex_search(lastModifiedStr.cbegin(), lastModifiedStr.cend(), match, lastModifiedPattern)) {
+                throw ConversionException("date/time denotation not in expected format");
+            }
+            const auto month = months.find(match[2]);
+            if (month == months.cend()) {
+                throw ConversionException("month is invalid");
+            }
+            lastModified = DateTime::fromDateAndTime(
+                matchToInt(match[3]), month->second, matchToInt(match[1]), matchToInt(match[4]), matchToInt(match[5]), matchToInt(match[6]));
+        } catch (const ConversionException &e) {
+            log(Phrases::WarningMessage, "Unable to parse \"Last-Modified\" header for database ", dbName, '@', dbArch, ": ", e.what(),
+                " (last modification time was \"", string_view(lastModifiedStr.data(), lastModifiedStr.size()), "\")\n");
+        }
+    }
+    return lastModified;
+}
+
+void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&dbQueries,
+    std::shared_ptr<DatabaseQuerySession> &dbQuerySession, bool force)
 {
     for (auto &query : dbQueries) {
         log(Phrases::InfoMessage, "Retrieving \"", query.databaseName, "\" from mirror: ", query.url, '\n');
-        auto session = runSessionFromUrl(
+        auto headHandler
-            setup.building.ioContext, setup.webServer.sslContext, query.url,
+            = force ? Session::HeadHandler() : [&log, &setup, dbName = query.databaseName, dbArch = query.databaseArch](Session &session3) mutable {
-            [&log, &setup, dbName = std::move(query.databaseName), dbArch = std::move(query.databaseArch), dbQuerySession](
+                  auto lastModified = parseLastModified(log, session3.headResponse.get(), dbName, dbArch);
-                Session &session2, const WebClient::HttpClientError &error) mutable {
+                  auto configReadLock = setup.config.lockToRead();
-                if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) {
+                  auto *const destinationDb = setup.config.findDatabase(dbName, dbArch);
-                    log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": ",
+                  if (!destinationDb) {
-                        error.what(), '\n');
+                      log(Phrases::InfoMessage, "Skip requesting database \"", dbName, '@', dbArch, "\" as it no longer exists\n");
-                    dbQuerySession->addResponse(std::move(dbName));
+                      session3.skip = true;
-                    return;
+                      return;
-                }
+                  }
+                  const auto lastUpdate = destinationDb->lastUpdate;
+                  configReadLock.unlock();
+                  if (lastModified > lastUpdate) {
+                      return;
+                  }
+                  log(Phrases::InfoMessage, "Skip requesting database \"", dbName, '@', dbArch,
+                      "\" from mirror; last modification time <= last update (", lastModified.toString(), " <= ", lastUpdate.toString(), ')', '\n');
+                  session3.skip = true;
+              };
+        auto handler = [&log, &setup, dbName = std::move(query.databaseName), dbArch = std::move(query.databaseArch), dbQuerySession, force](
+                           Session &session2, const WebClient::HttpClientError &error) mutable {
+            if (error.errorCode != boost::beast::errc::success && error.errorCode != boost::asio::ssl::error::stream_truncated) {
+                log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": ", error.what(),
+                    '\n');
+                dbQuerySession->addResponse(std::move(dbName));
+                return;
+            }
+            if (session2.skip) {
+                return;
+            }
 
-                const auto &response = get<FileResponse>(session2.response);
+            const auto &response = std::get<FileResponse>(session2.response);
-                const auto &message = response.get();
+            const auto &message = response.get();
-                if (message.result() != boost::beast::http::status::ok) {
+            if (message.result() != boost::beast::http::status::ok) {
-                    log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName,
+                log(Phrases::ErrorMessage, "Error retrieving database file \"", session2.destinationFilePath, "\" for ", dbName, ": mirror returned ",
-                        ": mirror returned ", message.result_int(), " response\n");
+                    message.result_int(), " response\n");
-                    dbQuerySession->addResponse(std::move(dbName));
+                dbQuerySession->addResponse(std::move(dbName));
-                    return;
+                return;
-                }
+            }
 
-                // find last modification time
+            // log/skip
-                auto lastModified = DateTime();
+            auto lastModified = parseLastModified(log, message, dbName, dbArch);
-                const auto lastModifiedHeader = message.find(boost::beast::http::field::last_modified);
+            if (lastModified.isNull()) {
-                if (lastModifiedHeader != message.cend()) {
+                log(Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from mirror response\n");
-                    // parse "Last-Modified" header which should be something like "<day-name>, <day> <month> <year> <hour>:<minute>:<second> GMT"
+                lastModified = DateTime::gmtNow();
-                    const auto lastModifiedStr = lastModifiedHeader->value();
+            } else if (!force) {
-                    static const auto lastModifiedPattern = regex("..., (\\d\\d) (...) (\\d\\d\\d\\d) (\\d\\d):(\\d\\d):(\\d\\d) GMT");
+                auto configReadLock = setup.config.lockToRead();
-                    static const auto months = unordered_map<string, int>{ { "Jan", 1 }, { "Feb", 2 }, { "Mar", 3 }, { "Apr", 4 }, { "May", 5 },
+                if (auto *const destinationDb = setup.config.findDatabase(dbName, dbArch)) {
-                        { "Jun", 6 }, { "Jul", 7 }, { "Aug", 8 }, { "Sep", 9 }, { "Oct", 10 }, { "Nov", 11 }, { "Dec", 12 } };
+                    if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) {
-                    try {
+                        configReadLock.unlock();
-                        cmatch match;
+                        log(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
-                        if (!regex_search(lastModifiedStr.cbegin(), lastModifiedStr.cend(), match, lastModifiedPattern)) {
+                            "\" from mirror response; last modification time <= last update (", lastModified.toString(),
-                            throw ConversionException("date/time denotation not in expected format");
+                            " <= ", lastUpdate.toString(), ')', '\n');
-                        }
-                        const auto month = months.find(match[2]);
-                        if (month == months.cend()) {
-                            throw ConversionException("month is invalid");
-                        }
-                        lastModified = DateTime::fromDateAndTime(matchToInt(match[3]), month->second, matchToInt(match[1]), matchToInt(match[4]),
-                            matchToInt(match[5]), matchToInt(match[6]));
-                    } catch (const ConversionException &e) {
-                        log(Phrases::WarningMessage, "Unable to parse \"Last-Modified\" header for database ", dbName, ": ", e.what(),
-                            " (last modification time was \"", string_view(lastModifiedStr.data(), lastModifiedStr.size()), "\")\n");
-                    }
-                }
-
-                // log
-                if (lastModified.isNull()) {
-                    log(Phrases::InfoMessage, "Loading database \"", dbName, "\" from mirror response\n");
-                    lastModified = DateTime::gmtNow();
-                } else {
-                    log(Phrases::InfoMessage, "Loading database \"", dbName,
-                        "\" from mirror response; last modification time: ", lastModified.toString(), '\n');
-                }
-
-                try {
-                    // load packages
-                    auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant);
-                    auto packages = Package::fromDatabaseFile(move(files));
-
-                    // insert packages
-                    auto lock = setup.config.lockToWrite();
-                    auto db = setup.config.findDatabase(dbName, dbArch);
-                    if (!db) {
-                        log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
                         return;
                     }
-                    db->replacePackages(packages, lastModified);
-                } catch (const std::runtime_error &e) {
-                    log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
-                    dbQuerySession->addResponse(std::move(dbName));
                 }
-            },
+            } else {
-            move(query.destinationFilePath));
+                log(Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch,
+                    "\" from mirror response; last modification time: ", lastModified.toString(), '\n');
+            }
+
+            try {
+                // load packages
+                auto files = extractFiles(session2.destinationFilePath, &Database::isFileRelevant);
+                auto packages = Package::fromDatabaseFile(std::move(files));
+
+                // insert packages
+                auto lock = setup.config.lockToWrite();
+                auto db = setup.config.findDatabase(dbName, dbArch);
+                if (!db) {
+                    log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
+                    return;
+                }
+                db->replacePackages(packages, lastModified);
+            } catch (const std::runtime_error &e) {
+                log(Phrases::ErrorMessage, "Unable to parse retrieved database file for \"", dbName, '@', dbArch, "\": ", e.what(), '\n');
+                dbQuerySession->addResponse(std::move(dbName));
+            }
+        };
+        auto session = runSessionFromUrl(setup.building.ioContext, setup.webServer.sslContext, query.url, std::move(handler), std::move(headHandler),
+            std::move(query.destinationFilePath));
     }
 }
 
 std::shared_ptr<DatabaseQuerySession> queryDatabases(
-    LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, DatabaseQuerySession::HandlerType &&handler)
+    LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, bool force, DatabaseQuerySession::HandlerType &&handler)
 {
     auto dbQuerySession = DatabaseQuerySession::create(setup.building.ioContext, move(handler));
-    queryDatabases(log, setup, move(urls), dbQuerySession);
+    queryDatabases(log, setup, move(urls), dbQuerySession, force);
     return dbQuerySession;
 }
 
 std::shared_ptr<DatabaseQuerySession> queryDatabases(LogContext &log, ServiceSetup &setup, std::shared_lock<shared_mutex> *configReadLock,
-    const std::vector<Database *> &dbs, DatabaseQuerySession::HandlerType &&handler)
+    const std::vector<Database *> &dbs, bool force, DatabaseQuerySession::HandlerType &&handler)
 {
-    shared_lock<shared_mutex> ownConfigReadLock;
     auto query = prepareDatabaseQuery(log, dbs, setup.building.loadFilesDbs);
     configReadLock->unlock();
-    return queryDatabases(log, setup, std::move(query.queryParamsForDbs), move(handler));
+    return queryDatabases(log, setup, std::move(query.queryParamsForDbs), force, move(handler));
 }
 
 PackageCachingSession::PackageCachingSession(PackageCachingDataForSession &data, boost::asio::io_context &ioContext,

librepomgr/webclient/database.h

@@ -30,11 +30,11 @@ struct DatabaseQuery {
 
 [[nodiscard]] DatabaseQuery prepareDatabaseQuery(LogContext &log, const std::vector<LibPkg::Database *> &dbs, bool withFiles);
 std::shared_ptr<DatabaseQuerySession> queryDatabases(
-    LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, DatabaseQuerySession::HandlerType &&handler);
+    LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, bool force, DatabaseQuerySession::HandlerType &&handler);
 std::shared_ptr<DatabaseQuerySession> queryDatabases(LogContext &log, ServiceSetup &setup, std::shared_lock<std::shared_mutex> *configReadLock,
-    const std::vector<LibPkg::Database *> &dbs, DatabaseQuerySession::HandlerType &&handler);
+    const std::vector<LibPkg::Database *> &dbs, bool force, DatabaseQuerySession::HandlerType &&handler);
-void queryDatabases(
+void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls,
-    LogContext &log, ServiceSetup &setup, std::vector<DatabaseQueryParams> &&urls, std::shared_ptr<DatabaseQuerySession> &dbQuerySession);
+    std::shared_ptr<DatabaseQuerySession> &dbQuerySession, bool force = false);
 
 struct PackageCachingDataForPackage {
     std::string_view url;

librepomgr/webclient/session.cpp

@@ -57,10 +57,10 @@ void Session::run(
 
     // set up an HTTP request message
     request.version(version);
-    request.method(verb);
     request.target(target);
     request.set(http::field::host, host);
     request.set(http::field::user_agent, APP_NAME " " APP_VERSION);
+    method = verb;
 
     // setup a file response
     if (!destinationFilePath.empty()) {
@@ -131,6 +131,17 @@ void Session::handshakeDone(boost::beast::error_code ec)
 void Session::sendRequest()
 {
     // send the HTTP request to the remote host
+    if (m_headHandler) {
+        request.method(boost::beast::http::verb::head);
+        std::visit(
+            [this](auto &&stream) {
+                boost::beast::http::async_write(
+                    stream, request, std::bind(&Session::headRequested, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
+            },
+            m_stream);
+        return;
+    }
+    request.method(method);
     std::visit(
         [this](auto &&stream) {
             boost::beast::http::async_write(
@@ -139,6 +150,24 @@ void Session::sendRequest()
         m_stream);
 }
 
+void Session::headRequested(boost::beast::error_code ec, std::size_t bytesTransferred)
+{
+    boost::ignore_unused(bytesTransferred);
+    if (ec) {
+        m_handler(*this, HttpClientError("sending HEAD request", ec));
+        return;
+    }
+
+    // receive the HTTP response
+    headResponse.skip(true);
+    std::visit(
+        [this](auto &stream) {
+            http::async_read(
+                stream, m_buffer, headResponse, std::bind(&Session::headReceived, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
+        },
+        m_stream);
+}
+
 void Session::requested(boost::beast::error_code ec, std::size_t bytesTransferred)
 {
     boost::ignore_unused(bytesTransferred);
@@ -222,6 +251,22 @@ bool Session::continueReadingChunks()
     return true;
 }
 
+void Session::headReceived(boost::beast::error_code ec, std::size_t bytesTransferred)
+{
+    boost::ignore_unused(bytesTransferred);
+    if (ec) {
+        m_handler(*this, HttpClientError("receiving HEAD response", ec));
+        return;
+    }
+    m_headHandler(*this);
+    m_headHandler = HeadHandler();
+    if (skip) {
+        sendRequest();
+    } else {
+        closeGracefully();
+    }
+}
+
 void Session::received(boost::beast::error_code ec, std::size_t bytesTransferred)
 {
     boost::ignore_unused(bytesTransferred);
@@ -229,8 +274,12 @@ void Session::received(boost::beast::error_code ec, std::size_t bytesTransferred
         m_handler(*this, HttpClientError("receiving response", ec));
         return;
     }
+    closeGracefully();
+}
 
-    // close the stream gracefully
+void Session::closeGracefully()
+{
+    auto ec = boost::beast::error_code();
     if (auto *const sslStream = std::get_if<SslStream>(&m_stream)) {
         // perform the SSL handshake
         sslStream->async_shutdown(std::bind(&Session::closed, shared_from_this(), std::placeholders::_1));
@@ -249,6 +298,14 @@ void Session::closed(boost::beast::error_code ec)
 std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
     std::string_view url, Session::Handler &&handler, std::string &&destinationPath, std::string_view userName, std::string_view password,
     boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
+{
+    return runSessionFromUrl(ioContext, sslContext, url, std::move(handler), Session::HeadHandler(), std::move(destinationPath), userName, password,
+        verb, bodyLimit, std::move(chunkHandler));
+}
+
+std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
+    std::string_view url, Session::Handler &&handler, Session::HeadHandler &&headHandler, std::string &&destinationPath, std::string_view userName,
+    std::string_view password, boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
 {
     std::string host, port, target;
     auto ssl = false;
@@ -284,8 +341,8 @@ std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asi
         port = ssl ? "443" : "80";
     }
 
-    auto session
+    auto session = ssl ? std::make_shared<Session>(ioContext, sslContext, std::move(handler), std::move(headHandler))
-        = ssl ? std::make_shared<Session>(ioContext, sslContext, std::move(handler)) : std::make_shared<Session>(ioContext, std::move(handler));
+                       : std::make_shared<Session>(ioContext, std::move(handler), std::move(headHandler));
     if (!userName.empty()) {
         const auto authInfo = userName % ":" + password;
         session->request.set(boost::beast::http::field::authorization,

librepomgr/webclient/session.h

@@ -45,6 +45,7 @@ inline LibRepoMgr::WebClient::HttpClientError::operator bool() const
 using Response = WebAPI::Response;
 using FileResponse = boost::beast::http::response_parser<boost::beast::http::file_body>;
 using StringResponse = boost::beast::http::response_parser<boost::beast::http::string_body>;
+using EmptyResponse = boost::beast::http::response_parser<boost::beast::http::empty_body>;
 using MultiResponse = std::variant<Response, FileResponse, StringResponse>;
 using Request = boost::beast::http::request<boost::beast::http::empty_body>;
 struct ChunkProcessing;
@@ -52,14 +53,17 @@ struct ChunkProcessing;
 class LIBREPOMGR_EXPORT Session : public std::enable_shared_from_this<Session> {
 public:
     using Handler = std::function<void(Session &, const HttpClientError &error)>;
+    using HeadHandler = std::function<void(Session &)>;
     using ChunkHandler = std::function<void(const boost::beast::http::chunk_extensions &chunkExtensions, std::string_view chunkData)>;
 
     template <typename ResponseType = Response> explicit Session(boost::asio::io_context &ioContext, const Handler &handler = Handler());
-    template <typename ResponseType = Response> explicit Session(boost::asio::io_context &ioContext, Handler &&handler = Handler());
+    template <typename ResponseType = Response>
+    explicit Session(boost::asio::io_context &ioContext, Handler &&handler = Handler(), HeadHandler &&headHandler = HeadHandler());
     template <typename ResponseType = Response>
     explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, const Handler &handler = Handler());
     template <typename ResponseType = Response>
-    explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler);
+    explicit Session(
+        boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler, HeadHandler &&headHandler = HeadHandler());
 
     void setChunkHandler(ChunkHandler &&handler);
     void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target,
@@ -83,25 +87,33 @@ private:
     void connected(boost::beast::error_code ec);
     void handshakeDone(boost::beast::error_code ec);
     void sendRequest();
+    void headRequested(boost::beast::error_code ec, std::size_t bytesTransferred);
     void requested(boost::beast::error_code ec, std::size_t bytesTransferred);
     void onChunkHeader(std::uint64_t chunkSize, boost::beast::string_view extensions, boost::beast::error_code &ec);
     std::size_t onChunkBody(std::uint64_t bytesLeftInThisChunk, boost::beast::string_view chunkBodyData, boost::beast::error_code &ec);
     void chunkReceived(boost::beast::error_code ec, std::size_t bytesTransferred);
     bool continueReadingChunks();
+    void headReceived(boost::beast::error_code ec, std::size_t bytesTransferred);
     void received(boost::beast::error_code ec, std::size_t bytesTransferred);
+    void closeGracefully();
     void closed(boost::beast::error_code ec);
 
 public:
     Request request;
     MultiResponse response;
+    EmptyResponse headResponse;
     std::string destinationFilePath;
+    boost::beast::http::verb method = boost::beast::http::verb::get;
+    bool skip = false;
 
 private:
     boost::asio::ip::tcp::resolver m_resolver;
     std::variant<RawSocket, SslStream> m_stream;
     boost::beast::flat_buffer m_buffer;
     std::unique_ptr<ChunkProcessing> m_chunkProcessing;
+    Request m_headRequest;
     Handler m_handler;
+    HeadHandler m_headHandler;
 };
 
 template <typename ResponseType>
@@ -114,11 +126,12 @@ inline Session::Session(boost::asio::io_context &ioContext, const Handler &handl
 }
 
 template <typename ResponseType>
-inline Session::Session(boost::asio::io_context &ioContext, Handler &&handler)
+inline Session::Session(boost::asio::io_context &ioContext, Handler &&handler, HeadHandler &&headHandler)
     : response(ResponseType{})
     , m_resolver(ioContext)
     , m_stream(RawSocket{ ioContext })
     , m_handler(std::move(handler))
+    , m_headHandler(std::move(headHandler))
 {
 }
 
@@ -132,11 +145,12 @@ inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::co
 }
 
 template <typename ResponseType>
-inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler)
+inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler, HeadHandler &&headHandler)
     : response(ResponseType{})
     , m_resolver(ioContext)
     , m_stream(SslStream{ ioContext, sslContext })
     , m_handler(std::move(handler))
+    , m_headHandler(std::move(headHandler))
 {
 }
 
@@ -145,6 +159,11 @@ LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSession
     std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
     boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
     Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
+LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext,
+    boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, Session::HeadHandler &&headHandler,
+    std::string &&destinationPath = std::string(), std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
+    boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
+    Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
 
 } // namespace WebClient
 } // namespace LibRepoMgr

pacfind/main.cpp

@@ -92,7 +92,7 @@ int main(int argc, const char *argv[])
             } else {
                 db.filesPath = db.filesPathFromRegularPath();
             }
-            db.loadPackages(true);
+            db.loadPackagesFromConfiguredPaths(true, true);
         } catch (const std::runtime_error &e) {
             std::cerr << "Unable to load database \"" << db.name << "\": " << e.what() << '\n';
         }

srv/main.cpp

@@ -30,12 +30,14 @@ int main(int argc, const char *argv[])
     OperationArgument runArg("run", 'r', "runs the server");
     ConfigValueArgument configFileArg("config-file", 'c', "specifies the path of the config file", { "path" });
     configFileArg.setEnvironmentVariable(PROJECT_VARNAME_UPPER "_CONFIG_FILE");
-    runArg.setSubArguments({ &configFileArg });
+    ConfigValueArgument forceLoadingDBsArg("force-loading-dbs", 'f', "forces loading DBs, even if DB files have not been modified since last parse");
+    runArg.setSubArguments({ &configFileArg, &forceLoadingDBsArg });
     runArg.setImplicit(true);
-    runArg.setCallback([&setup, &configFileArg](const ArgumentOccurrence &) {
+    runArg.setCallback([&setup, &configFileArg, &forceLoadingDBsArg](const ArgumentOccurrence &) {
-        if (configFileArg.firstValue()) {
+        if (const auto configFilePath = configFileArg.firstValue()) {
-            setup.configFilePath = configFileArg.firstValue();
+            setup.configFilePath = configFilePath;
         }
+        setup.building.forceLoadingDbs = forceLoadingDBsArg.isPresent();
         setup.run();
     });
     HelpArgument helpArg(parser);