Increase the max. download size for packages and allow configuring it

cli/main.cpp

@@ -608,7 +608,7 @@ int main(int argc, const char *argv[])
     LibRepoMgr::WebClient::runSessionFromUrl(ioContext, sslContext, url,
         std::bind(&handleResponse, std::ref(url), std::placeholders::_1, std::placeholders::_2, rawArg.isPresent() ? printRawData : printer,
             std::ref(returnCode)),
-        std::string(), config.userName, config.password, verb, chunkHandler);
+        std::string(), config.userName, config.password, verb, std::nullopt, chunkHandler);
     ioContext.run();
     return returnCode;
 }

librepomgr/buildactions/buildactionprivate.h

@@ -434,6 +434,7 @@ private:
     std::vector<DatabaseToConsider> m_relevantPackagesByDatabase;
     std::atomic_size_t m_remainingPackages;
     WebClient::PackageCachingDataForSession m_cachingData;
+    std::uint64_t m_packageDownloadSizeLimit;
 };
 
 struct LIBREPOMGR_EXPORT CheckForProblems : public InternalBuildAction {

librepomgr/buildactions/reloadlibrarydependencies.cpp

@@ -55,6 +55,7 @@ void ReloadLibraryDependencies::run()
     // use cache directory from global configuration
     auto buildLock = m_setup.building.lockToRead();
     const auto cacheDir = m_setup.building.packageCacheDir + '/';
+    m_packageDownloadSizeLimit = m_setup.building.packageDownloadSizeLimit;
     buildLock.unlock();
 
     // find relevant databases and packages
@@ -237,7 +238,7 @@ void LibRepoMgr::ReloadLibraryDependencies::downloadPackagesFromMirror()
     m_buildAction->appendOutput(Phrases::SuccessMessage, "Downloading ", packagesWhichNeedCaching, " binary packages from mirror ...\n");
     WebClient::cachePackages(m_buildAction->log(),
         std::make_shared<WebClient::PackageCachingSession>(m_cachingData, m_setup.building.ioContext, m_setup.webServer.sslContext,
-            std::bind(&ReloadLibraryDependencies::loadPackageInfoFromContents, this)));
+            std::bind(&ReloadLibraryDependencies::loadPackageInfoFromContents, this)), m_packageDownloadSizeLimit);
 }
 
 void ReloadLibraryDependencies::loadPackageInfoFromContents()

librepomgr/helper.h

@@ -39,7 +39,8 @@ inline std::optional<std::string_view> getLastValueSv(const std::multimap<std::s
     return std::nullopt;
 }
 
-template <typename TargetType> void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result);
+template <typename TargetType, Traits::DisableIf<std::is_integral<TargetType>> * = nullptr>
+void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result);
 
 template <>
 inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, boost::asio::ip::address &result)
@@ -59,7 +60,8 @@ inline void convertValue(const std::multimap<std::string, std::string> &multimap
     }
 }
 
-template <> inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, unsigned short &result)
+template <typename TargetType, Traits::EnableIf<std::is_integral<TargetType>> * = nullptr>
+inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result)
 {
     using namespace std;
     using namespace CppUtilities;
@@ -67,7 +69,7 @@ template <> inline void convertValue(const std::multimap<std::string, std::strin
 
     if (const char *const value = getLastValue(multimap, key)) {
         try {
-            result = stringToNumber<unsigned short>(value);
+            result = stringToNumber<TargetType>(value);
         } catch (const ConversionException &) {
             cerr << Phrases::ErrorMessage << "Specified number \"" << value << "\" for key \"" << key << "\" is invalid." << Phrases::End;
             return;

librepomgr/serversetup.cpp

@@ -118,6 +118,7 @@ void ServiceSetup::BuildSetup::applyConfig(const std::multimap<std::string, std:
     convertValue(multimap, "makechrootpkg_flags", makechrootpkgFlags);
     convertValue(multimap, "makepkg_flags", makepkgFlags);
     convertValue(multimap, "package_cache_dir", packageCacheDir);
+    convertValue(multimap, "package_download_size_limit", packageDownloadSizeLimit);
     convertValue(multimap, "test_files_dir", testFilesDir);
     convertValue(multimap, "load_files_dbs", loadFilesDbs);
 }

librepomgr/serversetup.h

@@ -100,6 +100,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
         std::vector<std::string> makechrootpkgFlags;
         std::vector<std::string> makepkgFlags;
         std::string packageCacheDir;
+        std::uint64_t packageDownloadSizeLimit = 500 * 1024 * 1024;
         std::string testFilesDir;
         BuildPresets presets;
         bool loadFilesDbs = false;

librepomgr/webclient/database.cpp

@@ -190,7 +190,7 @@ PackageCachingDataForPackage *PackageCachingSession::getCurrentDataAndSelectNext
     return data;
 }
 
-void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::size_t maxParallelDownloads)
+void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::optional<std::uint64_t> bodyLimit, std::size_t maxParallelDownloads)
 {
     for (std::size_t startedDownloads = 0; startedDownloads < maxParallelDownloads; ++startedDownloads) {
         auto *const cachingData = packageCachingSession->getCurrentDataAndSelectNext();
@@ -217,7 +217,7 @@ void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&pac
                 }
                 cachePackages(log, std::move(packageCachingSession), 1);
             },
-            std::string(cachingData->destinationFilePath));
+            std::string(cachingData->destinationFilePath), std::string_view(), std::string_view(), boost::beast::http::verb::get, bodyLimit);
     }
 }
 

librepomgr/webclient/database.h

@@ -6,6 +6,7 @@
 
 #include "./session.h"
 
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -44,10 +45,10 @@ using PackageCachingDataForDatabase = std::unordered_map<std::string_view, Packa
 using PackageCachingDataForSession = std::unordered_map<std::string_view, PackageCachingDataForDatabase>;
 
 struct PackageCachingSession;
-void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::size_t maxParallelDownloads = 8);
+void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::optional<std::uint64_t> bodyLimit = std::nullopt, std::size_t maxParallelDownloads = 8);
 
 struct PackageCachingSession : public MultiSession<void> {
-    friend void cachePackages(LogContext &, std::shared_ptr<PackageCachingSession> &&, std::size_t);
+    friend void cachePackages(LogContext &, std::shared_ptr<PackageCachingSession> &&, std::optional<std::uint64_t>, std::size_t);
     using SharedPointerType = std::shared_ptr<PackageCachingSession>;
 
     explicit PackageCachingSession(

librepomgr/webclient/session.cpp

@@ -41,7 +41,7 @@ void Session::setChunkHandler(ChunkHandler &&handler)
     m_chunkProcessing->handler = std::move(handler);
 }
 
-void Session::run(const char *host, const char *port, http::verb verb, const char *target, unsigned int version)
+void Session::run(const char *host, const char *port, http::verb verb, const char *target, std::optional<std::uint64_t> bodyLimit, unsigned int version)
 {
     // set SNI Hostname (many hosts need this to handshake successfully)
     auto *const sslStream = std::get_if<SslStream>(&m_stream);
@@ -65,7 +65,7 @@ void Session::run(const char *host, const char *port, http::verb verb, const cha
     if (!destinationFilePath.empty()) {
         auto &fileResponse = response.emplace<FileResponse>();
         boost::beast::error_code errorCode;
-        fileResponse.body_limit(100 * 1024 * 1024);
+        fileResponse.body_limit(bodyLimit.value_or(500 * 1024 * 1024));
         fileResponse.get().body().open(destinationFilePath.data(), file_mode::write, errorCode);
         if (errorCode != boost::beast::errc::success) {
             m_handler(*this, HttpClientError("opening output file", errorCode));
@@ -247,7 +247,7 @@ void Session::closed(boost::beast::error_code ec)
 
 std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
     std::string_view url, Session::Handler &&handler, std::string &&destinationPath, std::string_view userName, std::string_view password,
-    boost::beast::http::verb verb, Session::ChunkHandler &&chunkHandler)
+    boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
 {
     std::string host, port, target;
     auto ssl = false;
@@ -294,7 +294,7 @@ std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asi
     if (chunkHandler) {
         session->setChunkHandler(std::move(chunkHandler));
     }
-    session->run(host.data(), port.data(), verb, target.data());
+    session->run(host.data(), port.data(), verb, target.data(), bodyLimit);
     return std::variant<std::string, std::shared_ptr<Session>>(std::move(session));
 }
 

librepomgr/webclient/session.h

@@ -11,6 +11,7 @@
 #include <boost/asio/ssl.hpp>
 
 #include <functional>
+#include <optional>
 #include <stdexcept>
 #include <variant>
 
@@ -61,7 +62,7 @@ public:
     explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler);
 
     void setChunkHandler(ChunkHandler &&handler);
-    void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, unsigned int version = 11);
+    void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, std::optional<std::uint64_t> bodyLimit = std::nullopt, unsigned int version = 11);
 
 private:
     using RawSocket = boost::asio::ip::tcp::socket;
@@ -141,7 +142,8 @@ inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::co
 LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext,
     boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, std::string &&destinationPath = std::string(),
     std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
-    boost::beast::http::verb verb = boost::beast::http::verb::get, Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
+    boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
+    Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
 
 } // namespace WebClient
 } // namespace LibRepoMgr