diff --git a/cli/main.cpp b/cli/main.cpp index c655911..b7318b9 100644 --- a/cli/main.cpp +++ b/cli/main.cpp @@ -608,7 +608,7 @@ int main(int argc, const char *argv[]) LibRepoMgr::WebClient::runSessionFromUrl(ioContext, sslContext, url, std::bind(&handleResponse, std::ref(url), std::placeholders::_1, std::placeholders::_2, rawArg.isPresent() ? printRawData : printer, std::ref(returnCode)), - std::string(), config.userName, config.password, verb, chunkHandler); + std::string(), config.userName, config.password, verb, std::nullopt, chunkHandler); ioContext.run(); return returnCode; } diff --git a/librepomgr/buildactions/buildactionprivate.h b/librepomgr/buildactions/buildactionprivate.h index 479a714..2a63eb3 100644 --- a/librepomgr/buildactions/buildactionprivate.h +++ b/librepomgr/buildactions/buildactionprivate.h @@ -434,6 +434,7 @@ private: std::vector m_relevantPackagesByDatabase; std::atomic_size_t m_remainingPackages; WebClient::PackageCachingDataForSession m_cachingData; + std::uint64_t m_packageDownloadSizeLimit; }; struct LIBREPOMGR_EXPORT CheckForProblems : public InternalBuildAction { diff --git a/librepomgr/buildactions/reloadlibrarydependencies.cpp b/librepomgr/buildactions/reloadlibrarydependencies.cpp index aa83efc..26528d5 100644 --- a/librepomgr/buildactions/reloadlibrarydependencies.cpp +++ b/librepomgr/buildactions/reloadlibrarydependencies.cpp @@ -55,6 +55,7 @@ void ReloadLibraryDependencies::run() // use cache directory from global configuration auto buildLock = m_setup.building.lockToRead(); const auto cacheDir = m_setup.building.packageCacheDir + '/'; + m_packageDownloadSizeLimit = m_setup.building.packageDownloadSizeLimit; buildLock.unlock(); // find relevant databases and packages @@ -237,7 +238,7 @@ void LibRepoMgr::ReloadLibraryDependencies::downloadPackagesFromMirror() m_buildAction->appendOutput(Phrases::SuccessMessage, "Downloading ", packagesWhichNeedCaching, " binary packages from mirror ...\n"); WebClient::cachePackages(m_buildAction->log(), std::make_shared(m_cachingData, m_setup.building.ioContext, m_setup.webServer.sslContext, - std::bind(&ReloadLibraryDependencies::loadPackageInfoFromContents, this))); + std::bind(&ReloadLibraryDependencies::loadPackageInfoFromContents, this)), m_packageDownloadSizeLimit); } void ReloadLibraryDependencies::loadPackageInfoFromContents() diff --git a/librepomgr/helper.h b/librepomgr/helper.h index 0cf8209..fa58c9b 100644 --- a/librepomgr/helper.h +++ b/librepomgr/helper.h @@ -39,7 +39,8 @@ inline std::optional getLastValueSv(const std::multimap void convertValue(const std::multimap &multimap, const std::string &key, TargetType &result); +template > * = nullptr> +void convertValue(const std::multimap &multimap, const std::string &key, TargetType &result); template <> inline void convertValue(const std::multimap &multimap, const std::string &key, boost::asio::ip::address &result) @@ -59,7 +60,8 @@ inline void convertValue(const std::multimap &multimap } } -template <> inline void convertValue(const std::multimap &multimap, const std::string &key, unsigned short &result) +template > * = nullptr> +inline void convertValue(const std::multimap &multimap, const std::string &key, TargetType &result) { using namespace std; using namespace CppUtilities; @@ -67,7 +69,7 @@ template <> inline void convertValue(const std::multimap(value); + result = stringToNumber(value); } catch (const ConversionException &) { cerr << Phrases::ErrorMessage << "Specified number \"" << value << "\" for key \"" << key << "\" is invalid." << Phrases::End; return; diff --git a/librepomgr/serversetup.cpp b/librepomgr/serversetup.cpp index 4ce5f9f..487d7bf 100644 --- a/librepomgr/serversetup.cpp +++ b/librepomgr/serversetup.cpp @@ -118,6 +118,7 @@ void ServiceSetup::BuildSetup::applyConfig(const std::multimap makechrootpkgFlags; std::vector makepkgFlags; std::string packageCacheDir; + std::uint64_t packageDownloadSizeLimit = 500 * 1024 * 1024; std::string testFilesDir; BuildPresets presets; bool loadFilesDbs = false; diff --git a/librepomgr/webclient/database.cpp b/librepomgr/webclient/database.cpp index 82b6307..beb460b 100644 --- a/librepomgr/webclient/database.cpp +++ b/librepomgr/webclient/database.cpp @@ -190,7 +190,7 @@ PackageCachingDataForPackage *PackageCachingSession::getCurrentDataAndSelectNext return data; } -void cachePackages(LogContext &log, std::shared_ptr &&packageCachingSession, std::size_t maxParallelDownloads) +void cachePackages(LogContext &log, std::shared_ptr &&packageCachingSession, std::optional bodyLimit, std::size_t maxParallelDownloads) { for (std::size_t startedDownloads = 0; startedDownloads < maxParallelDownloads; ++startedDownloads) { auto *const cachingData = packageCachingSession->getCurrentDataAndSelectNext(); @@ -217,7 +217,7 @@ void cachePackages(LogContext &log, std::shared_ptr &&pac } cachePackages(log, std::move(packageCachingSession), 1); }, - std::string(cachingData->destinationFilePath)); + std::string(cachingData->destinationFilePath), std::string_view(), std::string_view(), boost::beast::http::verb::get, bodyLimit); } } diff --git a/librepomgr/webclient/database.h b/librepomgr/webclient/database.h index 5c8db4f..7b5e996 100644 --- a/librepomgr/webclient/database.h +++ b/librepomgr/webclient/database.h @@ -6,6 +6,7 @@ #include "./session.h" +#include #include #include @@ -44,10 +45,10 @@ using PackageCachingDataForDatabase = std::unordered_map; struct PackageCachingSession; -void cachePackages(LogContext &log, std::shared_ptr &&packageCachingSession, std::size_t maxParallelDownloads = 8); +void cachePackages(LogContext &log, std::shared_ptr &&packageCachingSession, std::optional bodyLimit = std::nullopt, std::size_t maxParallelDownloads = 8); struct PackageCachingSession : public MultiSession { - friend void cachePackages(LogContext &, std::shared_ptr &&, std::size_t); + friend void cachePackages(LogContext &, std::shared_ptr &&, std::optional, std::size_t); using SharedPointerType = std::shared_ptr; explicit PackageCachingSession( diff --git a/librepomgr/webclient/session.cpp b/librepomgr/webclient/session.cpp index f8fa24c..9be10df 100644 --- a/librepomgr/webclient/session.cpp +++ b/librepomgr/webclient/session.cpp @@ -41,7 +41,7 @@ void Session::setChunkHandler(ChunkHandler &&handler) m_chunkProcessing->handler = std::move(handler); } -void Session::run(const char *host, const char *port, http::verb verb, const char *target, unsigned int version) +void Session::run(const char *host, const char *port, http::verb verb, const char *target, std::optional bodyLimit, unsigned int version) { // set SNI Hostname (many hosts need this to handshake successfully) auto *const sslStream = std::get_if(&m_stream); @@ -65,7 +65,7 @@ void Session::run(const char *host, const char *port, http::verb verb, const cha if (!destinationFilePath.empty()) { auto &fileResponse = response.emplace(); boost::beast::error_code errorCode; - fileResponse.body_limit(100 * 1024 * 1024); + fileResponse.body_limit(bodyLimit.value_or(500 * 1024 * 1024)); fileResponse.get().body().open(destinationFilePath.data(), file_mode::write, errorCode); if (errorCode != boost::beast::errc::success) { m_handler(*this, HttpClientError("opening output file", errorCode)); @@ -247,7 +247,7 @@ void Session::closed(boost::beast::error_code ec) std::variant> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, std::string &&destinationPath, std::string_view userName, std::string_view password, - boost::beast::http::verb verb, Session::ChunkHandler &&chunkHandler) + boost::beast::http::verb verb, std::optional bodyLimit, Session::ChunkHandler &&chunkHandler) { std::string host, port, target; auto ssl = false; @@ -294,7 +294,7 @@ std::variant> runSessionFromUrl(boost::asi if (chunkHandler) { session->setChunkHandler(std::move(chunkHandler)); } - session->run(host.data(), port.data(), verb, target.data()); + session->run(host.data(), port.data(), verb, target.data(), bodyLimit); return std::variant>(std::move(session)); } diff --git a/librepomgr/webclient/session.h b/librepomgr/webclient/session.h index 1c19ad5..cc7a8f6 100644 --- a/librepomgr/webclient/session.h +++ b/librepomgr/webclient/session.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -61,7 +62,7 @@ public: explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler); void setChunkHandler(ChunkHandler &&handler); - void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, unsigned int version = 11); + void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, std::optional bodyLimit = std::nullopt, unsigned int version = 11); private: using RawSocket = boost::asio::ip::tcp::socket; @@ -141,7 +142,8 @@ inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::co LIBREPOMGR_EXPORT std::variant> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, std::string &&destinationPath = std::string(), std::string_view userName = std::string_view(), std::string_view password = std::string_view(), - boost::beast::http::verb verb = boost::beast::http::verb::get, Session::ChunkHandler &&chunkHandler = Session::ChunkHandler()); + boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional bodyLimit = std::nullopt, + Session::ChunkHandler &&chunkHandler = Session::ChunkHandler()); } // namespace WebClient } // namespace LibRepoMgr