Increase the max. download size for packages and allow configuring it
This commit is contained in:
parent
5ec527bebb
commit
c05a581322
|
@ -608,7 +608,7 @@ int main(int argc, const char *argv[])
|
|||
LibRepoMgr::WebClient::runSessionFromUrl(ioContext, sslContext, url,
|
||||
std::bind(&handleResponse, std::ref(url), std::placeholders::_1, std::placeholders::_2, rawArg.isPresent() ? printRawData : printer,
|
||||
std::ref(returnCode)),
|
||||
std::string(), config.userName, config.password, verb, chunkHandler);
|
||||
std::string(), config.userName, config.password, verb, std::nullopt, chunkHandler);
|
||||
ioContext.run();
|
||||
return returnCode;
|
||||
}
|
||||
|
|
|
@ -434,6 +434,7 @@ private:
|
|||
std::vector<DatabaseToConsider> m_relevantPackagesByDatabase;
|
||||
std::atomic_size_t m_remainingPackages;
|
||||
WebClient::PackageCachingDataForSession m_cachingData;
|
||||
std::uint64_t m_packageDownloadSizeLimit;
|
||||
};
|
||||
|
||||
struct LIBREPOMGR_EXPORT CheckForProblems : public InternalBuildAction {
|
||||
|
|
|
@ -55,6 +55,7 @@ void ReloadLibraryDependencies::run()
|
|||
// use cache directory from global configuration
|
||||
auto buildLock = m_setup.building.lockToRead();
|
||||
const auto cacheDir = m_setup.building.packageCacheDir + '/';
|
||||
m_packageDownloadSizeLimit = m_setup.building.packageDownloadSizeLimit;
|
||||
buildLock.unlock();
|
||||
|
||||
// find relevant databases and packages
|
||||
|
@ -237,7 +238,7 @@ void LibRepoMgr::ReloadLibraryDependencies::downloadPackagesFromMirror()
|
|||
m_buildAction->appendOutput(Phrases::SuccessMessage, "Downloading ", packagesWhichNeedCaching, " binary packages from mirror ...\n");
|
||||
WebClient::cachePackages(m_buildAction->log(),
|
||||
std::make_shared<WebClient::PackageCachingSession>(m_cachingData, m_setup.building.ioContext, m_setup.webServer.sslContext,
|
||||
std::bind(&ReloadLibraryDependencies::loadPackageInfoFromContents, this)));
|
||||
std::bind(&ReloadLibraryDependencies::loadPackageInfoFromContents, this)), m_packageDownloadSizeLimit);
|
||||
}
|
||||
|
||||
void ReloadLibraryDependencies::loadPackageInfoFromContents()
|
||||
|
|
|
@ -39,7 +39,8 @@ inline std::optional<std::string_view> getLastValueSv(const std::multimap<std::s
|
|||
return std::nullopt;
|
||||
}
|
||||
|
||||
template <typename TargetType> void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result);
|
||||
template <typename TargetType, Traits::DisableIf<std::is_integral<TargetType>> * = nullptr>
|
||||
void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result);
|
||||
|
||||
template <>
|
||||
inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, boost::asio::ip::address &result)
|
||||
|
@ -59,7 +60,8 @@ inline void convertValue(const std::multimap<std::string, std::string> &multimap
|
|||
}
|
||||
}
|
||||
|
||||
template <> inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, unsigned short &result)
|
||||
template <typename TargetType, Traits::EnableIf<std::is_integral<TargetType>> * = nullptr>
|
||||
inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result)
|
||||
{
|
||||
using namespace std;
|
||||
using namespace CppUtilities;
|
||||
|
@ -67,7 +69,7 @@ template <> inline void convertValue(const std::multimap<std::string, std::strin
|
|||
|
||||
if (const char *const value = getLastValue(multimap, key)) {
|
||||
try {
|
||||
result = stringToNumber<unsigned short>(value);
|
||||
result = stringToNumber<TargetType>(value);
|
||||
} catch (const ConversionException &) {
|
||||
cerr << Phrases::ErrorMessage << "Specified number \"" << value << "\" for key \"" << key << "\" is invalid." << Phrases::End;
|
||||
return;
|
||||
|
|
|
@ -118,6 +118,7 @@ void ServiceSetup::BuildSetup::applyConfig(const std::multimap<std::string, std:
|
|||
convertValue(multimap, "makechrootpkg_flags", makechrootpkgFlags);
|
||||
convertValue(multimap, "makepkg_flags", makepkgFlags);
|
||||
convertValue(multimap, "package_cache_dir", packageCacheDir);
|
||||
convertValue(multimap, "package_download_size_limit", packageDownloadSizeLimit);
|
||||
convertValue(multimap, "test_files_dir", testFilesDir);
|
||||
convertValue(multimap, "load_files_dbs", loadFilesDbs);
|
||||
}
|
||||
|
|
|
@ -100,6 +100,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
|
|||
std::vector<std::string> makechrootpkgFlags;
|
||||
std::vector<std::string> makepkgFlags;
|
||||
std::string packageCacheDir;
|
||||
std::uint64_t packageDownloadSizeLimit = 500 * 1024 * 1024;
|
||||
std::string testFilesDir;
|
||||
BuildPresets presets;
|
||||
bool loadFilesDbs = false;
|
||||
|
|
|
@ -190,7 +190,7 @@ PackageCachingDataForPackage *PackageCachingSession::getCurrentDataAndSelectNext
|
|||
return data;
|
||||
}
|
||||
|
||||
void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::size_t maxParallelDownloads)
|
||||
void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::optional<std::uint64_t> bodyLimit, std::size_t maxParallelDownloads)
|
||||
{
|
||||
for (std::size_t startedDownloads = 0; startedDownloads < maxParallelDownloads; ++startedDownloads) {
|
||||
auto *const cachingData = packageCachingSession->getCurrentDataAndSelectNext();
|
||||
|
@ -217,7 +217,7 @@ void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&pac
|
|||
}
|
||||
cachePackages(log, std::move(packageCachingSession), 1);
|
||||
},
|
||||
std::string(cachingData->destinationFilePath));
|
||||
std::string(cachingData->destinationFilePath), std::string_view(), std::string_view(), boost::beast::http::verb::get, bodyLimit);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -6,6 +6,7 @@
|
|||
|
||||
#include "./session.h"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
@ -44,10 +45,10 @@ using PackageCachingDataForDatabase = std::unordered_map<std::string_view, Packa
|
|||
using PackageCachingDataForSession = std::unordered_map<std::string_view, PackageCachingDataForDatabase>;
|
||||
|
||||
struct PackageCachingSession;
|
||||
void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::size_t maxParallelDownloads = 8);
|
||||
void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::optional<std::uint64_t> bodyLimit = std::nullopt, std::size_t maxParallelDownloads = 8);
|
||||
|
||||
struct PackageCachingSession : public MultiSession<void> {
|
||||
friend void cachePackages(LogContext &, std::shared_ptr<PackageCachingSession> &&, std::size_t);
|
||||
friend void cachePackages(LogContext &, std::shared_ptr<PackageCachingSession> &&, std::optional<std::uint64_t>, std::size_t);
|
||||
using SharedPointerType = std::shared_ptr<PackageCachingSession>;
|
||||
|
||||
explicit PackageCachingSession(
|
||||
|
|
|
@ -41,7 +41,7 @@ void Session::setChunkHandler(ChunkHandler &&handler)
|
|||
m_chunkProcessing->handler = std::move(handler);
|
||||
}
|
||||
|
||||
void Session::run(const char *host, const char *port, http::verb verb, const char *target, unsigned int version)
|
||||
void Session::run(const char *host, const char *port, http::verb verb, const char *target, std::optional<std::uint64_t> bodyLimit, unsigned int version)
|
||||
{
|
||||
// set SNI Hostname (many hosts need this to handshake successfully)
|
||||
auto *const sslStream = std::get_if<SslStream>(&m_stream);
|
||||
|
@ -65,7 +65,7 @@ void Session::run(const char *host, const char *port, http::verb verb, const cha
|
|||
if (!destinationFilePath.empty()) {
|
||||
auto &fileResponse = response.emplace<FileResponse>();
|
||||
boost::beast::error_code errorCode;
|
||||
fileResponse.body_limit(100 * 1024 * 1024);
|
||||
fileResponse.body_limit(bodyLimit.value_or(500 * 1024 * 1024));
|
||||
fileResponse.get().body().open(destinationFilePath.data(), file_mode::write, errorCode);
|
||||
if (errorCode != boost::beast::errc::success) {
|
||||
m_handler(*this, HttpClientError("opening output file", errorCode));
|
||||
|
@ -247,7 +247,7 @@ void Session::closed(boost::beast::error_code ec)
|
|||
|
||||
std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
|
||||
std::string_view url, Session::Handler &&handler, std::string &&destinationPath, std::string_view userName, std::string_view password,
|
||||
boost::beast::http::verb verb, Session::ChunkHandler &&chunkHandler)
|
||||
boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
|
||||
{
|
||||
std::string host, port, target;
|
||||
auto ssl = false;
|
||||
|
@ -294,7 +294,7 @@ std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asi
|
|||
if (chunkHandler) {
|
||||
session->setChunkHandler(std::move(chunkHandler));
|
||||
}
|
||||
session->run(host.data(), port.data(), verb, target.data());
|
||||
session->run(host.data(), port.data(), verb, target.data(), bodyLimit);
|
||||
return std::variant<std::string, std::shared_ptr<Session>>(std::move(session));
|
||||
}
|
||||
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
#include <boost/asio/ssl.hpp>
|
||||
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <stdexcept>
|
||||
#include <variant>
|
||||
|
||||
|
@ -61,7 +62,7 @@ public:
|
|||
explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler);
|
||||
|
||||
void setChunkHandler(ChunkHandler &&handler);
|
||||
void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, unsigned int version = 11);
|
||||
void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, std::optional<std::uint64_t> bodyLimit = std::nullopt, unsigned int version = 11);
|
||||
|
||||
private:
|
||||
using RawSocket = boost::asio::ip::tcp::socket;
|
||||
|
@ -141,7 +142,8 @@ inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::co
|
|||
LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext,
|
||||
boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, std::string &&destinationPath = std::string(),
|
||||
std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
|
||||
boost::beast::http::verb verb = boost::beast::http::verb::get, Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
|
||||
boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
|
||||
Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
|
||||
|
||||
} // namespace WebClient
|
||||
} // namespace LibRepoMgr
|
||||
|
|
Loading…
Reference in New Issue