Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Increase the max. download size for packages and allow configuring it

This commit is contained in:
Martchus 2021-07-12 14:45:44 +02:00
parent 5ec527bebb
commit c05a581322
10 changed files with 24 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cli/main.cpp
View File

@ -608,7 +608,7 @@ int main(int argc, const char *argv[])
LibRepoMgr::WebClient::runSessionFromUrl(ioContext, sslContext, url,
std::bind(&handleResponse, std::ref(url), std::placeholders::_1, std::placeholders::_2, rawArg.isPresent() ? printRawData : printer,
std::ref(returnCode)),
std::string(), config.userName, config.password, verb, chunkHandler);
std::string(), config.userName, config.password, verb, std::nullopt, chunkHandler);
ioContext.run();
return returnCode;
}

1
librepomgr/buildactions/buildactionprivate.h
View File

@ -434,6 +434,7 @@ private:
std::vector<DatabaseToConsider> m_relevantPackagesByDatabase;
std::atomic_size_t m_remainingPackages;
WebClient::PackageCachingDataForSession m_cachingData;
std::uint64_t m_packageDownloadSizeLimit;
};
struct LIBREPOMGR_EXPORT CheckForProblems : public InternalBuildAction {

3
librepomgr/buildactions/reloadlibrarydependencies.cpp
View File

@ -55,6 +55,7 @@ void ReloadLibraryDependencies::run()
// use cache directory from global configuration
auto buildLock = m_setup.building.lockToRead();
const auto cacheDir = m_setup.building.packageCacheDir + '/';
m_packageDownloadSizeLimit = m_setup.building.packageDownloadSizeLimit;
buildLock.unlock();
// find relevant databases and packages
@ -237,7 +238,7 @@ void LibRepoMgr::ReloadLibraryDependencies::downloadPackagesFromMirror()
m_buildAction->appendOutput(Phrases::SuccessMessage, "Downloading ", packagesWhichNeedCaching, " binary packages from mirror ...\n");
WebClient::cachePackages(m_buildAction->log(),
std::make_shared<WebClient::PackageCachingSession>(m_cachingData, m_setup.building.ioContext, m_setup.webServer.sslContext,
std::bind(&ReloadLibraryDependencies::loadPackageInfoFromContents, this)));
std::bind(&ReloadLibraryDependencies::loadPackageInfoFromContents, this)), m_packageDownloadSizeLimit);
}
void ReloadLibraryDependencies::loadPackageInfoFromContents()

8
librepomgr/helper.h
View File

@ -39,7 +39,8 @@ inline std::optional<std::string_view> getLastValueSv(const std::multimap<std::s
return std::nullopt;
}
template <typename TargetType> void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result);
template <typename TargetType, Traits::DisableIf<std::is_integral<TargetType>> * = nullptr>
void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result);
template <>
inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, boost::asio::ip::address &result)
@ -59,7 +60,8 @@ inline void convertValue(const std::multimap<std::string, std::string> &multimap
}
}
template <> inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, unsigned short &result)
template <typename TargetType, Traits::EnableIf<std::is_integral<TargetType>> * = nullptr>
inline void convertValue(const std::multimap<std::string, std::string> &multimap, const std::string &key, TargetType &result)
{
using namespace std;
using namespace CppUtilities;
@ -67,7 +69,7 @@ template <> inline void convertValue(const std::multimap<std::string, std::strin
if (const char *const value = getLastValue(multimap, key)) {
try {
result = stringToNumber<unsigned short>(value);
result = stringToNumber<TargetType>(value);
} catch (const ConversionException &) {
cerr << Phrases::ErrorMessage << "Specified number \"" << value << "\" for key \"" << key << "\" is invalid." << Phrases::End;
return;

1
librepomgr/serversetup.cpp
View File

@ -118,6 +118,7 @@ void ServiceSetup::BuildSetup::applyConfig(const std::multimap<std::string, std:
convertValue(multimap, "makechrootpkg_flags", makechrootpkgFlags);
convertValue(multimap, "makepkg_flags", makepkgFlags);
convertValue(multimap, "package_cache_dir", packageCacheDir);
convertValue(multimap, "package_download_size_limit", packageDownloadSizeLimit);
convertValue(multimap, "test_files_dir", testFilesDir);
convertValue(multimap, "load_files_dbs", loadFilesDbs);
}

1
librepomgr/serversetup.h
View File

@ -100,6 +100,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
std::vector<std::string> makechrootpkgFlags;
std::vector<std::string> makepkgFlags;
std::string packageCacheDir;
std::uint64_t packageDownloadSizeLimit = 500 * 1024 * 1024;
std::string testFilesDir;
BuildPresets presets;
bool loadFilesDbs = false;

4
librepomgr/webclient/database.cpp
View File

@ -190,7 +190,7 @@ PackageCachingDataForPackage *PackageCachingSession::getCurrentDataAndSelectNext
return data;
}
void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::size_t maxParallelDownloads)
void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::optional<std::uint64_t> bodyLimit, std::size_t maxParallelDownloads)
{
for (std::size_t startedDownloads = 0; startedDownloads < maxParallelDownloads; ++startedDownloads) {
auto *const cachingData = packageCachingSession->getCurrentDataAndSelectNext();
@ -217,7 +217,7 @@ void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&pac
}
cachePackages(log, std::move(packageCachingSession), 1);
},
std::string(cachingData->destinationFilePath));
std::string(cachingData->destinationFilePath), std::string_view(), std::string_view(), boost::beast::http::verb::get, bodyLimit);
}
}

5
librepomgr/webclient/database.h
View File

@ -6,6 +6,7 @@
#include "./session.h"
#include <optional>
#include <string>
#include <vector>
@ -44,10 +45,10 @@ using PackageCachingDataForDatabase = std::unordered_map<std::string_view, Packa
using PackageCachingDataForSession = std::unordered_map<std::string_view, PackageCachingDataForDatabase>;
struct PackageCachingSession;
void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::size_t maxParallelDownloads = 8);
void cachePackages(LogContext &log, std::shared_ptr<PackageCachingSession> &&packageCachingSession, std::optional<std::uint64_t> bodyLimit = std::nullopt, std::size_t maxParallelDownloads = 8);
struct PackageCachingSession : public MultiSession<void> {
friend void cachePackages(LogContext &, std::shared_ptr<PackageCachingSession> &&, std::size_t);
friend void cachePackages(LogContext &, std::shared_ptr<PackageCachingSession> &&, std::optional<std::uint64_t>, std::size_t);
using SharedPointerType = std::shared_ptr<PackageCachingSession>;
explicit PackageCachingSession(

8
librepomgr/webclient/session.cpp
View File

@ -41,7 +41,7 @@ void Session::setChunkHandler(ChunkHandler &&handler)
m_chunkProcessing->handler = std::move(handler);
}
void Session::run(const char *host, const char *port, http::verb verb, const char *target, unsigned int version)
void Session::run(const char *host, const char *port, http::verb verb, const char *target, std::optional<std::uint64_t> bodyLimit, unsigned int version)
{
// set SNI Hostname (many hosts need this to handshake successfully)
auto *const sslStream = std::get_if<SslStream>(&m_stream);
@ -65,7 +65,7 @@ void Session::run(const char *host, const char *port, http::verb verb, const cha
if (!destinationFilePath.empty()) {
auto &fileResponse = response.emplace<FileResponse>();
boost::beast::error_code errorCode;
fileResponse.body_limit(100 * 1024 * 1024);
fileResponse.body_limit(bodyLimit.value_or(500 * 1024 * 1024));
fileResponse.get().body().open(destinationFilePath.data(), file_mode::write, errorCode);
if (errorCode != boost::beast::errc::success) {
m_handler(*this, HttpClientError("opening output file", errorCode));
@ -247,7 +247,7 @@ void Session::closed(boost::beast::error_code ec)
std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext,
std::string_view url, Session::Handler &&handler, std::string &&destinationPath, std::string_view userName, std::string_view password,
boost::beast::http::verb verb, Session::ChunkHandler &&chunkHandler)
boost::beast::http::verb verb, std::optional<std::uint64_t> bodyLimit, Session::ChunkHandler &&chunkHandler)
{
std::string host, port, target;
auto ssl = false;
@ -294,7 +294,7 @@ std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asi
if (chunkHandler) {
session->setChunkHandler(std::move(chunkHandler));
}
session->run(host.data(), port.data(), verb, target.data());
session->run(host.data(), port.data(), verb, target.data(), bodyLimit);
return std::variant<std::string, std::shared_ptr<Session>>(std::move(session));
}

6
librepomgr/webclient/session.h
View File

@ -11,6 +11,7 @@
#include <boost/asio/ssl.hpp>
#include <functional>
#include <optional>
#include <stdexcept>
#include <variant>
@ -61,7 +62,7 @@ public:
explicit Session(boost::asio::io_context &ioContext, boost::asio::ssl::context &sslContext, Handler &&handler);
void setChunkHandler(ChunkHandler &&handler);
void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, unsigned int version = 11);
void run(const char *host, const char *port, boost::beast::http::verb verb, const char *target, std::optional<std::uint64_t> bodyLimit = std::nullopt, unsigned int version = 11);
private:
using RawSocket = boost::asio::ip::tcp::socket;
@ -141,7 +142,8 @@ inline Session::Session(boost::asio::io_context &ioContext, boost::asio::ssl::co
LIBREPOMGR_EXPORT std::variant<std::string, std::shared_ptr<Session>> runSessionFromUrl(boost::asio::io_context &ioContext,
boost::asio::ssl::context &sslContext, std::string_view url, Session::Handler &&handler, std::string &&destinationPath = std::string(),
std::string_view userName = std::string_view(), std::string_view password = std::string_view(),
boost::beast::http::verb verb = boost::beast::http::verb::get, Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
boost::beast::http::verb verb = boost::beast::http::verb::get, std::optional<std::uint64_t> bodyLimit = std::nullopt,
Session::ChunkHandler &&chunkHandler = Session::ChunkHandler());
} // namespace WebClient
} // namespace LibRepoMgr