From 1376a0157093d31699bb08f561adbae9e24a6e8e Mon Sep 17 00:00:00 2001 From: Martchus Date: Sun, 28 May 2023 23:43:13 +0200 Subject: [PATCH] Allow fetching official PKGBUILDs from GitLab The old approach of using a local checkout does not work anymore after the Git migration. One could use `pkgctl repo clone --universe` to get a similar checkout but this takes ages on my server. So let's better just download the individual sources as needed, similar to how it is done for AUR packages. --- librepomgr/buildactions/buildactionmeta.cpp | 6 ++ librepomgr/buildactions/buildactionmeta.h | 1 + librepomgr/buildactions/buildactionprivate.h | 2 + librepomgr/buildactions/preparebuild.cpp | 13 +++- librepomgr/webclient/aur.cpp | 63 +++++++++++++++----- librepomgr/webclient/aur.h | 6 +- 6 files changed, 70 insertions(+), 21 deletions(-) diff --git a/librepomgr/buildactions/buildactionmeta.cpp b/librepomgr/buildactions/buildactionmeta.cpp index 55796ba..1ecc97c 100644 --- a/librepomgr/buildactions/buildactionmeta.cpp +++ b/librepomgr/buildactions/buildactionmeta.cpp @@ -172,6 +172,12 @@ BuildActionMetaInfo::BuildActionMetaInfo() .desc = "Whether pulling-in further dependencies leads to an error causing subsequent build actions not to be executed automatically; useful to be able to review packages added to the build before conducting the build", .param = "pulling-in-new-deps-unexpected", }, + BuildActionFlagMetaInfo{ + .id = static_cast(PrepareBuildFlags::FetchOfficialPackageSources), + .name = "Fetch official package sources", + .desc = "Whether PKGBUILDs from official Arch Linux Git repositories should be downloaded (if not present, the AUR will still be checked; local PKGBUILDs still have precedence)", + .param = "fetch-official-pkgbuilds", + }, }, .settings = { BuildActionSettingMetaInfo{ diff --git a/librepomgr/buildactions/buildactionmeta.h b/librepomgr/buildactions/buildactionmeta.h index 3eeb66d..fb798ed 100644 --- a/librepomgr/buildactions/buildactionmeta.h +++ b/librepomgr/buildactions/buildactionmeta.h @@ -78,6 +78,7 @@ enum class PrepareBuildFlags : BuildActionFlagType { KeepPkgRelAndEpoch = (1 << 3), ResetChrootSettings = (1 << 4), PullingInFurtherDependenciesUnexpected = (1 << 5), + FetchOfficialPackageSources = (1 << 6), }; enum class ConductBuildFlags : BuildActionFlagType { None, diff --git a/librepomgr/buildactions/buildactionprivate.h b/librepomgr/buildactions/buildactionprivate.h index ebf10b8..eb6aad9 100644 --- a/librepomgr/buildactions/buildactionprivate.h +++ b/librepomgr/buildactions/buildactionprivate.h @@ -507,6 +507,7 @@ private: std::vector> m_batches; std::vector m_cyclicLeftovers; std::vector m_warnings; + std::unordered_set m_cleanedSourceDirs; bool m_forceBumpPackageVersion = false; bool m_cleanSourceDirectory = false; bool m_keepOrder = false; @@ -514,6 +515,7 @@ private: bool m_resetChrootSettings = false; bool m_pullingInFurtherDependenciesUnexpected = false; bool m_pulledInFurtherDependencies = false; + bool m_fetchOfficialSources = false; }; struct LIBREPOMGR_EXPORT BatchProcessingSession : public MultiSession { diff --git a/librepomgr/buildactions/preparebuild.cpp b/librepomgr/buildactions/preparebuild.cpp index 91b6cbb..ac04a6a 100644 --- a/librepomgr/buildactions/preparebuild.cpp +++ b/librepomgr/buildactions/preparebuild.cpp @@ -76,6 +76,7 @@ void PrepareBuild::run() m_keepPkgRelAndEpoch = flags & PrepareBuildFlags::KeepPkgRelAndEpoch; m_resetChrootSettings = flags & PrepareBuildFlags::ResetChrootSettings; m_pullingInFurtherDependenciesUnexpected = flags & PrepareBuildFlags::PullingInFurtherDependenciesUnexpected; + m_fetchOfficialSources = flags & PrepareBuildFlags::FetchOfficialPackageSources; if (m_forceBumpPackageVersion && m_keepPkgRelAndEpoch) { reportError("Can not force-bump pkgrel and keeping it at the same time."); return; @@ -384,7 +385,7 @@ void PrepareBuild::fetchMissingBuildData() // clean existing source directory if cleanup is enabled try { const auto sourceDirectoryExists = filesystem::exists(buildData.sourceDirectory); - if (m_cleanSourceDirectory && sourceDirectoryExists) { + if (m_cleanSourceDirectory && sourceDirectoryExists && m_cleanedSourceDirs.emplace(buildData.sourceDirectory).second) { filesystem::remove_all(buildData.sourceDirectory); } else if (sourceDirectoryExists) { // verify the PKGBUILD file exists @@ -400,7 +401,7 @@ void PrepareBuild::fetchMissingBuildData() makeSrcInfo(multiSession, buildData.sourceDirectory, packageName); needToGeneratedSrcInfo = true; } else { - string srcInfo; + auto srcInfo = std::string(); try { srcInfo = readFile(srcInfoPath, 0x10000); } catch (const std::ios_base::failure &e) { @@ -472,6 +473,7 @@ void PrepareBuild::fetchMissingBuildData() snapshotQueries.emplace_back(WebClient::AurSnapshotQueryParams{ .packageName = &packageName, .targetDirectory = &buildData.sourceDirectory, + .tryOfficial = m_fetchOfficialSources, }); addPackageToLogLine(logLines[1], packageName); } @@ -811,6 +813,7 @@ void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::Conta // populate build data from responses and add further dependencies auto furtherDependenciesNeeded = false; auto sourcesMissing = false; + auto needToFetchAgain = false; for (auto &response : responses) { if (response.packageName.empty()) { auto &buildData = m_buildDataByPackage["?"]; @@ -834,6 +837,10 @@ void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::Conta m_buildAction->artefacts.emplace_back(buildData.sourceDirectory + "/PKGBUILD"); // FIXME: add all files as artefacts continue; } + if (response.isOfficial && buildData.error.empty()) { + needToFetchAgain = true; + continue; + } if (buildData.error.empty()) { buildData.error = "no build data available"; } @@ -863,7 +870,7 @@ void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::Conta if (furtherDependenciesNeeded) { m_pulledInFurtherDependencies = true; } - if (!sourcesMissing && furtherDependenciesNeeded) { + if (!sourcesMissing && (furtherDependenciesNeeded || needToFetchAgain)) { fetchMissingBuildData(); return; } diff --git a/librepomgr/webclient/aur.cpp b/librepomgr/webclient/aur.cpp index 1c1282e..1bb26cf 100644 --- a/librepomgr/webclient/aur.cpp +++ b/librepomgr/webclient/aur.cpp @@ -85,7 +85,7 @@ std::shared_ptr queryAurPackagesInternal(LogContext &log, Servi log("Retrieving ", packages.size(), " packages from the AUR\n"); constexpr auto packagesPerQuery = 100; - auto multiSession = AurQuerySession::create(ioContext, move(handler)); + auto multiSession = AurQuerySession::create(ioContext, std::move(handler)); for (auto i = packages.cbegin(), end = packages.cend(); i != end;) { auto session = make_shared(ioContext, setup.webServer.sslContext, @@ -137,7 +137,7 @@ std::shared_ptr queryAurPackagesInternal(LogContext &log, Servi std::shared_ptr queryAurPackages(LogContext &log, ServiceSetup &setup, const std::vector &packages, boost::asio::io_context &ioContext, typename AurQuerySession::HandlerType &&handler) { - return queryAurPackagesInternal(log, setup, packages, ioContext, move(handler)); + return queryAurPackagesInternal(log, setup, packages, ioContext, std::move(handler)); } /*! @@ -149,7 +149,7 @@ std::shared_ptr queryAurPackages(LogContext &log, ServiceSetup const std::unordered_map> &packages, boost::asio::io_context &ioContext, typename AurQuerySession::HandlerType &&handler) { - return queryAurPackagesInternal(log, setup, packages, ioContext, move(handler)); + return queryAurPackagesInternal(log, setup, packages, ioContext, std::move(handler)); } /*! @@ -181,11 +181,16 @@ std::shared_ptr queryAurPackagesForDatabase(LogContext &log, Se return nullptr; } configReadLock->unlock(); - return queryAurPackages(log, setup, missingPackages, ioContext, move(handler)); + return queryAurPackages(log, setup, missingPackages, ioContext, std::move(handler)); } /*! * \brief Queries the AUR asynchronously to get the latest snapshot for the specified \a packageNames. + * \remarks + * If the "tryOfficial" flag in the parameter is set then this function attempts to download the sources from official Git repositories + * first. If the official repositories are unavailable or the package cannot be found there it will still fallback to downloading the + * sources from AUR. + * */ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector &queryParams, boost::asio::io_context &ioContext, std::shared_ptr &multiSession) @@ -193,8 +198,15 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector(ioContext, setup.webServer.sslContext, - [multiSession, params](WebClient::Session &session2, const WebClient::HttpClientError &error) mutable { + [multiSession, params = params, &log, &setup, &ioContext](WebClient::Session &session2, const WebClient::HttpClientError &error) mutable { if (error.errorCode != boost::beast::errc::success && error.errorCode.message() != "stream truncated") { + // download from AUR after all if the sources cannot be found in the official Arch Linux Git repositories + if (params.tryOfficial) { + params.tryOfficial = false; + queryAurSnapshots(log, setup, { params }, ioContext, multiSession); + return; + } + multiSession->addResponse(WebClient::AurSnapshotResult{ .packageName = *params.packageName, .error = "Unable to retrieve AUR snapshot tarball for package " % *params.packageName % ": " + error.what() }); return; @@ -203,6 +215,12 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector(session2.response); if (response.result() != boost::beast::http::status::ok) { + // download from AUR after all if the sources cannot be found in the official Arch Linux Git repositories + if (response.result() != boost::beast::http::status::not_found && params.tryOfficial) { + params.tryOfficial = false; + queryAurSnapshots(log, setup, { params }, ioContext, multiSession); + return; + } multiSession->addResponse(WebClient::AurSnapshotResult{ .packageName = *params.packageName, .error = "Unable to retrieve AUR snapshot tarball for package " % *params.packageName % ": AUR returned " % response.result_int() @@ -225,7 +243,10 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vectorsize()); + auto directoryPath = string_view{ directory.first }.substr(params.packageName->size()); + if (directoryPath.starts_with("-main")) { + directoryPath = directoryPath.substr(5); + } if (directoryPath.empty()) { for (const auto &rootFile : directory.second) { if (rootFile.name == ".SRCINFO") { @@ -274,23 +295,33 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vectorname.empty()) { - result.error = "Unable to parse .SRCINFO: no package name present"; - } else if (!result.packages.front().pkg->sourceInfo.has_value()) { - result.error = "Unable to parse .SRCINFO: no source info present"; + if (params.tryOfficial) { + result.isOfficial = true; + } else { + if (!haveSrcFileInfo) { + result.error = ".SRCINFO is missing"; + } + if (result.packages.empty() || result.packages.front().pkg->name.empty()) { + result.error = "Unable to parse .SRCINFO: no package name present"; + } else if (!result.packages.front().pkg->sourceInfo.has_value()) { + result.error = "Unable to parse .SRCINFO: no source info present"; + } } - multiSession->addResponse(move(result)); + multiSession->addResponse(std::move(result)); }); // run query, e.g. https: //aur.archlinux.org/cgit/aur.git/snapshot/mingw-w64-configure.tar.gz - const auto url = "/cgit/aur.git/snapshot/" % WebAPI::Url::encodeValue(*params.packageName) + ".tar.gz"; - session->run("aur.archlinux.org", "443", boost::beast::http::verb::get, url.data(), 11); + const auto encodedPackageName = WebAPI::Url::encodeValue(*params.packageName); + if (params.tryOfficial) { + const auto url = "/archlinux/packaging/packages/" % encodedPackageName % "/-/archive/main/" % encodedPackageName + "-main.tar.gz"; + session->run("gitlab.archlinux.org", "443", boost::beast::http::verb::get, url.data(), 11); + } else { + const auto url = "/cgit/aur.git/snapshot/" % encodedPackageName + ".tar.gz"; + session->run("aur.archlinux.org", "443", boost::beast::http::verb::get, url.data(), 11); + } } } diff --git a/librepomgr/webclient/aur.h b/librepomgr/webclient/aur.h index aeebe21..d07c408 100644 --- a/librepomgr/webclient/aur.h +++ b/librepomgr/webclient/aur.h @@ -21,10 +21,12 @@ struct AurSnapshotResult { std::string errorOutput; std::vector packages; std::string error; + bool isOfficial = false; }; struct AurSnapshotQueryParams { - const std::string *packageName; - const std::string *targetDirectory; + const std::string *packageName = nullptr; + const std::string *targetDirectory = nullptr; + bool tryOfficial = false; }; using AurQuerySession = MultiSession;