Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Allow fetching official PKGBUILDs from GitLab 

The old approach of using a local checkout does not work anymore
after the Git migration. One could use
`pkgctl repo clone --universe` to get a similar checkout but this
takes ages on my server. So let's better just download the
individual sources as needed, similar to how it is done for AUR
packages.
This commit is contained in:
Martchus 2023-05-28 23:43:13 +02:00
parent feadac7a66
commit 1376a01570
6 changed files with 70 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
librepomgr/buildactions/buildactionmeta.cpp
View File

@ -172,6 +172,12 @@ BuildActionMetaInfo::BuildActionMetaInfo()
.desc = "Whether pulling-in further dependencies leads to an error causing subsequent build actions not to be executed automatically; useful to be able to review packages added to the build before conducting the build",
.param = "pulling-in-new-deps-unexpected",
},
BuildActionFlagMetaInfo{
.id = static_cast<BuildActionFlagType>(PrepareBuildFlags::FetchOfficialPackageSources),
.name = "Fetch official package sources",
.desc = "Whether PKGBUILDs from official Arch Linux Git repositories should be downloaded (if not present, the AUR will still be checked; local PKGBUILDs still have precedence)",
.param = "fetch-official-pkgbuilds",
},
},
.settings = {
BuildActionSettingMetaInfo{

1
librepomgr/buildactions/buildactionmeta.h
View File

@ -78,6 +78,7 @@ enum class PrepareBuildFlags : BuildActionFlagType {
KeepPkgRelAndEpoch = (1 << 3),
ResetChrootSettings = (1 << 4),
PullingInFurtherDependenciesUnexpected = (1 << 5),
FetchOfficialPackageSources = (1 << 6),
};
enum class ConductBuildFlags : BuildActionFlagType {
None,

2
librepomgr/buildactions/buildactionprivate.h
View File

@ -507,6 +507,7 @@ private:
std::vector<std::vector<std::string>> m_batches;
std::vector<std::string> m_cyclicLeftovers;
std::vector<std::string> m_warnings;
std::unordered_set<std::string> m_cleanedSourceDirs;
bool m_forceBumpPackageVersion = false;
bool m_cleanSourceDirectory = false;
bool m_keepOrder = false;
@ -514,6 +515,7 @@ private:
bool m_resetChrootSettings = false;
bool m_pullingInFurtherDependenciesUnexpected = false;
bool m_pulledInFurtherDependencies = false;
bool m_fetchOfficialSources = false;
};
struct LIBREPOMGR_EXPORT BatchProcessingSession : public MultiSession<std::string> {

13
librepomgr/buildactions/preparebuild.cpp
View File

@ -76,6 +76,7 @@ void PrepareBuild::run()
m_keepPkgRelAndEpoch = flags & PrepareBuildFlags::KeepPkgRelAndEpoch;
m_resetChrootSettings = flags & PrepareBuildFlags::ResetChrootSettings;
m_pullingInFurtherDependenciesUnexpected = flags & PrepareBuildFlags::PullingInFurtherDependenciesUnexpected;
m_fetchOfficialSources = flags & PrepareBuildFlags::FetchOfficialPackageSources;
if (m_forceBumpPackageVersion && m_keepPkgRelAndEpoch) {
reportError("Can not force-bump pkgrel and keeping it at the same time.");
return;
@ -384,7 +385,7 @@ void PrepareBuild::fetchMissingBuildData()
// clean existing source directory if cleanup is enabled
try {
const auto sourceDirectoryExists = filesystem::exists(buildData.sourceDirectory);
if (m_cleanSourceDirectory && sourceDirectoryExists) {
if (m_cleanSourceDirectory && sourceDirectoryExists && m_cleanedSourceDirs.emplace(buildData.sourceDirectory).second) {
filesystem::remove_all(buildData.sourceDirectory);
} else if (sourceDirectoryExists) {
// verify the PKGBUILD file exists
@ -400,7 +401,7 @@ void PrepareBuild::fetchMissingBuildData()
makeSrcInfo(multiSession, buildData.sourceDirectory, packageName);
needToGeneratedSrcInfo = true;
} else {
string srcInfo;
auto srcInfo = std::string();
try {
srcInfo = readFile(srcInfoPath, 0x10000);
} catch (const std::ios_base::failure &e) {
@ -472,6 +473,7 @@ void PrepareBuild::fetchMissingBuildData()
snapshotQueries.emplace_back(WebClient::AurSnapshotQueryParams{
.packageName = &packageName,
.targetDirectory = &buildData.sourceDirectory,
.tryOfficial = m_fetchOfficialSources,
});
addPackageToLogLine(logLines[1], packageName);
}
@ -811,6 +813,7 @@ void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::Conta
// populate build data from responses and add further dependencies
auto furtherDependenciesNeeded = false;
auto sourcesMissing = false;
auto needToFetchAgain = false;
for (auto &response : responses) {
if (response.packageName.empty()) {
auto &buildData = m_buildDataByPackage["?"];
@ -834,6 +837,10 @@ void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::Conta
m_buildAction->artefacts.emplace_back(buildData.sourceDirectory + "/PKGBUILD"); // FIXME: add all files as artefacts
continue;
}
if (response.isOfficial && buildData.error.empty()) {
needToFetchAgain = true;
continue;
}
if (buildData.error.empty()) {
buildData.error = "no build data available";
}
@ -863,7 +870,7 @@ void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::Conta
if (furtherDependenciesNeeded) {
m_pulledInFurtherDependencies = true;
}
if (!sourcesMissing && furtherDependenciesNeeded) {
if (!sourcesMissing && (furtherDependenciesNeeded || needToFetchAgain)) {
fetchMissingBuildData();
return;
}

63
librepomgr/webclient/aur.cpp
View File

@ -85,7 +85,7 @@ std::shared_ptr<AurQuerySession> queryAurPackagesInternal(LogContext &log, Servi
log("Retrieving ", packages.size(), " packages from the AUR\n");
constexpr auto packagesPerQuery = 100;
auto multiSession = AurQuerySession::create(ioContext, move(handler));
auto multiSession = AurQuerySession::create(ioContext, std::move(handler));
for (auto i = packages.cbegin(), end = packages.cend(); i != end;) {
auto session = make_shared<WebClient::Session>(ioContext, setup.webServer.sslContext,
@ -137,7 +137,7 @@ std::shared_ptr<AurQuerySession> queryAurPackagesInternal(LogContext &log, Servi
std::shared_ptr<AurQuerySession> queryAurPackages(LogContext &log, ServiceSetup &setup, const std::vector<string> &packages,
boost::asio::io_context &ioContext, typename AurQuerySession::HandlerType &&handler)
{
return queryAurPackagesInternal(log, setup, packages, ioContext, move(handler));
return queryAurPackagesInternal(log, setup, packages, ioContext, std::move(handler));
}
/*!
@ -149,7 +149,7 @@ std::shared_ptr<AurQuerySession> queryAurPackages(LogContext &log, ServiceSetup
const std::unordered_map<string, std::shared_ptr<Package>> &packages, boost::asio::io_context &ioContext,
typename AurQuerySession::HandlerType &&handler)
{
return queryAurPackagesInternal(log, setup, packages, ioContext, move(handler));
return queryAurPackagesInternal(log, setup, packages, ioContext, std::move(handler));
}
/*!
@ -181,11 +181,16 @@ std::shared_ptr<AurQuerySession> queryAurPackagesForDatabase(LogContext &log, Se
return nullptr;
}
configReadLock->unlock();
return queryAurPackages(log, setup, missingPackages, ioContext, move(handler));
return queryAurPackages(log, setup, missingPackages, ioContext, std::move(handler));
}
/*!
* \brief Queries the AUR asynchronously to get the latest snapshot for the specified \a packageNames.
* \remarks
* If the "tryOfficial" flag in the parameter is set then this function attempts to download the sources from official Git repositories
* first. If the official repositories are unavailable or the package cannot be found there it will still fallback to downloading the
* sources from AUR.
*
*/
void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector<AurSnapshotQueryParams> &queryParams,
boost::asio::io_context &ioContext, std::shared_ptr<AurSnapshotQuerySession> &multiSession)
@ -193,8 +198,15 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector<A
CPP_UTILITIES_UNUSED(log)
for (const auto &params : queryParams) {
auto session = std::make_shared<WebClient::Session>(ioContext, setup.webServer.sslContext,
[multiSession, params](WebClient::Session &session2, const WebClient::HttpClientError &error) mutable {
[multiSession, params = params, &log, &setup, &ioContext](WebClient::Session &session2, const WebClient::HttpClientError &error) mutable {
if (error.errorCode != boost::beast::errc::success && error.errorCode.message() != "stream truncated") {
// download from AUR after all if the sources cannot be found in the official Arch Linux Git repositories
if (params.tryOfficial) {
params.tryOfficial = false;
queryAurSnapshots(log, setup, { params }, ioContext, multiSession);
return;
}
multiSession->addResponse(WebClient::AurSnapshotResult{ .packageName = *params.packageName,
.error = "Unable to retrieve AUR snapshot tarball for package " % *params.packageName % ": " + error.what() });
return;
@ -203,6 +215,12 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector<A
// parse retrieved archive
const auto &response = get<Response>(session2.response);
if (response.result() != boost::beast::http::status::ok) {
// download from AUR after all if the sources cannot be found in the official Arch Linux Git repositories
if (response.result() != boost::beast::http::status::not_found && params.tryOfficial) {
params.tryOfficial = false;
queryAurSnapshots(log, setup, { params }, ioContext, multiSession);
return;
}
multiSession->addResponse(WebClient::AurSnapshotResult{ .packageName = *params.packageName,
.error
= "Unable to retrieve AUR snapshot tarball for package " % *params.packageName % ": AUR returned " % response.result_int()
@ -225,7 +243,10 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector<A
if (!startsWith(directory.first, *params.packageName)) {
continue;
}
const auto directoryPath = string_view{ directory.first }.substr(params.packageName->size());
auto directoryPath = string_view{ directory.first }.substr(params.packageName->size());
if (directoryPath.starts_with("-main")) {
directoryPath = directoryPath.substr(5);
}
if (directoryPath.empty()) {
for (const auto &rootFile : directory.second) {
if (rootFile.name == ".SRCINFO") {
@ -274,23 +295,33 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector<A
}
// validate what we've got and add response
if (!haveSrcFileInfo) {
result.error = ".SRCINFO is missing";
}
if (!havePkgbuild) {
result.error = "PKGINFO is missing";
}
if (result.packages.empty() || result.packages.front().pkg->name.empty()) {
result.error = "Unable to parse .SRCINFO: no package name present";
} else if (!result.packages.front().pkg->sourceInfo.has_value()) {
result.error = "Unable to parse .SRCINFO: no source info present";
if (params.tryOfficial) {
result.isOfficial = true;
} else {
if (!haveSrcFileInfo) {
result.error = ".SRCINFO is missing";
}
if (result.packages.empty() || result.packages.front().pkg->name.empty()) {
result.error = "Unable to parse .SRCINFO: no package name present";
} else if (!result.packages.front().pkg->sourceInfo.has_value()) {
result.error = "Unable to parse .SRCINFO: no source info present";
}
}
multiSession->addResponse(move(result));
multiSession->addResponse(std::move(result));
});
// run query, e.g. https: //aur.archlinux.org/cgit/aur.git/snapshot/mingw-w64-configure.tar.gz
const auto url = "/cgit/aur.git/snapshot/" % WebAPI::Url::encodeValue(*params.packageName) + ".tar.gz";
session->run("aur.archlinux.org", "443", boost::beast::http::verb::get, url.data(), 11);
const auto encodedPackageName = WebAPI::Url::encodeValue(*params.packageName);
if (params.tryOfficial) {
const auto url = "/archlinux/packaging/packages/" % encodedPackageName % "/-/archive/main/" % encodedPackageName + "-main.tar.gz";
session->run("gitlab.archlinux.org", "443", boost::beast::http::verb::get, url.data(), 11);
} else {
const auto url = "/cgit/aur.git/snapshot/" % encodedPackageName + ".tar.gz";
session->run("aur.archlinux.org", "443", boost::beast::http::verb::get, url.data(), 11);
}
}
}

6
librepomgr/webclient/aur.h
View File

@ -21,10 +21,12 @@ struct AurSnapshotResult {
std::string errorOutput;
std::vector<LibPkg::PackageSpec> packages;
std::string error;
bool isOfficial = false;
};
struct AurSnapshotQueryParams {
const std::string *packageName;
const std::string *targetDirectory;
const std::string *packageName = nullptr;
const std::string *targetDirectory = nullptr;
bool tryOfficial = false;
};
using AurQuerySession = MultiSession<LibPkg::PackageSpec>;