832 lines
41 KiB
C++
832 lines
41 KiB
C++
#include "./buildactionprivate.h"
|
|
|
|
#include "../logging.h"
|
|
|
|
#include "../libpkg/parser/utils.h"
|
|
|
|
#include <c++utilities/chrono/datetime.h>
|
|
#include <c++utilities/conversion/stringbuilder.h>
|
|
#include <c++utilities/conversion/stringconversion.h>
|
|
#include <c++utilities/io/ansiescapecodes.h>
|
|
#include <c++utilities/io/path.h>
|
|
|
|
#include <boost/process/search_path.hpp>
|
|
#include <boost/process/start_dir.hpp>
|
|
|
|
#include <filesystem>
|
|
#include <iostream>
|
|
#include <ranges>
|
|
|
|
using namespace std;
|
|
using namespace std::literals::string_view_literals;
|
|
using namespace CppUtilities;
|
|
using namespace CppUtilities::EscapeCodes;
|
|
|
|
namespace LibRepoMgr {
|
|
|
|
PackageMovementAction::PackageMovementAction(ServiceSetup &setup, const std::shared_ptr<BuildAction> &buildAction)
|
|
: InternalBuildAction(setup, buildAction)
|
|
{
|
|
}
|
|
|
|
bool PackageMovementAction::prepareRepoAction(RequiredDatabases requiredDatabases)
|
|
{
|
|
// initialize build action
|
|
auto configReadLock = init(BuildActionAccess::ReadConfig, requiredDatabases | RequiredDatabases::OneDestination, RequiredParameters::Packages);
|
|
if (std::holds_alternative<std::monostate>(configReadLock)) {
|
|
return false;
|
|
}
|
|
|
|
auto setupLock = m_setup.lockToRead();
|
|
m_repoRemovePath = findExecutable(m_setup.building.repoRemovePath);
|
|
if (requiredDatabases & RequiredDatabases::OneSource) {
|
|
m_repoAddPath = findExecutable(m_setup.building.repoAddPath);
|
|
}
|
|
setupLock.unlock();
|
|
|
|
// check executables
|
|
if (!checkExecutable(m_repoRemovePath)) {
|
|
reportError("Unable to find repo-remove executable \"" % m_setup.building.repoRemovePath + "\" in PATH.");
|
|
return false;
|
|
}
|
|
if (requiredDatabases & RequiredDatabases::OneSource && !checkExecutable(m_repoAddPath)) {
|
|
reportError("Unable to find repo-add executable \"" % m_setup.building.repoAddPath + "\" in PATH.");
|
|
return false;
|
|
}
|
|
|
|
// locate databases and packages
|
|
const auto *const destinationDb = *m_destinationDbs.begin();
|
|
m_destinationRepoDirectory = destinationDb->localPkgDir;
|
|
m_destinationDatabaseFile = fileName(destinationDb->path);
|
|
m_destinationDatabaseLockName = ServiceSetup::Locks::forDatabase(*destinationDb);
|
|
if (requiredDatabases & RequiredDatabases::OneSource) {
|
|
const auto *const sourceDb = *m_sourceDbs.begin();
|
|
m_sourceRepoDirectory = sourceDb->localPkgDir;
|
|
m_sourceDatabaseFile = fileName(sourceDb->path);
|
|
m_sourceDatabaseLockName = ServiceSetup::Locks::forDatabase(*sourceDb);
|
|
}
|
|
locatePackages();
|
|
configReadLock = std::monostate();
|
|
|
|
// error-out early if not even a single package could be located
|
|
if (m_packageLocations.empty()) {
|
|
m_result.errorMessage = "none of the specified packages could be located";
|
|
reportResultWithData(BuildActionResult::Failure);
|
|
return false;
|
|
}
|
|
|
|
// init working directory
|
|
initWorkingDirectory();
|
|
if (!m_result.errorMessage.empty()) {
|
|
reportResultWithData(BuildActionResult::Failure);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void PackageMovementAction::initWorkingDirectory()
|
|
{
|
|
if (m_buildAction->directory.empty()) {
|
|
auto directory = argsToString(m_buildAction->type == BuildActionType::MovePackages ? "repo-move-" : "repo-remove-",
|
|
DateTime::gmtNow().toIsoStringWithCustomDelimiters(TimeSpan(), '-', '-'), '-',
|
|
std::filesystem::path(m_destinationDatabaseFile).stem().string());
|
|
auto buildActionLock = m_setup.building.lockToWrite();
|
|
m_buildAction->directory = std::move(directory);
|
|
}
|
|
m_workingDirectory = determineWorkingDirectory(repoManagementWorkingDirectory);
|
|
try {
|
|
std::filesystem::create_directories(m_workingDirectory);
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_buildAction->log()(Phrases::ErrorMessage, "Unable to make working directory: ", e.what(), '\n');
|
|
m_result.errorMessage = argsToString("unable to make working directory: ", e.what());
|
|
}
|
|
}
|
|
|
|
void PackageMovementAction::locatePackages()
|
|
{
|
|
// determine repo path and package paths
|
|
LibPkg::Database *db;
|
|
if (m_sourceDbs.empty()) {
|
|
db = *m_destinationDbs.begin();
|
|
} else {
|
|
db = *m_sourceDbs.begin();
|
|
}
|
|
const auto &packages = db->packages;
|
|
for (const auto &packageName : m_buildAction->packageNames) {
|
|
const auto &package = packages.find(packageName);
|
|
if (package == packages.end()) {
|
|
m_result.failedPackages.emplace_back(packageName, "package not listed in database file");
|
|
continue;
|
|
}
|
|
auto packageLocation = db->locatePackage(package->second->computeFileName());
|
|
if (packageLocation.error.has_value()) {
|
|
m_result.failedPackages.emplace_back(
|
|
packageName, argsToString("unable to locate package within repo directory: ", packageLocation.error.value().what()));
|
|
continue;
|
|
}
|
|
if (!packageLocation.exists) {
|
|
m_result.failedPackages.emplace_back(packageName, "package not present within repo directory");
|
|
continue;
|
|
}
|
|
m_packageLocations.emplace_back(packageName, std::move(packageLocation), true);
|
|
}
|
|
}
|
|
|
|
void PackageMovementAction::reportResultWithData(BuildActionResult result)
|
|
{
|
|
auto buildLock = m_setup.building.lockToWrite();
|
|
m_buildAction->resultData = std::move(m_result);
|
|
reportResult(result);
|
|
}
|
|
|
|
RemovePackages::RemovePackages(ServiceSetup &setup, const std::shared_ptr<BuildAction> &buildAction)
|
|
: PackageMovementAction(setup, buildAction)
|
|
{
|
|
}
|
|
|
|
void RemovePackages::run()
|
|
{
|
|
if (!prepareRepoAction(RequiredDatabases::OneDestination)) {
|
|
return;
|
|
}
|
|
|
|
// make list of package names to pass to repo-remove
|
|
m_result.processedPackages.reserve(m_packageLocations.size());
|
|
for (const auto &[packageName, packageLocation, ok] : m_packageLocations) {
|
|
m_result.processedPackages.emplace_back(packageName);
|
|
}
|
|
|
|
// remove package from database file
|
|
auto repoRemoveProcess = m_buildAction->makeBuildProcess("repo-remove", m_workingDirectory + "/repo-remove.log",
|
|
std::bind(&RemovePackages::handleRepoRemoveResult, this, std::placeholders::_1, std::placeholders::_2));
|
|
repoRemoveProcess->locks().emplace_back(m_setup.locks.acquireToWrite(m_buildAction->log(), std::move(m_destinationDatabaseLockName)));
|
|
repoRemoveProcess->launch(
|
|
boost::process::start_dir(m_destinationRepoDirectory), m_repoRemovePath, m_destinationDatabaseFile, m_result.processedPackages);
|
|
m_buildAction->log()(Phrases::InfoMessage, "Invoking repo-remove within \"", m_destinationRepoDirectory, "\" for \"", m_destinationDatabaseFile,
|
|
"\", see logfile for details\n");
|
|
}
|
|
|
|
void RemovePackages::handleRepoRemoveResult(boost::process::child &&child, ProcessResult &&result)
|
|
{
|
|
CPP_UTILITIES_UNUSED(child)
|
|
if (result.errorCode) {
|
|
const auto errorCodeMessage = result.errorCode.message();
|
|
const auto &errorMessage = result.error.empty() ? errorCodeMessage : result.error;
|
|
m_result.errorMessage = "unable to remove packages: " + errorMessage;
|
|
m_buildAction->log()(Phrases::ErrorMessage, "Unable to invoke repo-remove: ", errorMessage, '\n');
|
|
} else if (result.exitCode != 0) {
|
|
m_result.errorMessage = argsToString("unable to remove package: repo-remove returned with exit code ", result.exitCode);
|
|
m_buildAction->log()(Phrases::ErrorMessage, "repo-remove invocation exited with non-zero exit code: ", result.exitCode, '\n');
|
|
} else {
|
|
movePackagesToArchive();
|
|
return;
|
|
}
|
|
m_result.failedPackages.reserve(m_result.failedPackages.size() + m_result.processedPackages.size());
|
|
for (auto &processedPackage : m_result.processedPackages) {
|
|
m_result.failedPackages.emplace_back(std::move(processedPackage), "repo-remove error");
|
|
}
|
|
m_result.processedPackages.clear();
|
|
reportResultWithData(BuildActionResult::Failure);
|
|
}
|
|
|
|
void RemovePackages::movePackagesToArchive()
|
|
{
|
|
m_buildAction->log()(Phrases::InfoMessage, "Moving packages to archive directory");
|
|
std::filesystem::path archivePath, destPath, signatureFile;
|
|
auto processedPackageIterator = m_result.processedPackages.begin();
|
|
for (const auto &[packageName, packageLocation, ok] : m_packageLocations) {
|
|
try {
|
|
archivePath = packageLocation.pathWithinRepo.parent_path() / "archive";
|
|
destPath = archivePath / packageLocation.pathWithinRepo.filename();
|
|
signatureFile = argsToString(packageLocation.pathWithinRepo, ".sig");
|
|
std::filesystem::create_directory(archivePath);
|
|
std::filesystem::rename(packageLocation.pathWithinRepo, destPath);
|
|
if (std::filesystem::exists(signatureFile)) {
|
|
std::filesystem::rename(signatureFile, argsToString(destPath, ".sig"));
|
|
}
|
|
if (packageLocation.storageLocation.empty()) {
|
|
continue;
|
|
}
|
|
// FIXME: The file at the storage location *might* still be used elsewhere. Better leave that to a repo cleanup task (to be implemented later).
|
|
archivePath = packageLocation.storageLocation.parent_path() / "archive";
|
|
destPath = archivePath / packageLocation.storageLocation.filename();
|
|
signatureFile = argsToString(packageLocation.storageLocation, ".sig");
|
|
std::filesystem::create_directory(archivePath);
|
|
std::filesystem::rename(packageLocation.storageLocation, destPath);
|
|
if (std::filesystem::exists(signatureFile)) {
|
|
std::filesystem::rename(signatureFile, argsToString(destPath, ".sig"));
|
|
}
|
|
++processedPackageIterator;
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
processedPackageIterator = m_result.processedPackages.erase(processedPackageIterator);
|
|
m_result.failedPackages.emplace_back(packageName, argsToString("unable to archive: ", e.what()));
|
|
}
|
|
}
|
|
if (m_result.failedPackages.empty()) {
|
|
reportResultWithData(BuildActionResult::Success);
|
|
return;
|
|
}
|
|
m_result.errorMessage = argsToString("failed to remove ", m_result.failedPackages.size(), " packages");
|
|
reportResultWithData(BuildActionResult::Failure);
|
|
}
|
|
|
|
MovePackages::MovePackages(ServiceSetup &setup, const std::shared_ptr<BuildAction> &buildAction)
|
|
: PackageMovementAction(setup, buildAction)
|
|
{
|
|
}
|
|
|
|
void MovePackages::run()
|
|
{
|
|
if (!prepareRepoAction(RequiredDatabases::OneSource | RequiredDatabases::OneDestination)) {
|
|
return;
|
|
}
|
|
|
|
// copy packages from the source repo to the destination repo
|
|
// make list of package names to pass to repo-add
|
|
m_result.processedPackages.reserve(m_packageLocations.size());
|
|
for (auto &[packageName, packageLocation, ok] : m_packageLocations) {
|
|
try {
|
|
const auto destPath = m_destinationRepoDirectory / packageLocation.pathWithinRepo.filename();
|
|
const auto signatureFile = std::filesystem::path(argsToString(packageLocation.pathWithinRepo, ".sig"));
|
|
if (packageLocation.storageLocation.empty()) {
|
|
std::filesystem::copy_file(packageLocation.pathWithinRepo, destPath);
|
|
if (std::filesystem::exists(signatureFile)) {
|
|
std::filesystem::copy_file(signatureFile, argsToString(destPath, ".sig"));
|
|
}
|
|
} else {
|
|
const auto symlinkTarget = std::filesystem::read_symlink(packageLocation.pathWithinRepo);
|
|
if (symlinkTarget.is_absolute()) {
|
|
ok = false;
|
|
m_result.failedPackages.emplace_back(packageName,
|
|
argsToString("unable to copy to destination repo: \"", packageLocation.pathWithinRepo,
|
|
"\" is a symlink with absolute target path (only relative target paths supported)"));
|
|
continue;
|
|
}
|
|
const auto newStorageLocation = m_destinationRepoDirectory / symlinkTarget;
|
|
const auto storageSignatureFile = std::filesystem::path(argsToString(packageLocation.storageLocation, ".sig"));
|
|
std::filesystem::create_directory(
|
|
newStorageLocation.parent_path()); // ensure the parent, e.g. the "any" directory exists; assume further parents already exist
|
|
std::filesystem::copy(packageLocation.pathWithinRepo, destPath, std::filesystem::copy_options::copy_symlinks);
|
|
std::filesystem::copy_file(packageLocation.storageLocation, newStorageLocation);
|
|
if (std::filesystem::exists(signatureFile)) {
|
|
std::filesystem::copy(signatureFile, argsToString(destPath, ".sig"), std::filesystem::copy_options::copy_symlinks);
|
|
}
|
|
if (std::filesystem::exists(storageSignatureFile)) {
|
|
std::filesystem::copy_file(storageSignatureFile, argsToString(newStorageLocation, ".sig"));
|
|
}
|
|
}
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
ok = false;
|
|
m_result.failedPackages.emplace_back(packageName, argsToString("unable to copy to destination repo: ", e.what()));
|
|
continue;
|
|
}
|
|
m_fileNames.emplace_back(packageLocation.pathWithinRepo.filename());
|
|
m_result.processedPackages.emplace_back(packageName);
|
|
}
|
|
|
|
// error-out early if not even a single package could be copied
|
|
if (m_fileNames.empty()) {
|
|
m_result.errorMessage = "none of the specified packages could be copied to the destination repo";
|
|
reportResultWithData(BuildActionResult::Failure);
|
|
return;
|
|
}
|
|
|
|
// conclude build action when both, repo-add and repo-remove have been exited and handled
|
|
const auto processSession = MultiSession<void>::create(m_setup.building.ioContext, std::bind(&MovePackages::conclude, this));
|
|
|
|
// add packages to database file of destination repo
|
|
auto repoAddProcess = m_buildAction->makeBuildProcess("repo-add", m_workingDirectory + "/repo-add.log",
|
|
std::bind(&MovePackages::handleRepoAddResult, this, processSession, std::placeholders::_1, std::placeholders::_2));
|
|
repoAddProcess->locks().emplace_back(m_setup.locks.acquireToWrite(m_buildAction->log(), std::move(m_destinationDatabaseLockName)));
|
|
repoAddProcess->launch(boost::process::start_dir(m_destinationRepoDirectory), m_repoAddPath, m_destinationDatabaseFile, m_fileNames);
|
|
|
|
// remove package from database file of source repo
|
|
auto repoRemoveProcess = m_buildAction->makeBuildProcess("repo-remove", m_workingDirectory + "/repo-remove.log",
|
|
std::bind(&MovePackages::handleRepoRemoveResult, this, processSession, std::placeholders::_1, std::placeholders::_2));
|
|
repoRemoveProcess->locks().emplace_back(m_setup.locks.acquireToWrite(m_buildAction->log(), std::move(m_sourceDatabaseLockName)));
|
|
repoRemoveProcess->launch(boost::process::start_dir(m_sourceRepoDirectory), m_repoRemovePath, m_sourceDatabaseFile, m_result.processedPackages);
|
|
|
|
m_buildAction->log()(ps(Phrases::InfoMessage), "Invoking repo-add within \"", m_destinationRepoDirectory, "\" for \"", m_destinationDatabaseFile,
|
|
"\", see logfile for details\n", ps(Phrases::InfoMessage), "Invoking repo-remove within \"", m_sourceRepoDirectory, "\" for \"",
|
|
m_sourceDatabaseFile, "\", see logfile for details\n");
|
|
}
|
|
|
|
void MovePackages::handleRepoRemoveResult(MultiSession<void>::SharedPointerType processSession, boost::process::child &&child, ProcessResult &&result)
|
|
{
|
|
// handle error
|
|
CPP_UTILITIES_UNUSED(processSession)
|
|
CPP_UTILITIES_UNUSED(child)
|
|
if (result.errorCode) {
|
|
const auto errorCodeMessage = result.errorCode.message();
|
|
const auto &errorMessage = result.error.empty() ? errorCodeMessage : result.error;
|
|
m_result.errorMessage = "unable to remove packages: " + errorMessage;
|
|
m_buildAction->log()(Phrases::ErrorMessage, "Unable to invoke repo-remove: ", errorMessage, '\n');
|
|
return;
|
|
} else if (result.exitCode != 0) {
|
|
m_result.errorMessage = argsToString("unable to remove package: repo-remove returned with exit code ", result.exitCode);
|
|
m_buildAction->log()(Phrases::ErrorMessage, "repo-remove invocation exited with non-zero exit code: ", result.exitCode, '\n');
|
|
return;
|
|
}
|
|
|
|
// remove packages from source repo
|
|
for (auto &[packageName, packageLocation, ok] : m_packageLocations) {
|
|
// ignore packages which we've couldn't even copy to destination repo
|
|
if (!ok) {
|
|
continue;
|
|
}
|
|
// delete package within source repo; leave package at storage location because some other repo might still link to it (cleanup action takes care of that)
|
|
try {
|
|
std::filesystem::remove(packageLocation.pathWithinRepo);
|
|
std::filesystem::remove(argsToString(packageLocation.pathWithinRepo, ".sig"));
|
|
} catch (const std::runtime_error &e) {
|
|
ok = false;
|
|
m_result.failedPackages.emplace_back(packageName, argsToString("unable to remove from source repo: ", e.what()));
|
|
m_result.processedPackages.erase(
|
|
std::remove(m_result.processedPackages.begin(), m_result.processedPackages.end(), packageName), m_result.processedPackages.end());
|
|
}
|
|
}
|
|
}
|
|
|
|
void MovePackages::handleRepoAddResult(MultiSession<void>::SharedPointerType processSession, boost::process::child &&child, ProcessResult &&result)
|
|
{
|
|
// handle error
|
|
CPP_UTILITIES_UNUSED(processSession)
|
|
CPP_UTILITIES_UNUSED(child)
|
|
if (result.errorCode) {
|
|
const auto errorCodeMessage = result.errorCode.message();
|
|
const auto &errorMessage = result.error.empty() ? errorCodeMessage : result.error;
|
|
m_addErrorMessage = "unable to add packages: " + errorMessage;
|
|
m_buildAction->log()(Phrases::ErrorMessage, "Unable to invoke repo-add: ", errorMessage, '\n');
|
|
return;
|
|
} else if (result.exitCode != 0) {
|
|
m_addErrorMessage = argsToString("unable to add packages: repo-add returned with exit code ", result.exitCode);
|
|
m_buildAction->log()(Phrases::ErrorMessage, "repo-add invocation exited with non-zero exit code: ", result.exitCode, '\n');
|
|
return;
|
|
}
|
|
|
|
// nothing more to do; packages have already been copied before invoking repo-add
|
|
}
|
|
|
|
void MovePackages::conclude()
|
|
{
|
|
// check for errors
|
|
const auto hasRepoRemoveError = !m_result.errorMessage.empty();
|
|
const auto hasRepoAddError = !m_addErrorMessage.empty();
|
|
std::string_view failureReason;
|
|
if (hasRepoAddError && hasRepoRemoveError) {
|
|
failureReason = "repo-add and repo-remove error";
|
|
m_result.errorMessage = argsToString(m_result.errorMessage, ',', ' ', m_addErrorMessage);
|
|
} else if (hasRepoAddError) {
|
|
failureReason = "repo-add error";
|
|
m_result.errorMessage = std::move(m_addErrorMessage);
|
|
} else if (hasRepoRemoveError) {
|
|
failureReason = "repo-remove error";
|
|
}
|
|
|
|
// report success if there are no repo-add/repo-remove errors or otherwise failed packages; otherwise report failure
|
|
if (!hasRepoAddError && !hasRepoRemoveError) {
|
|
if (m_result.errorMessage.empty() && !m_result.failedPackages.empty()) {
|
|
m_result.errorMessage = argsToString("failed to move ", m_result.failedPackages.size(), " packages");
|
|
}
|
|
reportResultWithData(m_result.failedPackages.empty() ? BuildActionResult::Success : BuildActionResult::Failure);
|
|
return;
|
|
}
|
|
|
|
// consider all packages failed if there are repo-add/repo-remove errors
|
|
m_result.failedPackages.reserve(m_result.failedPackages.size() + m_result.processedPackages.size());
|
|
for (auto &processedPackage : m_result.processedPackages) {
|
|
m_result.failedPackages.emplace_back(std::move(processedPackage), failureReason);
|
|
}
|
|
m_result.processedPackages.clear();
|
|
reportResultWithData(BuildActionResult::Failure);
|
|
}
|
|
|
|
CheckForProblems::CheckForProblems(ServiceSetup &setup, const std::shared_ptr<BuildAction> &buildAction)
|
|
: InternalBuildAction(setup, buildAction)
|
|
{
|
|
}
|
|
|
|
void CheckForProblems::run()
|
|
{
|
|
// read settings
|
|
const auto flags = static_cast<CheckForProblemsFlags>(m_buildAction->flags);
|
|
m_requirePackageSignatures = flags & CheckForProblemsFlags::RequirePackageSignatures;
|
|
auto &metaInfo = m_setup.building.metaInfo;
|
|
auto metaInfoLock = metaInfo.lockToRead();
|
|
const auto &typeInfo = metaInfo.typeInfoForId(BuildActionType::CheckForProblems);
|
|
const auto ignoreDepsSetting = typeInfo.settings[static_cast<std::size_t>(CheckForProblemsSettings::IgnoreDeps)].param;
|
|
const auto ignoreLibDepsSetting = typeInfo.settings[static_cast<std::size_t>(CheckForProblemsSettings::IgnoreLibDeps)].param;
|
|
metaInfoLock.unlock();
|
|
const auto ignoreDeps = splitStringSimple<std::unordered_set<std::string_view>>(std::string_view(findSetting(ignoreDepsSetting)), " ");
|
|
const auto ignoreLibDeps = splitStringSimple<std::unordered_set<std::string_view>>(std::string_view(findSetting(ignoreLibDepsSetting)), " ");
|
|
|
|
// initialize build action
|
|
auto configReadLock = init(BuildActionAccess::ReadConfig, RequiredDatabases::OneOrMoreDestinations, RequiredParameters::None);
|
|
if (std::holds_alternative<std::monostate>(configReadLock)) {
|
|
return;
|
|
}
|
|
|
|
auto result = std::unordered_map<std::string, std::vector<RepositoryProblem>>();
|
|
for (auto *const db : m_destinationDbs) {
|
|
// acquire locks
|
|
const auto archLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(*db));
|
|
const auto anyLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(db->name, "any"));
|
|
const auto srcLock = m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(db->name, "src"));
|
|
|
|
// check whether files exist
|
|
auto &problems = result[db->name];
|
|
try {
|
|
if (db->path.empty() || !std::filesystem::is_regular_file(db->path)) {
|
|
problems.emplace_back(RepositoryProblem{ .desc = "db file \"" % db->path + "\" is not a regular file" });
|
|
}
|
|
const auto filesPath = db->filesPath.empty() ? db->filesPathFromRegularPath() : db->filesPath;
|
|
if (filesPath.empty() || !std::filesystem::is_regular_file(filesPath)) {
|
|
problems.emplace_back(RepositoryProblem{ .desc = "files db file \"" % filesPath + "\" is not a regular file" });
|
|
}
|
|
if (db->localPkgDir.empty()) {
|
|
goto checkForUnresolvedPackages; // skip checking for presence of package if the local package directory is not configured
|
|
}
|
|
if (!std::filesystem::is_directory(db->localPkgDir)) {
|
|
problems.emplace_back(
|
|
RepositoryProblem{ .desc = "configured local package directory \"" % db->localPkgDir + "\" is not a directory" });
|
|
}
|
|
for (const auto &[pkgName, pkg] : db->packages) {
|
|
if (!pkg->packageInfo) {
|
|
problems.emplace_back(RepositoryProblem{ .desc = "no package info present", .pkg = pkgName });
|
|
continue;
|
|
}
|
|
const auto packageLocation = db->locatePackage(pkg->packageInfo->fileName);
|
|
if (!packageLocation.exists) {
|
|
problems.emplace_back(
|
|
RepositoryProblem{ .desc = "binary package \"" % pkg->packageInfo->fileName + "\" not present", .pkg = pkgName });
|
|
}
|
|
if (m_requirePackageSignatures) {
|
|
const auto signatureLocation = db->locatePackage(pkg->packageInfo->fileName + ".sig");
|
|
if (!signatureLocation.exists) {
|
|
problems.emplace_back(RepositoryProblem{
|
|
.desc = "signature file for package \"" % pkg->packageInfo->fileName + "\" not present", .pkg = pkgName });
|
|
}
|
|
}
|
|
}
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
problems.emplace_back(RepositoryProblem{ .desc = argsToString("unable to check presence of files: ", e.what()) });
|
|
}
|
|
|
|
// check for unresolved dependencies and missing libraries
|
|
checkForUnresolvedPackages:
|
|
auto unresolvedPackages = db->detectUnresolvedPackages(
|
|
m_setup.config, std::vector<std::shared_ptr<LibPkg::Package>>(), LibPkg::DependencySet(), ignoreDeps, ignoreLibDeps);
|
|
for (auto &[package, unresolvedDeps] : unresolvedPackages) {
|
|
problems.emplace_back(RepositoryProblem{ .desc = std::move(unresolvedDeps), .pkg = package->name });
|
|
}
|
|
}
|
|
|
|
const auto buildLock = m_setup.building.lockToWrite();
|
|
m_buildAction->resultData = std::move(result);
|
|
reportResult(BuildActionResult::Success);
|
|
}
|
|
|
|
CleanRepository::CleanRepository(ServiceSetup &setup, const std::shared_ptr<BuildAction> &buildAction)
|
|
: InternalBuildAction(setup, buildAction)
|
|
{
|
|
}
|
|
|
|
void CleanRepository::handleFatalError(InternalBuildAction::InitReturnType &init)
|
|
{
|
|
init = std::monostate();
|
|
m_buildAction->appendOutput(Phrases::ErrorMessage, "Cleanup aborted due to fatal errors\n");
|
|
const auto buildLock = m_setup.building.lockToWrite();
|
|
m_buildAction->resultData = std::move(m_messages);
|
|
reportResult(BuildActionResult::Failure);
|
|
}
|
|
|
|
void CleanRepository::run()
|
|
{
|
|
// initialize build action
|
|
const auto flags = static_cast<CleanRepositoryFlags>(m_buildAction->flags);
|
|
m_dryRun = flags & CleanRepositoryFlags::DryRun;
|
|
m_buildAction->appendOutput(Phrases::InfoMessage, m_dryRun ? "Preparing cleanup, dry run\n" : "Preparing cleanup\n");
|
|
auto configReadLock = init(BuildActionAccess::ReadConfig, RequiredDatabases::OneOrMoreDestinations, RequiredParameters::None);
|
|
if (std::holds_alternative<std::monostate>(configReadLock)) {
|
|
return;
|
|
}
|
|
|
|
// find relevant repository directories and acquire locks
|
|
// note: Only using a shared lock here because the cleanup isn't supposed to touch any files which actually still belong to
|
|
// the repository.
|
|
enum class RepoDirType {
|
|
New,
|
|
ArchSpecific,
|
|
Any,
|
|
Src,
|
|
};
|
|
struct RepoDir {
|
|
std::filesystem::path canonicalPath;
|
|
std::vector<std::pair<std::filesystem::path, std::string>> toArchive; // old packages not belonging to the DB anymore
|
|
std::vector<std::filesystem::path> toDelete; // non-package junk files
|
|
std::unordered_set<LibPkg::Database *> relevantDbs;
|
|
std::variant<std::monostate, SharedLoggingLock, UniqueLoggingLock> lock;
|
|
RepoDirType type = RepoDirType::New;
|
|
};
|
|
std::unordered_map<std::string, RepoDir> repoDirs;
|
|
bool fatalError = false;
|
|
const auto addAnyAndSrcDir = [this, &repoDirs](LibPkg::Database &db) {
|
|
// find the "any" directory which contains arch neutral packages which are possibly shared between databases
|
|
try {
|
|
auto anyPath = std::filesystem::canonical(db.localPkgDir + "/../any");
|
|
auto &anyDir = repoDirs[anyPath.string()];
|
|
if (anyDir.type == RepoDirType::New) {
|
|
anyDir.type = RepoDirType::Any;
|
|
anyDir.lock.emplace<SharedLoggingLock>(
|
|
m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(db.name, "any")));
|
|
anyDir.canonicalPath = std::move(anyPath);
|
|
}
|
|
anyDir.relevantDbs.emplace(&db);
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_messages.errors.emplace_back("Unable to consider \"any\" dir of \"" % db.name % "\": " + e.what());
|
|
}
|
|
// find the "src" directory which contains source package
|
|
try {
|
|
auto srcPath = std::filesystem::canonical(db.localPkgDir + "/../src");
|
|
auto &srcDir = repoDirs[srcPath.string()];
|
|
if (srcDir.type == RepoDirType::New) {
|
|
srcDir.type = RepoDirType::Src;
|
|
srcDir.lock.emplace<SharedLoggingLock>(
|
|
m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(db.name, "src")));
|
|
srcDir.canonicalPath = std::move(srcPath);
|
|
}
|
|
srcDir.relevantDbs.emplace(&db);
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_messages.errors.emplace_back("Unable to consider \"src\" dir of \"" % db.name % "\": " + e.what());
|
|
}
|
|
};
|
|
for (auto *const db : m_destinationDbs) {
|
|
if (db->localPkgDir.empty()) {
|
|
m_messages.errors.emplace_back("Unable to clean \"" % db->name + "\": no local package directory configured");
|
|
continue;
|
|
}
|
|
// find the "arch-specific" directory which contains the packages (or links to the them in the "any" directory) and the database files
|
|
auto parentPath = std::filesystem::path();
|
|
try {
|
|
auto archSpecificPath = std::filesystem::canonical(db->localPkgDir);
|
|
const auto dbFile = argsToString(archSpecificPath, '/', db->name + ".db");
|
|
const auto lastModified = LibPkg::lastModified(dbFile);
|
|
if (lastModified != db->lastUpdate) {
|
|
m_messages.errors.emplace_back("The db file's last modification (" % lastModified.toString() % ") does not match the last db update ("
|
|
% db->lastUpdate.toString()
|
|
+ ").");
|
|
fatalError = true;
|
|
}
|
|
auto &archSpecificDir = repoDirs[archSpecificPath.string()];
|
|
parentPath = archSpecificPath.parent_path();
|
|
if (archSpecificDir.type == RepoDirType::New) {
|
|
archSpecificDir.type = RepoDirType::ArchSpecific;
|
|
archSpecificDir.lock.emplace<SharedLoggingLock>(
|
|
m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(*db)));
|
|
archSpecificDir.canonicalPath = std::move(archSpecificPath);
|
|
}
|
|
archSpecificDir.relevantDbs.emplace(db);
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_messages.errors.emplace_back("Unable consider \"arch-specific\" dir of \"" % db->name % "\": " + e.what());
|
|
}
|
|
// find the "any" and "src" directory
|
|
addAnyAndSrcDir(*db);
|
|
// require the parent path to be present
|
|
if (parentPath.empty()) {
|
|
fatalError = true;
|
|
continue;
|
|
}
|
|
// find other directories next to the "arch-specific" package directory
|
|
// note: These directories belong to other databases representing the same repository but for other architectures.
|
|
try {
|
|
for (const auto &otherDir : std::filesystem::directory_iterator(parentPath)) {
|
|
if (!otherDir.is_directory() || otherDir.path() == db->localPkgDir || otherDir.path() == "any" || otherDir.path() == "src") {
|
|
continue;
|
|
}
|
|
repoDirs[otherDir.path().string()];
|
|
}
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_messages.errors.emplace_back("Unable consider find repositories next to \"" % db->name % "\": " + e.what());
|
|
fatalError = true;
|
|
}
|
|
}
|
|
if (fatalError) {
|
|
handleFatalError(configReadLock);
|
|
return;
|
|
}
|
|
|
|
// find relevant databases for repo dirs discovered in "find other directories next to …" step
|
|
auto otherDbs = std::vector<std::unique_ptr<LibPkg::Database>>();
|
|
for (auto &[dirName, dirInfo] : repoDirs) {
|
|
if (dirInfo.type != RepoDirType::New) {
|
|
continue;
|
|
}
|
|
auto dbFilePaths = std::vector<std::filesystem::path>();
|
|
try {
|
|
// find the database file
|
|
dirInfo.canonicalPath = std::filesystem::canonical(dirName);
|
|
for (const auto &repoItem : std::filesystem::directory_iterator(dirInfo.canonicalPath)) {
|
|
if (!repoItem.is_regular_file() && !repoItem.is_symlink()) {
|
|
continue;
|
|
}
|
|
if (repoItem.path().extension() == ".db") {
|
|
dbFilePaths.emplace_back(repoItem.path());
|
|
}
|
|
}
|
|
if (dbFilePaths.empty()) {
|
|
throw std::runtime_error("no *.db file present");
|
|
}
|
|
if (dbFilePaths.size() > 1) {
|
|
auto dbFileNames
|
|
#if defined(__GNUC__) && !defined(__clang__)
|
|
= dbFilePaths | std::views::transform([](const std::filesystem::path &path) { return std::string(path.filename()); });
|
|
#else
|
|
= [&dbFilePaths] {
|
|
auto res = std::vector<std::string>();
|
|
res.reserve(dbFilePaths.size());
|
|
for (const auto &path : dbFilePaths) {
|
|
res.emplace_back(path.filename());
|
|
}
|
|
return res;
|
|
}();
|
|
#endif
|
|
throw std::runtime_error(
|
|
"multiple/ambiguous *.db files present: " + joinStrings<decltype(dbFileNames), std::string>(dbFileNames, ", "));
|
|
}
|
|
// initialize temporary database object for the repository
|
|
auto &db = otherDbs.emplace_back(std::make_unique<LibPkg::Database>(dbFilePaths.front().stem(), dbFilePaths.front()));
|
|
db->arch = dirInfo.canonicalPath.stem();
|
|
db->loadPackages();
|
|
dirInfo.relevantDbs.emplace(db.get());
|
|
// acquire lock for db directory
|
|
dirInfo.lock.emplace<SharedLoggingLock>(m_setup.locks.acquireToRead(m_buildAction->log(), ServiceSetup::Locks::forDatabase(*db)));
|
|
// find the "any" and "src" directory
|
|
db->localPkgDir = dirInfo.canonicalPath.string();
|
|
addAnyAndSrcDir(*db);
|
|
// consider the repository dir arch-specific
|
|
dirInfo.type = RepoDirType::ArchSpecific;
|
|
} catch (const std::runtime_error &e) {
|
|
m_messages.errors.emplace_back("Unable read database file in repo dir \"" % dirName % "\": " + e.what());
|
|
fatalError = true;
|
|
}
|
|
}
|
|
if (fatalError) {
|
|
handleFatalError(configReadLock);
|
|
return;
|
|
}
|
|
|
|
// verify that each repo dir has at least one relevant database now
|
|
for (auto &[dirName, dirInfo] : repoDirs) {
|
|
if (dirInfo.relevantDbs.empty()) {
|
|
m_messages.errors.emplace_back("Unable to associate a database with repo dir \"" % dirName + "\".");
|
|
fatalError = true;
|
|
}
|
|
}
|
|
if (fatalError) {
|
|
handleFatalError(configReadLock);
|
|
return;
|
|
}
|
|
|
|
// flag packages no longer referenced by any database for moving it to the archive folder; flag chunk files for deletion
|
|
for (auto &[dirName, dirInfo] : repoDirs) {
|
|
try {
|
|
for (const auto &repoItem : std::filesystem::directory_iterator(dirInfo.canonicalPath)) {
|
|
// skip directories and files which are not regular files or symlinks
|
|
if (repoItem.is_directory() || repoItem.is_other()) {
|
|
continue;
|
|
}
|
|
|
|
// skip the database file itself
|
|
const auto fileName = repoItem.path().filename().string();
|
|
if (fileName.find(".db") != std::string::npos || fileName.find(".files") != std::string::npos) {
|
|
continue;
|
|
}
|
|
|
|
// delete other non-package files
|
|
if (fileName.find(".pkg") == std::string::npos && fileName.find(".src") == std::string::npos) {
|
|
dirInfo.toDelete.emplace_back(repoItem);
|
|
continue;
|
|
}
|
|
|
|
// delete orphaned signatures, otherwise skip signatures as they are handled alongside the related package
|
|
if (fileName.ends_with(".sig")) {
|
|
if (!std::filesystem::exists(std::filesystem::symlink_status(
|
|
dirInfo.canonicalPath / std::string_view(fileName.data(), fileName.data() + fileName.size() - 4)))) {
|
|
dirInfo.toDelete.emplace_back(repoItem);
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// determine package name from file name
|
|
const auto [packageName, error] = [&fileName] {
|
|
try {
|
|
const auto [name, version, arch] = LibPkg::Package::fileNameComponents(fileName);
|
|
return std::pair(std::string(name), false);
|
|
} catch (const std::runtime_error &e) {
|
|
return std::pair(std::string(e.what()), true);
|
|
}
|
|
}();
|
|
if (error) {
|
|
m_messages.warnings.emplace_back(
|
|
"Unable to parse package name of \"" % fileName % "\" (" % packageName + "). Not touching it to be safe.");
|
|
continue;
|
|
}
|
|
|
|
// check whether the file is still referenced by and relevant database and move it to archive if not
|
|
auto fileStillReferenced = false;
|
|
auto actuallyReferencedFileNames = std::vector<std::string_view>();
|
|
for (const auto *const db : dirInfo.relevantDbs) {
|
|
const auto i = db->packages.find(packageName);
|
|
if (i == db->packages.end()) {
|
|
continue;
|
|
}
|
|
const auto &pkg = i->second;
|
|
const auto &pkgInfo = pkg->packageInfo;
|
|
if (!pkgInfo || pkgInfo->fileName.empty()) {
|
|
m_messages.warnings.emplace_back(
|
|
"Database entry for package \"" % pkg->name % "\" misses the file name. Not touching \"" % fileName + "\" to be safe.");
|
|
fileStillReferenced = true;
|
|
continue;
|
|
}
|
|
if (pkgInfo->fileName == fileName) {
|
|
fileStillReferenced = true;
|
|
break;
|
|
} else {
|
|
actuallyReferencedFileNames.emplace_back(pkgInfo->fileName);
|
|
}
|
|
}
|
|
if (!fileStillReferenced) {
|
|
dirInfo.toArchive.emplace_back(
|
|
std::pair(repoItem, joinStrings<decltype(actuallyReferencedFileNames), std::string>(actuallyReferencedFileNames, ", ")));
|
|
}
|
|
}
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_messages.errors.emplace_back("Unable to iterate though repo directory \"" % dirName % "\": " + e.what());
|
|
}
|
|
}
|
|
|
|
configReadLock = std::monostate();
|
|
|
|
// do the actual file system operations
|
|
for (auto &[dirName, dirInfo] : repoDirs) {
|
|
// skip source repos for now
|
|
// note: So far we would get false-positives if pkgname does not contain pkgbase.
|
|
if (dirInfo.type == RepoDirType::Src) {
|
|
continue;
|
|
}
|
|
// delete files
|
|
std::size_t processesItems = 0;
|
|
for (auto &toDelete : dirInfo.toDelete) {
|
|
try {
|
|
if (!m_dryRun) {
|
|
const auto signatureFile = std::filesystem::path(argsToString(toDelete, ".sig"));
|
|
std::filesystem::remove(toDelete);
|
|
if (std::filesystem::exists(std::filesystem::symlink_status(signatureFile))) {
|
|
std::filesystem::remove(signatureFile);
|
|
}
|
|
}
|
|
++processesItems;
|
|
m_messages.notes.emplace_back("Deleted " + toDelete.string());
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_messages.errors.emplace_back(argsToString("Unable to delete: ", e.what()));
|
|
}
|
|
}
|
|
// archive files
|
|
const auto archiveDir = dirInfo.canonicalPath / "archive";
|
|
try {
|
|
if (!std::filesystem::is_directory(archiveDir)) {
|
|
std::filesystem::create_directory(archiveDir, dirInfo.canonicalPath);
|
|
}
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_messages.errors.emplace_back(argsToString("Unable to create archive directory: ", e.what()));
|
|
continue;
|
|
}
|
|
for (const auto &[path, referencedPath] : dirInfo.toArchive) {
|
|
try {
|
|
if (!m_dryRun) {
|
|
const auto destPath = archiveDir / path.filename();
|
|
const auto signatureFile = std::filesystem::path(argsToString(path, ".sig"));
|
|
std::filesystem::rename(path, destPath);
|
|
if (std::filesystem::exists(std::filesystem::symlink_status(signatureFile))) {
|
|
std::filesystem::rename(signatureFile, argsToString(destPath, ".sig"));
|
|
}
|
|
}
|
|
++processesItems;
|
|
m_messages.notes.emplace_back(
|
|
"Archived " % path.string() % " (current version: " % (referencedPath.empty() ? "removed"sv : referencedPath) + ")");
|
|
} catch (const std::filesystem::filesystem_error &e) {
|
|
m_messages.errors.emplace_back(argsToString("Unable to archive: ", e.what()));
|
|
}
|
|
}
|
|
m_buildAction->appendOutput(Phrases::InfoMessage, "Archived/deleted ", processesItems, " files in \"", dirName, '\"', '\n');
|
|
}
|
|
repoDirs.clear();
|
|
|
|
const auto res = m_messages.errors.empty() ? BuildActionResult::Success : BuildActionResult::Failure;
|
|
const auto buildLock = m_setup.building.lockToWrite();
|
|
m_buildAction->resultData = std::move(m_messages);
|
|
reportResult(res);
|
|
}
|
|
|
|
} // namespace LibRepoMgr
|