From 9b6be51dd3e2bbd8d02a2fe4a25042d34d659b12 Mon Sep 17 00:00:00 2001 From: Martchus Date: Fri, 4 Mar 2022 21:51:25 +0100 Subject: [PATCH] Allow reloading library dependencies of specific packages --- librepomgr/buildactions/buildactionmeta.cpp | 2 +- librepomgr/buildactions/buildactionprivate.h | 5 + .../reloadlibrarydependencies.cpp | 194 ++++++++++-------- 3 files changed, 114 insertions(+), 87 deletions(-) diff --git a/librepomgr/buildactions/buildactionmeta.cpp b/librepomgr/buildactions/buildactionmeta.cpp index f04177a..b46d646 100644 --- a/librepomgr/buildactions/buildactionmeta.cpp +++ b/librepomgr/buildactions/buildactionmeta.cpp @@ -128,7 +128,7 @@ BuildActionMetaInfo::BuildActionMetaInfo() .directory = false, .sourceDb = false, .destinationDb = true, - .packageNames = false, + .packageNames = true, }, BuildActionTypeMetaInfo{ .id = BuildActionType::PrepareBuild, diff --git a/librepomgr/buildactions/buildactionprivate.h b/librepomgr/buildactions/buildactionprivate.h index 25baca0..d29b60f 100644 --- a/librepomgr/buildactions/buildactionprivate.h +++ b/librepomgr/buildactions/buildactionprivate.h @@ -409,6 +409,9 @@ private: std::vector packages; }; + bool addRelevantPackage(LibPkg::StorageID packageID, const std::shared_ptr &package, const LibPkg::Database *db, + bool isDestinationDb, DatabaseToConsider &relevantDbInfo, + std::unordered_map> &relevantPkgs); void downloadPackagesFromMirror(); void loadPackageInfoFromContents(); void conclude(); @@ -419,6 +422,8 @@ private: std::atomic_size_t m_remainingPackages; WebClient::PackageCachingDataForSession m_cachingData; std::uint64_t m_packageDownloadSizeLimit; + std::string m_cacheDir; + bool m_force = false; }; struct LIBREPOMGR_EXPORT CheckForProblems : public InternalBuildAction { diff --git a/librepomgr/buildactions/reloadlibrarydependencies.cpp b/librepomgr/buildactions/reloadlibrarydependencies.cpp index 42d5808..94dc623 100644 --- a/librepomgr/buildactions/reloadlibrarydependencies.cpp +++ b/librepomgr/buildactions/reloadlibrarydependencies.cpp @@ -27,7 +27,7 @@ void ReloadLibraryDependencies::run() { // read configuration const auto flags = static_cast(m_buildAction->flags); - const auto force = flags & ReloadLibraryDependenciesFlags::ForceReload; + m_force = flags & ReloadLibraryDependenciesFlags::ForceReload; const auto skipDependencies = flags & ReloadLibraryDependenciesFlags::SkipDependencies; auto &metaInfo = m_setup.building.metaInfo; auto metaInfoLock = metaInfo.lockToRead(); @@ -47,14 +47,14 @@ void ReloadLibraryDependencies::run() // initialize m_remainingPackages = 0; - auto configReadLock = init(BuildActionAccess::ReadConfig, RequiredDatabases::MaybeDestination, RequiredParameters::None); + auto configReadLock = init(BuildActionAccess::ReadConfig, RequiredDatabases::MaybeDestination, RequiredParameters::MaybePackages); if (std::holds_alternative(configReadLock)) { return; } // use cache directory from global configuration auto buildLock = m_setup.building.lockToRead(); - const auto cacheDir = m_setup.building.packageCacheDir + '/'; + m_cacheDir = m_setup.building.packageCacheDir + '/'; m_packageDownloadSizeLimit = m_setup.building.packageDownloadSizeLimit; buildLock.unlock(); @@ -104,96 +104,37 @@ void ReloadLibraryDependencies::run() for (auto *const db : relevantDbs) { const auto isDestinationDb = m_destinationDbs.empty() || m_destinationDbs.find(db) != m_destinationDbs.end(); auto &relevantDbInfo = m_relevantPackagesByDatabase.emplace_back(DatabaseToConsider{ .name = db->name, .arch = db->arch }); - db->allPackages([&](LibPkg::StorageID packageID, const std::shared_ptr &package) { - // allow aborting the build action - if (reportAbortedIfAborted()) { - return true; - } - // skip if package should be excluded - if (!packageExcludeRegexValue.empty() && std::regex_match(package->name, packageExcludeRegex)) { - m_messages.notes.emplace_back(db->name % '/' % package->name + ": matches exclude regex"); - return false; - } - // skip if the package info is missing (we need the binary package's file name here) - const auto &packageInfo = package->packageInfo; - if (!packageInfo) { - m_messages.errors.emplace_back(db->name % '/' % package->name + ": no package info"); - return false; - } - // skip the package if it is not part of the destination DB or required by a package of the destination DB - if (!isDestinationDb && relevantPkgs.find(packageID) == relevantPkgs.end()) { - if (m_skippingNote.tellp()) { - m_skippingNote << ", "; + if (m_buildAction->packageNames.empty()) { + db->allPackages([&](LibPkg::StorageID packageID, const std::shared_ptr &package) { + // allow aborting the build action + if (reportAbortedIfAborted()) { + return true; } - m_skippingNote << db->name << '/' << package->name; - return false; - } - // find the package on disk; otherwise add an URL to download it from the configured mirror - std::string path, url, cachePath; - std::error_code ec; - const auto &fileName = packageInfo->fileName; - const auto &arch = packageInfo->arch; - if (!db->localPkgDir.empty()) { - path = db->localPkgDir % '/' + fileName; - } else if (std::filesystem::file_size(cachePath = cacheDir + fileName, ec) && !ec) { - path = std::move(cachePath); - } else if (std::filesystem::file_size(cachePath = cacheDir % arch % '/' + fileName, ec) && !ec) { - path = std::move(cachePath); - } else { - for (const auto &possibleCachePath : m_setup.config.packageCacheDirs) { - if (std::filesystem::file_size(path = possibleCachePath % '/' + fileName, ec) && !ec) { - break; - } - path.clear(); - } - } - if (path.empty() && !db->mirrors.empty()) { - const auto &mirror = db->mirrors.front(); // just use the first mirror for now - if (startsWith(mirror, "file:")) { - std::error_code ecCanonical; - const auto canonPath = std::filesystem::canonical( - argsToString(std::string_view(mirror.data() + 5, mirror.size() - 5), '/', fileName), ecCanonical); - if (!ecCanonical) { - path = canonPath.string(); - } - } else { - path = std::move(cachePath); - url = mirror % (endsWith(mirror, "/") ? std::string() : "/") + fileName; - } - } - if (path.empty()) { - m_messages.errors.emplace_back(db->name % '/' % package->name + ": binary package not found and no mirror configured"); - return false; - } - // skip if the package info has already been loaded from package contents and the present binary package is not newer - auto lastModified = DateTime(); - if (url.empty()) { - lastModified = LibPkg::lastModified(path); - if (!force && package->origin == LibPkg::PackageOrigin::PackageContents && package->timestamp >= lastModified) { - m_messages.notes.emplace_back(db->name % '/' % package->name % ": skipping because \"" % path % "\" is newer (" - % package->timestamp.toString() % " >= " % lastModified.toString() - + ")\n"); + // skip if package should be excluded + if (!packageExcludeRegexValue.empty() && std::regex_match(package->name, packageExcludeRegex)) { + m_messages.notes.emplace_back(db->name % '/' % package->name + ": matches exclude regex"); return false; } + return addRelevantPackage(packageID, package, db, isDestinationDb, relevantDbInfo, relevantPkgs); + }); + } else { + for (const auto &packageDenotation : m_buildAction->packageNames) { + const auto [dbName, dbArch, packageName] = LibPkg::Config::parsePackageDenotation(packageDenotation); + if ((!dbName.empty() && dbName != db->name) || (!dbArch.empty() && dbArch != db->arch)) { + continue; + } + auto [packageID, package] = db->findPackageWithID(std::string(packageName)); // FIXME: allow passing string_view here + if (!package) { + continue; + } + addRelevantPackage(packageID, package, db, isDestinationDb, relevantDbInfo, relevantPkgs); } - // add the full path to the binary package to relevant packages - auto &relevantPkg = relevantDbInfo.packages.emplace_back( - PackageToConsider{ .path = std::move(path), .url = std::move(url), .lastModified = lastModified }); - // create a temporary package object to hold the info parsed from the .PKGINFO file - relevantPkg.info.name = package->name; - // -> assign certain fields which are used by addDepsAndProvidesFromOtherPackage() to check whether the packages are matching - relevantPkg.info.version = package->version; - relevantPkg.info.packageInfo = std::make_unique(); - relevantPkg.info.packageInfo->buildDate = package->packageInfo->buildDate; - // -> gather source info such as make and check dependencies as well - relevantPkg.info.sourceInfo = std::make_shared(); - ++m_remainingPackages; - return false; - }); + } if (reportAbortedIfAborted()) { return; } } + configReadLock = std::monostate{}; m_buildAction->appendOutput(Phrases::SubMessage, "Found ", m_remainingPackages.load(), "\n"); @@ -214,6 +155,87 @@ void ReloadLibraryDependencies::run() downloadPackagesFromMirror(); } +bool ReloadLibraryDependencies::addRelevantPackage(LibPkg::StorageID packageID, const std::shared_ptr &package, + const LibPkg::Database *db, bool isDestinationDb, DatabaseToConsider &relevantDbInfo, + std::unordered_map> &relevantPkgs) +{ + // skip if the package info is missing (we need the binary package's file name here) + const auto &packageInfo = package->packageInfo; + if (!packageInfo) { + m_messages.errors.emplace_back(db->name % '/' % package->name + ": no package info"); + return false; + } + // skip the package if it is not part of the destination DB or required by a package of the destination DB + if (!isDestinationDb && relevantPkgs.find(packageID) == relevantPkgs.end()) { + if (m_skippingNote.tellp()) { + m_skippingNote << ", "; + } + m_skippingNote << db->name << '/' << package->name; + return false; + } + // find the package on disk; otherwise add an URL to download it from the configured mirror + std::string path, url, cachePath; + std::error_code ec; + const auto &fileName = packageInfo->fileName; + const auto &arch = packageInfo->arch; + if (!db->localPkgDir.empty()) { + path = db->localPkgDir % '/' + fileName; + } else if (std::filesystem::file_size(cachePath = m_cacheDir + fileName, ec) && !ec) { + path = std::move(cachePath); + } else if (std::filesystem::file_size(cachePath = m_cacheDir % arch % '/' + fileName, ec) && !ec) { + path = std::move(cachePath); + } else { + for (const auto &possibleCachePath : m_setup.config.packageCacheDirs) { + if (std::filesystem::file_size(path = possibleCachePath % '/' + fileName, ec) && !ec) { + break; + } + path.clear(); + } + } + if (path.empty() && !db->mirrors.empty()) { + const auto &mirror = db->mirrors.front(); // just use the first mirror for now + if (startsWith(mirror, "file:")) { + std::error_code ecCanonical; + const auto canonPath + = std::filesystem::canonical(argsToString(std::string_view(mirror.data() + 5, mirror.size() - 5), '/', fileName), ecCanonical); + if (!ecCanonical) { + path = canonPath.string(); + } + } else { + path = std::move(cachePath); + url = mirror % (endsWith(mirror, "/") ? std::string() : "/") + fileName; + } + } + if (path.empty()) { + m_messages.errors.emplace_back(db->name % '/' % package->name + ": binary package not found and no mirror configured"); + return false; + } + // skip if the package info has already been loaded from package contents and the present binary package is not newer + auto lastModified = DateTime(); + if (url.empty()) { + lastModified = LibPkg::lastModified(path); + if (!m_force && package->origin == LibPkg::PackageOrigin::PackageContents && package->timestamp >= lastModified) { + m_messages.notes.emplace_back(db->name % '/' % package->name % ": skipping because \"" % path % "\" is newer (" + % package->timestamp.toString() % " >= " % lastModified.toString() + + ")\n"); + return false; + } + } + // add the full path to the binary package to relevant packages + auto &relevantPkg + = relevantDbInfo.packages.emplace_back(PackageToConsider{ .path = std::move(path), .url = std::move(url), .lastModified = lastModified }); + // create a temporary package object to hold the info parsed from the .PKGINFO file + relevantPkg.info.name = package->name; + // -> assign certain fields which are used by addDepsAndProvidesFromOtherPackage() to check whether the packages are matching + relevantPkg.info.version = package->version; + relevantPkg.info.packageInfo = std::make_unique(); + relevantPkg.info.packageInfo->buildDate = package->packageInfo->buildDate; + // -> gather source info such as make and check dependencies as well + relevantPkg.info.sourceInfo = std::make_shared(); + ++m_remainingPackages; + return false; +} + void LibRepoMgr::ReloadLibraryDependencies::downloadPackagesFromMirror() { // prepare caching data @@ -306,7 +328,7 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents() currentPkg.info.addInfoFromPkgInfoFile(file.content); return; } - currentPkg.info.addDepsAndProvidesFromContainedFile(file, dllsReferencedByImportLibs); + currentPkg.info.addDepsAndProvidesFromContainedFile(directoryPath, file, dllsReferencedByImportLibs); }, [¤tPkg](std::string &&directoryPath) { if (directoryPath.empty()) {