diff --git a/README.md b/README.md index 65e8c20..863de43 100644 --- a/README.md +++ b/README.md @@ -114,10 +114,16 @@ find $local_db_path -type d -exec chmod 775 {} \+ find $local_db_path -type f -exec chmod 664 {} \+ ``` -### Setting up the chroot for the build +### Setting up the chroot for the build with `makechrootpkg` The overall reasoning and procedure is already outlined [in the Wiki](https://wiki.archlinux.org/index.php/DeveloperWiki:Building_in_a_clean_chroot). +Alternatively, one can also skip this section and use +[`makecontainerpkg`](https://github.com/Martchus/PKGBUILDs#container-image-building-packages-within-a-container=) +instead (which should work on any environment with an appropriately configured +container runtime, e.g. a running Docker daemon and `buildservice-git` being +added to the `docker` group). + I also like to be able to build for other architectures than x86_64 so the server expects a directory layout like this: diff --git a/librepomgr/buildactions/buildactionmeta.cpp b/librepomgr/buildactions/buildactionmeta.cpp index 848c6d4..8e713cc 100644 --- a/librepomgr/buildactions/buildactionmeta.cpp +++ b/librepomgr/buildactions/buildactionmeta.cpp @@ -209,6 +209,12 @@ BuildActionMetaInfo::BuildActionMetaInfo() .desc = "Adds \"breaking\" packages only to the destination DB's staging repsitory and emits a rebuild list", .param = "auto-staging", }, + BuildActionFlagMetaInfo{ + .id = static_cast(ConductBuildFlags::UseContainer), + .name = "Use container", + .desc = "Builds within a container via `makecontainerpkg` instead of using `makechrootpkg`; eliminates the need to having pacman/devtools and the chroot setup on the build host by using docker/podman instead", + .param = "use-container", + }, }, .settings = { BuildActionSettingMetaInfo{ diff --git a/librepomgr/buildactions/buildactionmeta.h b/librepomgr/buildactions/buildactionmeta.h index 72b5156..e8eefea 100644 --- a/librepomgr/buildactions/buildactionmeta.h +++ b/librepomgr/buildactions/buildactionmeta.h @@ -83,6 +83,7 @@ enum class ConductBuildFlags : BuildActionFlagType { SaveChrootOfFailures = (1 << 1), UpdateChecksums = (1 << 2), AutoStaging = (1 << 3), + UseContainer = (1 << 4), }; enum class CheckForProblemsFlags : BuildActionFlagType { None, diff --git a/librepomgr/buildactions/buildactionprivate.h b/librepomgr/buildactions/buildactionprivate.h index d29b60f..dd7db58 100644 --- a/librepomgr/buildactions/buildactionprivate.h +++ b/librepomgr/buildactions/buildactionprivate.h @@ -592,6 +592,8 @@ private: const std::string &packageName, PackageBuildProgress &packageProgress, const std::string &buildDirectory); InvocationResult invokeMakechrootpkg( const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName, bool hasFailuresInPreviousBatches); + InvocationResult invokeMakecontainerpkg(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName, + PackageBuildProgress &packageProgress, const std::vector &makepkgFlags); void addPackageToRepo( const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName, PackageBuildProgress &packageProgress); void invokeGpg(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName, @@ -629,6 +631,7 @@ private: std::string m_chrootRootUser; boost::filesystem::path m_makePkgPath; boost::filesystem::path m_makeChrootPkgPath; + boost::filesystem::path m_makeContainerPkgPath; boost::filesystem::path m_updatePkgSumsPath; boost::filesystem::path m_repoAddPath; boost::filesystem::path m_gpgPath; @@ -644,6 +647,7 @@ private: bool m_saveChrootDirsOfFailures; bool m_updateChecksums; bool m_autoStaging; + bool m_useContainer; }; #ifdef LIBREPOMGR_DUMMY_BUILD_ACTION_ENABLED diff --git a/librepomgr/buildactions/conductbuild.cpp b/librepomgr/buildactions/conductbuild.cpp index 379dc3c..ed86ced 100644 --- a/librepomgr/buildactions/conductbuild.cpp +++ b/librepomgr/buildactions/conductbuild.cpp @@ -183,6 +183,7 @@ ConductBuild::ConductBuild(ServiceSetup &setup, const std::shared_ptrlog()(Phrases::InfoMessage, "Making source package for ", packageName, " via ", m_makePkgPath.string(), '\n', ps(Phrases::SubMessage), "build dir: ", buildDirectory, '\n'); - processSession->launch(boost::process::start_dir(buildDirectory), m_makePkgPath, "-f", "--nodeps", "--nobuild", "--source", additionalFlags); + if (m_useContainer) { + processSession->launch( + boost::process::start_dir(buildDirectory), m_makeContainerPkgPath, "--", "-f", "--nodeps", "--nobuild", "--source", additionalFlags); + } else { + processSession->launch(boost::process::start_dir(buildDirectory), m_makePkgPath, "-f", "--nodeps", "--nobuild", "--source", additionalFlags); + } return InvocationResult::Ok; } @@ -897,6 +916,33 @@ InvocationResult ConductBuild::invokeMakechrootpkg( return InvocationResult::Error; } + // determine options/variables to pass + std::vector makechrootpkgFlags, makepkgFlags; + // -> cleanup/upgrade + if (!packageProgress.skipChrootCleanup) { + makechrootpkgFlags.emplace_back("-c"); + } + if (!packageProgress.skipChrootUpgrade) { + makechrootpkgFlags.emplace_back("-u"); + } + // -> ccache support (see https://wiki.archlinux.org/index.php/Ccache#makechrootpkg) + if (!m_globalCcacheDir.empty()) { + makechrootpkgFlags.emplace_back("-d"); + makechrootpkgFlags.emplace_back(m_globalCcacheDir + "/:/ccache"); + makepkgFlags.emplace_back("CCACHE_DIR=/ccache"); + } + // -> directory for testfiles (required by tagparser and tageditor) + if (!m_globalTestFilesDir.empty()) { + makechrootpkgFlags.emplace_back("-d"); + makechrootpkgFlags.emplace_back(m_globalTestFilesDir + "/:/testfiles"); + makepkgFlags.emplace_back("TEST_FILE_PATH=/testfiles"); + } + + // invoke makecontainerpkg instead if container-flag set + if (m_useContainer) { + return invokeMakecontainerpkg(makepkgchrootSession, packageName, packageProgress, makepkgFlags); + } + // do some sanity checks with the chroot const auto chrootDir = packageProgress.chrootDirectory % "/arch-" + m_buildPreparation.targetArch; const auto buildRoot = chrootDir % '/' + m_chrootRootUser; @@ -925,28 +971,6 @@ InvocationResult ConductBuild::invokeMakechrootpkg( return InvocationResult::Error; } - // determine options/variables to pass - std::vector makechrootpkgFlags, makepkgFlags; - // -> cleanup/upgrade - if (!packageProgress.skipChrootCleanup) { - makechrootpkgFlags.emplace_back("-c"); - } - if (!packageProgress.skipChrootUpgrade) { - makechrootpkgFlags.emplace_back("-u"); - } - // -> ccache support (see https://wiki.archlinux.org/index.php/Ccache#makechrootpkg) - if (!m_globalCcacheDir.empty()) { - makechrootpkgFlags.emplace_back("-d"); - makechrootpkgFlags.emplace_back(m_globalCcacheDir + "/:/ccache"); - makepkgFlags.emplace_back("CCACHE_DIR=/ccache"); - } - // -> directory for testfiles (required by tagparser and tageditor) - if (!m_globalTestFilesDir.empty()) { - makechrootpkgFlags.emplace_back("-d"); - makechrootpkgFlags.emplace_back(m_globalTestFilesDir + "/:/testfiles"); - makepkgFlags.emplace_back("TEST_FILE_PATH=/testfiles"); - } - // prepare process session auto processSession = m_buildAction->makeBuildProcess(packageName + " build", packageProgress.buildDirectory + "/build.log", std::bind(&ConductBuild::handleMakechrootpkgErrorsAndAddPackageToRepo, this, makepkgchrootSession, std::ref(packageName), @@ -987,6 +1011,56 @@ InvocationResult ConductBuild::invokeMakechrootpkg( return InvocationResult::Ok; } +InvocationResult ConductBuild::invokeMakecontainerpkg(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, + const std::string &packageName, PackageBuildProgress &packageProgress, const std::vector &makepkgFlags) +{ + // skip initial checks as this function is only supposed to be called from within invokeMakechrootpkg + + // copy config files into chroot directory + try { + std::filesystem::copy_file(makepkgchrootSession->isStagingEnabled() ? m_pacmanStagingConfigPath : m_pacmanConfigPath, + packageProgress.buildDirectory + "/pacman.conf", std::filesystem::copy_options::overwrite_existing); + std::filesystem::copy_file( + m_makepkgConfigPath, packageProgress.buildDirectory + "/makepkg.conf", std::filesystem::copy_options::overwrite_existing); + } catch (const std::filesystem::filesystem_error &e) { + packageProgress.error = "Unable to copy config files into build directory \"" % packageProgress.buildDirectory % "\": " + e.what(); + return InvocationResult::Error; + } + + // determine options/variables to pass + // -> package cache + auto makecontainerpkgFlags = std::vector(); + if (!m_globalPackageCacheDir.empty()) { + makecontainerpkgFlags.emplace_back("-v"); + makecontainerpkgFlags.emplace_back(m_globalPackageCacheDir + "/:/var/cache/pacman/pkg/"); + } + // -> ccache support (see https://wiki.archlinux.org/index.php/Ccache#makechrootpkg) + if (!m_globalCcacheDir.empty()) { + makecontainerpkgFlags.emplace_back("-v"); + makecontainerpkgFlags.emplace_back(m_globalCcacheDir + "/:/ccache"); + } + // -> directory for testfiles (required by tagparser and tageditor) + if (!m_globalTestFilesDir.empty()) { + makecontainerpkgFlags.emplace_back("-v"); + makecontainerpkgFlags.emplace_back(m_globalTestFilesDir + "/:/testfiles"); + } + + // prepare process session + auto processSession = m_buildAction->makeBuildProcess(packageName + " build", packageProgress.buildDirectory + "/build.log", + std::bind(&ConductBuild::handleMakechrootpkgErrorsAndAddPackageToRepo, this, makepkgchrootSession, std::ref(packageName), + std::ref(packageProgress), std::placeholders::_1, std::placeholders::_2)); + processSession->registerNewDataHandler(BufferSearch("Updated version: ", "\e\n", "Starting build", + std::bind( + &ConductBuild::assignNewVersion, this, std::ref(packageName), std::ref(packageProgress), std::placeholders::_1, std::placeholders::_2))); + + // invoke makechrootpkg to build package + m_buildAction->log()(Phrases::InfoMessage, "Building ", packageName, " within container via ", m_makeContainerPkgPath.string(), '\n', + ps(Phrases::SubMessage), "build dir: ", packageProgress.buildDirectory, '\n'); + processSession->launch(boost::process::start_dir(packageProgress.buildDirectory), m_makeContainerPkgPath, makecontainerpkgFlags, "--", + makepkgFlags, packageProgress.makepkgFlags); + return InvocationResult::Ok; +} + void ConductBuild::addPackageToRepo( const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const string &packageName, PackageBuildProgress &packageProgress) { diff --git a/librepomgr/serversetup.cpp b/librepomgr/serversetup.cpp index 53aca3b..199e7b2 100644 --- a/librepomgr/serversetup.cpp +++ b/librepomgr/serversetup.cpp @@ -154,6 +154,7 @@ void ServiceSetup::BuildSetup::applyConfig(const std::multimap