Prevent running out of memory when signing split packages
* Launch only up to 4 gpg processes at a time * Prevent `gpg: signing failed: Nicht genügend Hauptspeicher verfügbar`
This commit is contained in:
parent
56ad8ab761
commit
e4755a283b
|
@ -539,6 +539,24 @@ struct BinaryPackageInfo {
|
||||||
bool artefactAlreadyPresent = false;
|
bool artefactAlreadyPresent = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct SigningSession : public MultiSession<std::string> {
|
||||||
|
explicit SigningSession(
|
||||||
|
std::vector<BinaryPackageInfo> &&binaryPackages, const std::string *repoPath, boost::asio::io_context &ioContext, HandlerType &&handler);
|
||||||
|
std::vector<BinaryPackageInfo> binaryPackages;
|
||||||
|
std::vector<BinaryPackageInfo>::iterator currentPackage;
|
||||||
|
const std::string *repoPath;
|
||||||
|
std::mutex mutex;
|
||||||
|
};
|
||||||
|
|
||||||
|
inline SigningSession::SigningSession(std::vector<BinaryPackageInfo> &&binaryPackages, const std::string *repoPath,
|
||||||
|
boost::asio::io_context &ioContext, SigningSession::HandlerType &&handler)
|
||||||
|
: MultiSession<std::string>(ioContext, std::move(handler))
|
||||||
|
, binaryPackages(std::move(binaryPackages))
|
||||||
|
, currentPackage(this->binaryPackages.begin())
|
||||||
|
, repoPath(repoPath)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
enum class InvocationResult {
|
enum class InvocationResult {
|
||||||
Ok,
|
Ok,
|
||||||
Skipped,
|
Skipped,
|
||||||
|
@ -579,6 +597,7 @@ private:
|
||||||
const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName, PackageBuildProgress &packageProgress);
|
const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName, PackageBuildProgress &packageProgress);
|
||||||
void invokeGpg(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName,
|
void invokeGpg(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName,
|
||||||
PackageBuildProgress &packageProgress, std::vector<BinaryPackageInfo> &&binaryPackages, BuildResult &&buildResult);
|
PackageBuildProgress &packageProgress, std::vector<BinaryPackageInfo> &&binaryPackages, BuildResult &&buildResult);
|
||||||
|
void invokeGpg(const std::shared_ptr<SigningSession> &signingSession, const std::string &packageName, PackageBuildProgress &packageProgress);
|
||||||
void invokeRepoAdd(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName,
|
void invokeRepoAdd(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const std::string &packageName,
|
||||||
PackageBuildProgress &packageProgress, BuildResult &&buildResult);
|
PackageBuildProgress &packageProgress, BuildResult &&buildResult);
|
||||||
void checkDownloadErrorsAndMakePackages(BatchProcessingSession::ContainerType &&failedPackages);
|
void checkDownloadErrorsAndMakePackages(BatchProcessingSession::ContainerType &&failedPackages);
|
||||||
|
|
|
@ -1138,62 +1138,79 @@ void ConductBuild::addPackageToRepo(
|
||||||
void ConductBuild::invokeGpg(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const string &packageName,
|
void ConductBuild::invokeGpg(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const string &packageName,
|
||||||
PackageBuildProgress &packageProgress, std::vector<BinaryPackageInfo> &&binaryPackages, BuildResult &&buildResult)
|
PackageBuildProgress &packageProgress, std::vector<BinaryPackageInfo> &&binaryPackages, BuildResult &&buildResult)
|
||||||
{
|
{
|
||||||
const auto *const repoPath = buildResult.repoPath;
|
const auto signingSession = std::make_shared<SigningSession>(std::move(binaryPackages), buildResult.repoPath, m_setup.building.ioContext,
|
||||||
auto signingSession = MultiSession<std::string>::create(m_setup.building.ioContext,
|
|
||||||
[this, makepkgchrootSession, &packageName, &packageProgress, buildResult = std::move(buildResult)](
|
[this, makepkgchrootSession, &packageName, &packageProgress, buildResult = std::move(buildResult)](
|
||||||
MultiSession<std::string>::ContainerType &&failedPackages) mutable {
|
MultiSession<std::string>::ContainerType &&failedPackages) mutable {
|
||||||
checkGpgErrorsAndContinueAddingPackagesToRepo(
|
checkGpgErrorsAndContinueAddingPackagesToRepo(
|
||||||
makepkgchrootSession, packageName, packageProgress, std::move(buildResult), std::move(failedPackages));
|
makepkgchrootSession, packageName, packageProgress, std::move(buildResult), std::move(failedPackages));
|
||||||
});
|
});
|
||||||
for (const auto &binaryPackage : binaryPackages) {
|
constexpr auto gpgParallelLimit = 4;
|
||||||
auto processSession
|
const auto lock = std::unique_lock<std::mutex>(signingSession->mutex);
|
||||||
= m_buildAction->makeBuildProcess("gpg for " + binaryPackage.name, packageProgress.buildDirectory % "/gpg-" % binaryPackage.name + ".log",
|
for (auto i = 0; i != gpgParallelLimit && signingSession->currentPackage != signingSession->binaryPackages.end();
|
||||||
[this, signingSession, &packageProgress, repoPath, isAny = binaryPackage.isAny, binaryPackageName = binaryPackage.fileName](
|
++i, ++signingSession->currentPackage) {
|
||||||
boost::process::child &&child, ProcessResult &&result) mutable {
|
invokeGpg(signingSession, packageName, packageProgress);
|
||||||
if (result.errorCode) {
|
|
||||||
// check for invocation error
|
|
||||||
m_buildAction->log()(
|
|
||||||
Phrases::ErrorMessage, "Unable to invoke gpg for ", binaryPackageName, ": ", result.errorCode.message(), '\n');
|
|
||||||
} else if (child.exit_code() != 0) {
|
|
||||||
// check for bad exit code
|
|
||||||
m_buildAction->log()(Phrases::ErrorMessage, "gpg invocation for ", binaryPackageName,
|
|
||||||
" exited with non-zero exit code: ", child.exit_code(), '\n');
|
|
||||||
} else {
|
|
||||||
// move signature to repository
|
|
||||||
try {
|
|
||||||
const auto buildDirSignaturePath
|
|
||||||
= std::filesystem::path(argsToString(packageProgress.buildDirectory, '/', binaryPackageName, ".sig"));
|
|
||||||
if (!std::filesystem::exists(buildDirSignaturePath)) {
|
|
||||||
m_buildAction->log()(Phrases::ErrorMessage, "Signature of \"", binaryPackageName,
|
|
||||||
"\" could not be created: ", buildDirSignaturePath, " does not exist after invoking gpg\n");
|
|
||||||
} else if (!isAny) {
|
|
||||||
std::filesystem::copy(buildDirSignaturePath, *repoPath % '/' % binaryPackageName + ".sig",
|
|
||||||
std::filesystem::copy_options::update_existing);
|
|
||||||
return;
|
|
||||||
} else {
|
|
||||||
std::filesystem::copy(buildDirSignaturePath, argsToString(repoPath, "/../any/", binaryPackageName, ".sig"),
|
|
||||||
std::filesystem::copy_options::update_existing);
|
|
||||||
const auto symlink = std::filesystem::path(argsToString(repoPath, '/', binaryPackageName, ".sig"));
|
|
||||||
if (std::filesystem::exists(symlink) && !std::filesystem::is_symlink(symlink)) {
|
|
||||||
std::filesystem::remove(symlink);
|
|
||||||
}
|
|
||||||
std::filesystem::create_symlink("../any/" % binaryPackageName + ".sig", symlink);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
} catch (const std::filesystem::filesystem_error &e) {
|
|
||||||
m_buildAction->log()(
|
|
||||||
Phrases::ErrorMessage, "Unable to copy signature of \"", binaryPackageName, "\" to repository: ", e.what(), '\n');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// consider the package failed
|
|
||||||
signingSession->addResponse(std::move(binaryPackageName));
|
|
||||||
});
|
|
||||||
processSession->launch(boost::process::start_dir(packageProgress.buildDirectory), m_gpgPath, "--detach-sign", "--yes", "--use-agent",
|
|
||||||
"--no-armor", "-u", m_gpgKey, binaryPackage.fileName);
|
|
||||||
m_buildAction->log()(Phrases::InfoMessage, "Signing ", binaryPackage.fileName, '\n');
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void ConductBuild::invokeGpg(
|
||||||
|
const std::shared_ptr<SigningSession> &signingSession, const std::string &packageName, PackageBuildProgress &packageProgress)
|
||||||
|
{
|
||||||
|
const auto &binaryPackage = *signingSession->currentPackage;
|
||||||
|
auto processSession = m_buildAction->makeBuildProcess("gpg for " + binaryPackage.name,
|
||||||
|
packageProgress.buildDirectory % "/gpg-" % binaryPackage.name + ".log",
|
||||||
|
[this, signingSession, &packageName, &packageProgress, isAny = binaryPackage.isAny, binaryPackageName = binaryPackage.fileName](
|
||||||
|
boost::process::child &&child, ProcessResult &&result) mutable {
|
||||||
|
// make the next gpg invocation
|
||||||
|
if (const auto lock = std::unique_lock<std::mutex>(signingSession->mutex);
|
||||||
|
signingSession->currentPackage != signingSession->binaryPackages.end()
|
||||||
|
&& ++signingSession->currentPackage != signingSession->binaryPackages.end()) {
|
||||||
|
invokeGpg(signingSession, packageName, packageProgress);
|
||||||
|
}
|
||||||
|
|
||||||
|
// handle results of gpg invocation
|
||||||
|
if (result.errorCode) {
|
||||||
|
// check for invocation error
|
||||||
|
m_buildAction->log()(Phrases::ErrorMessage, "Unable to invoke gpg for ", binaryPackageName, ": ", result.errorCode.message(), '\n');
|
||||||
|
} else if (child.exit_code() != 0) {
|
||||||
|
// check for bad exit code
|
||||||
|
m_buildAction->log()(
|
||||||
|
Phrases::ErrorMessage, "gpg invocation for ", binaryPackageName, " exited with non-zero exit code: ", child.exit_code(), '\n');
|
||||||
|
} else {
|
||||||
|
// move signature to repository
|
||||||
|
try {
|
||||||
|
const auto buildDirSignaturePath
|
||||||
|
= std::filesystem::path(argsToString(packageProgress.buildDirectory, '/', binaryPackageName, ".sig"));
|
||||||
|
if (!std::filesystem::exists(buildDirSignaturePath)) {
|
||||||
|
m_buildAction->log()(Phrases::ErrorMessage, "Signature of \"", binaryPackageName,
|
||||||
|
"\" could not be created: ", buildDirSignaturePath, " does not exist after invoking gpg\n");
|
||||||
|
} else if (!isAny) {
|
||||||
|
std::filesystem::copy(buildDirSignaturePath, *signingSession->repoPath % '/' % binaryPackageName + ".sig",
|
||||||
|
std::filesystem::copy_options::update_existing);
|
||||||
|
return;
|
||||||
|
} else {
|
||||||
|
std::filesystem::copy(buildDirSignaturePath, argsToString(signingSession->repoPath, "/../any/", binaryPackageName, ".sig"),
|
||||||
|
std::filesystem::copy_options::update_existing);
|
||||||
|
const auto symlink = std::filesystem::path(argsToString(signingSession->repoPath, '/', binaryPackageName, ".sig"));
|
||||||
|
if (std::filesystem::exists(symlink) && !std::filesystem::is_symlink(symlink)) {
|
||||||
|
std::filesystem::remove(symlink);
|
||||||
|
}
|
||||||
|
std::filesystem::create_symlink("../any/" % binaryPackageName + ".sig", symlink);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
} catch (const std::filesystem::filesystem_error &e) {
|
||||||
|
m_buildAction->log()(
|
||||||
|
Phrases::ErrorMessage, "Unable to copy signature of \"", binaryPackageName, "\" to repository: ", e.what(), '\n');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// consider the package failed
|
||||||
|
signingSession->addResponse(std::move(binaryPackageName));
|
||||||
|
});
|
||||||
|
processSession->launch(boost::process::start_dir(packageProgress.buildDirectory), m_gpgPath, "--detach-sign", "--yes", "--use-agent",
|
||||||
|
"--no-armor", "-u", m_gpgKey, binaryPackage.fileName);
|
||||||
|
m_buildAction->log()(Phrases::InfoMessage, "Signing ", binaryPackage.fileName, '\n');
|
||||||
|
}
|
||||||
|
|
||||||
void ConductBuild::invokeRepoAdd(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const string &packageName,
|
void ConductBuild::invokeRepoAdd(const BatchProcessingSession::SharedPointerType &makepkgchrootSession, const string &packageName,
|
||||||
PackageBuildProgress &packageProgress, BuildResult &&buildResult)
|
PackageBuildProgress &packageProgress, BuildResult &&buildResult)
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in New Issue