Use locks to prevent multiple actions accessing the same db files and chroot dirs
This commit is contained in:
parent
bc993f1d78
commit
b7f27cb0a0
|
@ -138,7 +138,7 @@ public:
|
||||||
template <typename... ChildArgs> void launch(ChildArgs &&...childArgs);
|
template <typename... ChildArgs> void launch(ChildArgs &&...childArgs);
|
||||||
void registerWebSession(std::shared_ptr<WebAPI::Session> &&webSession);
|
void registerWebSession(std::shared_ptr<WebAPI::Session> &&webSession);
|
||||||
void registerNewDataHandler(std::function<void(BufferType, std::size_t)> &&handler);
|
void registerNewDataHandler(std::function<void(BufferType, std::size_t)> &&handler);
|
||||||
void assignLocks(AssociatedLocks &&locks = AssociatedLocks());
|
AssociatedLocks &locks();
|
||||||
bool hasExited() const;
|
bool hasExited() const;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
@ -215,9 +215,9 @@ inline BuildProcessSession::BuildProcessSession(BuildAction *buildAction, boost:
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
inline void BuildProcessSession::assignLocks(AssociatedLocks &&locks)
|
inline AssociatedLocks &BuildProcessSession::locks()
|
||||||
{
|
{
|
||||||
m_locks = std::move(locks);
|
return m_locks;
|
||||||
}
|
}
|
||||||
|
|
||||||
inline bool BuildProcessSession::hasExited() const
|
inline bool BuildProcessSession::hasExited() const
|
||||||
|
@ -341,8 +341,10 @@ private:
|
||||||
protected:
|
protected:
|
||||||
std::string m_sourceRepoDirectory;
|
std::string m_sourceRepoDirectory;
|
||||||
std::string m_sourceDatabaseFile;
|
std::string m_sourceDatabaseFile;
|
||||||
|
std::string m_sourceDatabaseLockName;
|
||||||
std::string m_destinationRepoDirectory;
|
std::string m_destinationRepoDirectory;
|
||||||
std::string m_destinationDatabaseFile;
|
std::string m_destinationDatabaseFile;
|
||||||
|
std::string m_destinationDatabaseLockName;
|
||||||
std::string m_workingDirectory;
|
std::string m_workingDirectory;
|
||||||
std::vector<std::string> m_fileNames;
|
std::vector<std::string> m_fileNames;
|
||||||
PackageMovementResult m_result;
|
PackageMovementResult m_result;
|
||||||
|
|
|
@ -901,19 +901,6 @@ InvocationResult ConductBuild::invokeMakechrootpkg(
|
||||||
return InvocationResult::Error;
|
return InvocationResult::Error;
|
||||||
}
|
}
|
||||||
|
|
||||||
// FIXME: lock the chroot directory to prevent other build tasks from using it (or does makechrootpkg already lock it?)
|
|
||||||
|
|
||||||
// copy config files into chroot directory
|
|
||||||
try {
|
|
||||||
std::filesystem::copy_file(makepkgchrootSession->isStagingEnabled() ? m_pacmanStagingConfigPath : m_pacmanConfigPath,
|
|
||||||
buildRoot + "/etc/pacman.conf", std::filesystem::copy_options::overwrite_existing);
|
|
||||||
std::filesystem::copy_file(m_makepkgConfigPath, buildRoot + "/etc/makepkg.conf", std::filesystem::copy_options::overwrite_existing);
|
|
||||||
} catch (const std::filesystem::filesystem_error &e) {
|
|
||||||
auto writeLock = lockToWrite(lock);
|
|
||||||
packageProgress.error = "Unable to configure chroot \"" % buildRoot % "\": " + e.what();
|
|
||||||
return InvocationResult::Error;
|
|
||||||
}
|
|
||||||
|
|
||||||
// determine options/variables to pass
|
// determine options/variables to pass
|
||||||
std::vector<std::string> makechrootpkgFlags, makepkgFlags;
|
std::vector<std::string> makechrootpkgFlags, makepkgFlags;
|
||||||
// -> cleanup/upgrade
|
// -> cleanup/upgrade
|
||||||
|
@ -936,16 +923,35 @@ InvocationResult ConductBuild::invokeMakechrootpkg(
|
||||||
makepkgFlags.emplace_back("TEST_FILE_PATH=/testfiles");
|
makepkgFlags.emplace_back("TEST_FILE_PATH=/testfiles");
|
||||||
}
|
}
|
||||||
|
|
||||||
// invoke makechrootpkg to build package
|
// prepare process session
|
||||||
m_buildAction->log()(Phrases::InfoMessage, "Building ", packageName, '\n');
|
|
||||||
auto processSession = m_buildAction->makeBuildProcess(packageName + " build", packageProgress.buildDirectory + "/build.log",
|
auto processSession = m_buildAction->makeBuildProcess(packageName + " build", packageProgress.buildDirectory + "/build.log",
|
||||||
std::bind(&ConductBuild::handleMakechrootpkgErrorsAndAddPackageToRepo, this, makepkgchrootSession, std::ref(packageName),
|
std::bind(&ConductBuild::handleMakechrootpkgErrorsAndAddPackageToRepo, this, makepkgchrootSession, std::ref(packageName),
|
||||||
std::ref(packageProgress), std::placeholders::_1, std::placeholders::_2));
|
std::ref(packageProgress), std::placeholders::_1, std::placeholders::_2));
|
||||||
processSession->registerNewDataHandler(BufferSearch("Updated version: ", "\e\n", "Starting build",
|
processSession->registerNewDataHandler(BufferSearch("Updated version: ", "\e\n", "Starting build",
|
||||||
std::bind(&ConductBuild::assignNewVersion, this, std::ref(packageName), std::ref(packageProgress), std::placeholders::_1)));
|
std::bind(&ConductBuild::assignNewVersion, this, std::ref(packageName), std::ref(packageProgress), std::placeholders::_1)));
|
||||||
|
processSession->registerNewDataHandler(BufferSearch("Synchronizing chroot copy", "done", std::string_view(),
|
||||||
|
[processSession = processSession.get()](std::string &&) { processSession->locks().clear(); }));
|
||||||
|
|
||||||
|
// lock the chroot directory to prevent other build tasks from using it
|
||||||
|
m_buildAction->log()(Phrases::InfoMessage, "Building ", packageName, '\n');
|
||||||
|
auto chrootLock = m_setup.locks.acquireToWrite(buildRoot);
|
||||||
|
|
||||||
|
// copy config files into chroot directory
|
||||||
|
try {
|
||||||
|
std::filesystem::copy_file(makepkgchrootSession->isStagingEnabled() ? m_pacmanStagingConfigPath : m_pacmanConfigPath,
|
||||||
|
buildRoot + "/etc/pacman.conf", std::filesystem::copy_options::overwrite_existing);
|
||||||
|
std::filesystem::copy_file(m_makepkgConfigPath, buildRoot + "/etc/makepkg.conf", std::filesystem::copy_options::overwrite_existing);
|
||||||
|
} catch (const std::filesystem::filesystem_error &e) {
|
||||||
|
auto writeLock = lockToWrite(lock);
|
||||||
|
packageProgress.error = "Unable to configure chroot \"" % buildRoot % "\": " + e.what();
|
||||||
|
return InvocationResult::Error;
|
||||||
|
}
|
||||||
|
|
||||||
|
// invoke makechrootpkg to build package
|
||||||
m_buildAction->log()(Phrases::InfoMessage, "Invoking makechrootpkg for ", packageName, " via ", m_makeChrootPkgPath.string(), '\n',
|
m_buildAction->log()(Phrases::InfoMessage, "Invoking makechrootpkg for ", packageName, " via ", m_makeChrootPkgPath.string(), '\n',
|
||||||
ps(Phrases::SubMessage), "build dir: ", packageProgress.buildDirectory, '\n', ps(Phrases::SubMessage), "chroot dir: ", chrootDir, '\n',
|
ps(Phrases::SubMessage), "build dir: ", packageProgress.buildDirectory, '\n', ps(Phrases::SubMessage), "chroot dir: ", chrootDir, '\n',
|
||||||
ps(Phrases::SubMessage), "chroot user: ", packageProgress.chrootUser, '\n');
|
ps(Phrases::SubMessage), "chroot user: ", packageProgress.chrootUser, '\n');
|
||||||
|
processSession->locks().emplace_back(std::move(chrootLock));
|
||||||
processSession->launch(boost::process::start_dir(packageProgress.buildDirectory), m_makeChrootPkgPath, makechrootpkgFlags, "-C",
|
processSession->launch(boost::process::start_dir(packageProgress.buildDirectory), m_makeChrootPkgPath, makechrootpkgFlags, "-C",
|
||||||
m_globalPackageCacheDir, "-r", chrootDir, "-l", packageProgress.chrootUser, packageProgress.makechrootpkgFlags, "--", makepkgFlags,
|
m_globalPackageCacheDir, "-r", chrootDir, "-l", packageProgress.chrootUser, packageProgress.makechrootpkgFlags, "--", makepkgFlags,
|
||||||
packageProgress.makepkgFlags);
|
packageProgress.makepkgFlags);
|
||||||
|
@ -1111,6 +1117,8 @@ void ConductBuild::addPackageToRepo(
|
||||||
auto processSession = m_buildAction->makeBuildProcess("repo-add for " + packageName, packageProgress.buildDirectory + "/repo-add.log",
|
auto processSession = m_buildAction->makeBuildProcess("repo-add for " + packageName, packageProgress.buildDirectory + "/repo-add.log",
|
||||||
std::bind(&ConductBuild::handleRepoAddErrorsAndMakeNextPackage, this, makepkgchrootSession, std::ref(packageName), std::ref(packageProgress),
|
std::bind(&ConductBuild::handleRepoAddErrorsAndMakeNextPackage, this, makepkgchrootSession, std::ref(packageName), std::ref(packageProgress),
|
||||||
std::placeholders::_1, std::placeholders::_2));
|
std::placeholders::_1, std::placeholders::_2));
|
||||||
|
processSession->locks().emplace_back(m_setup.locks.acquireToWrite(
|
||||||
|
ServiceSetup::Locks::forDatabase(needsStaging ? m_buildPreparation.targetDb : m_buildPreparation.stagingDb, m_buildPreparation.targetArch)));
|
||||||
processSession->launch(boost::process::start_dir(repoPath), m_repoAddPath, dbFilePath, binaryPackageNames);
|
processSession->launch(boost::process::start_dir(repoPath), m_repoAddPath, dbFilePath, binaryPackageNames);
|
||||||
m_buildAction->log()(Phrases::InfoMessage, "Adding ", packageName, " to repo\n", ps(Phrases::SubMessage), "repo path: ", repoPath, '\n',
|
m_buildAction->log()(Phrases::InfoMessage, "Adding ", packageName, " to repo\n", ps(Phrases::SubMessage), "repo path: ", repoPath, '\n',
|
||||||
ps(Phrases::SubMessage), "db path: ", dbFilePath, '\n', ps(Phrases::SubMessage), "package(s): ", joinStrings(binaryPackageNames), '\n');
|
ps(Phrases::SubMessage), "db path: ", dbFilePath, '\n', ps(Phrases::SubMessage), "package(s): ", joinStrings(binaryPackageNames), '\n');
|
||||||
|
|
|
@ -74,11 +74,13 @@ void ReloadDatabase::run()
|
||||||
m_buildAction->appendOutput(
|
m_buildAction->appendOutput(
|
||||||
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
||||||
try {
|
try {
|
||||||
|
auto dbFileLock = m_setup.locks.acquireToRead({ ServiceSetup::Locks::forDatabase(dbName, dbArch) });
|
||||||
const auto lastModified = LibPkg::lastModified(dbPath);
|
const auto lastModified = LibPkg::lastModified(dbPath);
|
||||||
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
|
auto dbFile = LibPkg::extractFiles(dbPath, &LibPkg::Database::isFileRelevant);
|
||||||
auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile));
|
auto packages = LibPkg::Package::fromDatabaseFile(move(dbFile));
|
||||||
auto lock = m_setup.config.lockToWrite();
|
dbFileLock.unlock();
|
||||||
auto db = m_setup.config.findDatabase(dbName, dbArch);
|
const auto configLock = m_setup.config.lockToWrite();
|
||||||
|
auto *const db = m_setup.config.findDatabase(dbName, dbArch);
|
||||||
if (!db) {
|
if (!db) {
|
||||||
m_buildAction->appendOutput(
|
m_buildAction->appendOutput(
|
||||||
Phrases::ErrorMessage, "Loaded database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
Phrases::ErrorMessage, "Loaded database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
||||||
|
|
|
@ -55,10 +55,12 @@ bool PackageMovementAction::prepareRepoAction(RequiredDatabases requiredDatabase
|
||||||
const auto *const destinationDb = *m_destinationDbs.begin();
|
const auto *const destinationDb = *m_destinationDbs.begin();
|
||||||
m_destinationRepoDirectory = destinationDb->localPkgDir;
|
m_destinationRepoDirectory = destinationDb->localPkgDir;
|
||||||
m_destinationDatabaseFile = fileName(destinationDb->path);
|
m_destinationDatabaseFile = fileName(destinationDb->path);
|
||||||
|
m_destinationDatabaseLockName = ServiceSetup::Locks::forDatabase(*destinationDb);
|
||||||
if (requiredDatabases & RequiredDatabases::OneSource) {
|
if (requiredDatabases & RequiredDatabases::OneSource) {
|
||||||
const auto *const sourceDb = *m_sourceDbs.begin();
|
const auto *const sourceDb = *m_sourceDbs.begin();
|
||||||
m_sourceRepoDirectory = sourceDb->localPkgDir;
|
m_sourceRepoDirectory = sourceDb->localPkgDir;
|
||||||
m_sourceDatabaseFile = fileName(sourceDb->path);
|
m_sourceDatabaseFile = fileName(sourceDb->path);
|
||||||
|
m_sourceDatabaseLockName = ServiceSetup::Locks::forDatabase(*sourceDb);
|
||||||
}
|
}
|
||||||
locatePackages();
|
locatePackages();
|
||||||
configReadLock = std::monostate();
|
configReadLock = std::monostate();
|
||||||
|
@ -154,6 +156,7 @@ void RemovePackages::run()
|
||||||
// remove package from database file
|
// remove package from database file
|
||||||
auto repoRemoveProcess = m_buildAction->makeBuildProcess("repo-remove", m_workingDirectory + "/repo-remove.log",
|
auto repoRemoveProcess = m_buildAction->makeBuildProcess("repo-remove", m_workingDirectory + "/repo-remove.log",
|
||||||
std::bind(&RemovePackages::handleRepoRemoveResult, this, std::placeholders::_1, std::placeholders::_2));
|
std::bind(&RemovePackages::handleRepoRemoveResult, this, std::placeholders::_1, std::placeholders::_2));
|
||||||
|
repoRemoveProcess->locks().emplace_back(m_setup.locks.acquireToWrite(m_destinationDatabaseLockName));
|
||||||
repoRemoveProcess->launch(
|
repoRemoveProcess->launch(
|
||||||
boost::process::start_dir(m_destinationRepoDirectory), m_repoRemovePath, m_destinationDatabaseFile, m_result.processedPackages);
|
boost::process::start_dir(m_destinationRepoDirectory), m_repoRemovePath, m_destinationDatabaseFile, m_result.processedPackages);
|
||||||
m_buildAction->log()(Phrases::InfoMessage, "Invoking repo-remove within \"", m_destinationRepoDirectory, "\" for \"", m_destinationDatabaseFile,
|
m_buildAction->log()(Phrases::InfoMessage, "Invoking repo-remove within \"", m_destinationRepoDirectory, "\" for \"", m_destinationDatabaseFile,
|
||||||
|
@ -270,11 +273,13 @@ void MovePackages::run()
|
||||||
// add packages to database file of destination repo
|
// add packages to database file of destination repo
|
||||||
auto repoAddProcess = m_buildAction->makeBuildProcess("repo-add", m_workingDirectory + "/repo-add.log",
|
auto repoAddProcess = m_buildAction->makeBuildProcess("repo-add", m_workingDirectory + "/repo-add.log",
|
||||||
std::bind(&MovePackages::handleRepoAddResult, this, processSession, std::placeholders::_1, std::placeholders::_2));
|
std::bind(&MovePackages::handleRepoAddResult, this, processSession, std::placeholders::_1, std::placeholders::_2));
|
||||||
|
repoAddProcess->locks().emplace_back(m_setup.locks.acquireToWrite(m_destinationDatabaseLockName));
|
||||||
repoAddProcess->launch(boost::process::start_dir(m_destinationRepoDirectory), m_repoAddPath, m_destinationDatabaseFile, m_fileNames);
|
repoAddProcess->launch(boost::process::start_dir(m_destinationRepoDirectory), m_repoAddPath, m_destinationDatabaseFile, m_fileNames);
|
||||||
|
|
||||||
// remove package from database file of source repo
|
// remove package from database file of source repo
|
||||||
auto repoRemoveProcess = m_buildAction->makeBuildProcess("repo-remove", m_workingDirectory + "/repo-remove.log",
|
auto repoRemoveProcess = m_buildAction->makeBuildProcess("repo-remove", m_workingDirectory + "/repo-remove.log",
|
||||||
std::bind(&MovePackages::handleRepoRemoveResult, this, processSession, std::placeholders::_1, std::placeholders::_2));
|
std::bind(&MovePackages::handleRepoRemoveResult, this, processSession, std::placeholders::_1, std::placeholders::_2));
|
||||||
|
repoRemoveProcess->locks().emplace_back(m_setup.locks.acquireToWrite(m_sourceDatabaseLockName));
|
||||||
repoRemoveProcess->launch(boost::process::start_dir(m_sourceRepoDirectory), m_repoRemovePath, m_sourceDatabaseFile, m_result.processedPackages);
|
repoRemoveProcess->launch(boost::process::start_dir(m_sourceRepoDirectory), m_repoRemovePath, m_sourceDatabaseFile, m_result.processedPackages);
|
||||||
|
|
||||||
m_buildAction->log()(ps(Phrases::InfoMessage), "Invoking repo-add within \"", m_destinationRepoDirectory, "\" for \"", m_destinationDatabaseFile,
|
m_buildAction->log()(ps(Phrases::InfoMessage), "Invoking repo-add within \"", m_destinationRepoDirectory, "\" for \"", m_destinationDatabaseFile,
|
||||||
|
|
|
@ -663,6 +663,16 @@ void ServiceSetup::Locks::clear()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::string ServiceSetup::Locks::forDatabase(std::string_view dbName, std::string_view dbArch)
|
||||||
|
{
|
||||||
|
return dbName % '@' + dbArch;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::string ServiceSetup::Locks::forDatabase(const LibPkg::Database &db)
|
||||||
|
{
|
||||||
|
return forDatabase(db.name, db.arch);
|
||||||
|
}
|
||||||
|
|
||||||
ServiceStatus::ServiceStatus(const ServiceSetup &setup)
|
ServiceStatus::ServiceStatus(const ServiceSetup &setup)
|
||||||
: version(applicationInfo.version)
|
: version(applicationInfo.version)
|
||||||
, config(setup.config.computeStatus())
|
, config(setup.config.computeStatus())
|
||||||
|
|
|
@ -125,6 +125,8 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
|
||||||
[[nodiscard]] std::unique_lock<std::shared_mutex> acquireToWrite(const std::string &lockName);
|
[[nodiscard]] std::unique_lock<std::shared_mutex> acquireToWrite(const std::string &lockName);
|
||||||
[[nodiscard]] std::unique_lock<std::shared_mutex> acquireToWrite(std::shared_lock<std::shared_mutex> &readLock, const std::string &lockName);
|
[[nodiscard]] std::unique_lock<std::shared_mutex> acquireToWrite(std::shared_lock<std::shared_mutex> &readLock, const std::string &lockName);
|
||||||
void clear();
|
void clear();
|
||||||
|
static std::string forDatabase(std::string_view dbName, std::string_view dbArch);
|
||||||
|
static std::string forDatabase(const LibPkg::Database &db);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::mutex m_mutex;
|
std::mutex m_mutex;
|
||||||
|
|
Loading…
Reference in New Issue