Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

lmdb: Use lmdb to store build actions

This commit is contained in:
Martchus 2022-02-05 22:09:52 +01:00
parent 3c4f81bd55
commit 557fd1a738
20 changed files with 597 additions and 641 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
3rdparty/lmdb-safe vendored

@ -1 +1 @@
Subproject commit fbcdc05e9c389ec93d5eb05e92ece188aeb48239
Subproject commit 1517bdce2931353f6d929fe3ad9bd03ce10a66ab

11
README.md
View File

@ -6,8 +6,8 @@ repositories. It is built on top of the official tools provided by the `pacman`
At this point the project is rather raw and there are many things left to implement and
to improve (checkout the TODOs section below). Currently builds are exclusively triggered
manually (although the API brings the possibility for automation), scalability is limited
through the partial use of an in-memory database. Builds are only conducted on one host.
manually (although the API brings the possibility for automation) and are only conducted
on one host (although one can simply setup the build service on multiple hosts).
On the upside, this project can easily be used to together with other build scripts. It
doesn't care if your repository's DB file is updated by another application; just be sure
@ -333,11 +333,8 @@ editing the presets JSON file (e.g. `/etc/buildservice-git/presets.json` in the
only `pkgrel` changes or when building from VCS sources or when some sources just remain the same).
## TODOs and further ideas for improvement
* [ ] Use persistent, non-in-memory database (e.g. lmdb or leveldb) to improve scalability
* [x] Use db for packages
* [ ] Use db for build actions
* [ ] Allow triggering tasks automatically/periodically
* [ ] Allow to run `makechrootpkg` on a remote host (e.g. via SSH) to work can be spread across multiple hosts
* [ ] Allow to run `makechrootpkg` on a remote host (e.g. via SSH) so work can be spread across multiple hosts
* [ ] More advanced search options
* [ ] Refresh build action details on the web UI automatically while an action is pending
* [ ] Fix permission error when stopping a process
@ -351,7 +348,7 @@ editing the presets JSON file (e.g. `/etc/buildservice-git/presets.json` in the
* Start process producing lots of output very fast
* Let different clients connect and disconnect fast
* [ ] Improve test coverage
* [ ] Add fancy graphs for dependencies on the web UI
* [ ] Add fancy graphs for dependencies of build actions on the web UI
* [ ] Include `xterm.js` via JavaScript modules (blocked by https://github.com/xtermjs/xterm.js/issues/2878)
## Build instructions and dependencies

1
libpkg/CMakeLists.txt
View File

@ -19,6 +19,7 @@ set(SRC_FILES
data/database.cpp
data/config.cpp
data/lockable.cpp
data/storagegeneric.h
data/storageprivate.h
data/storage.cpp
algo/search.cpp

162
libpkg/data/storagegeneric.h Normal file
View File

@ -0,0 +1,162 @@
#ifndef LIBPKG_DATA_STORAGE_GENERIC_H
#define LIBPKG_DATA_STORAGE_GENERIC_H
#include "../lmdb-safe/lmdb-reflective.hh"
#include "../lmdb-safe/lmdb-safe.hh"
#include "../lmdb-safe/lmdb-typed.hh"
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index_container.hpp>
#include <memory>
#include <mutex>
namespace LibPkg {
using StorageID = std::uint32_t;
template <typename StorageType, typename EntryType> struct StorageCacheRef {
using Storage = StorageType;
explicit StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr<EntryType> &entry);
explicit StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName);
bool operator==(const StorageCacheRef &other) const;
const StorageType *relatedStorage = nullptr;
const std::string *entryName;
};
template <typename StorageType, typename EntryType>
inline StorageCacheRef<StorageType, EntryType>::StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr<EntryType> &entry)
: relatedStorage(&relatedStorage)
, entryName(&entry->name)
{
}
template <typename StorageType, typename EntryType>
inline StorageCacheRef<StorageType, EntryType>::StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName)
: relatedStorage(&relatedStorage)
, entryName(&entryName)
{
}
template <typename StorageType, typename EntryType>
inline bool StorageCacheRef<StorageType, EntryType>::operator==(const StorageCacheRef<StorageType, EntryType> &other) const
{
return relatedStorage == other.relatedStorage && *entryName == *other.entryName;
}
template <typename StorageRefType, typename EntryType> struct StorageCacheEntry {
using Ref = StorageRefType;
using Entry = EntryType;
using Storage = typename Ref::Storage;
explicit StorageCacheEntry(const StorageRefType &ref, StorageID id);
StorageRefType ref;
StorageID id;
std::shared_ptr<EntryType> entry;
};
template <typename StorageRefType, typename EntryType>
inline StorageCacheEntry<StorageRefType, EntryType>::StorageCacheEntry(const StorageRefType &ref, StorageID id)
: ref(ref)
, id(id)
{
}
template <typename StorageEntryType> struct StorageEntryByID {
struct result_type {
StorageID id = 0;
const typename StorageEntryType::Storage *storage = nullptr;
bool operator==(const result_type &other) const
{
return id == other.id && storage == other.storage;
}
};
result_type operator()(const StorageEntryType &storageEntry) const
{
return result_type{ storageEntry.id, storageEntry.ref.relatedStorage };
}
};
template <typename StorageEntryType> class StorageCacheEntries {
public:
using Ref = typename StorageEntryType::Ref;
using Entry = typename StorageEntryType::Entry;
using Storage = typename StorageEntryType::Storage;
using StorageEntry = StorageEntryType;
using ByID = StorageEntryByID<StorageEntry>;
using EntryList = boost::multi_index::multi_index_container<StorageEntry,
boost::multi_index::indexed_by<boost::multi_index::sequenced<>,
boost::multi_index::hashed_unique<boost::multi_index::tag<typename ByID::result_type>, ByID>,
boost::multi_index::hashed_unique<boost::multi_index::tag<Ref>, BOOST_MULTI_INDEX_MEMBER(StorageEntryType, Ref, ref)>>>;
using iterator = typename EntryList::iterator;
explicit StorageCacheEntries(std::size_t limit = 1000);
template <typename IndexType> StorageEntry *find(const IndexType &ref);
StorageEntry &insert(StorageEntry &&entry);
std::size_t erase(const Ref &ref);
std::size_t clear(const Storage &storage);
iterator begin();
iterator end();
void setLimit(std::size_t limit);
private:
EntryList m_entries;
std::size_t m_limit;
};
template <typename StorageEntryType>
inline StorageCacheEntries<StorageEntryType>::StorageCacheEntries(std::size_t limit)
: m_limit(limit)
{
}
template <typename StorageEntryType> inline std::size_t StorageCacheEntries<StorageEntryType>::erase(const Ref &ref)
{
return m_entries.template get<typename StorageEntryType::Ref>().erase(ref);
}
template <typename StorageEntryType> inline auto StorageCacheEntries<StorageEntryType>::begin() -> iterator
{
return m_entries.begin();
}
template <typename StorageEntryType> inline auto StorageCacheEntries<StorageEntryType>::end() -> iterator
{
return m_entries.end();
}
template <typename StorageEntriesType, typename StorageType, typename SpecType> struct StorageCache {
using Entries = StorageEntriesType;
using Entry = typename Entries::Entry;
using ROTxn = typename StorageType::ROTransaction;
using RWTxn = typename StorageType::RWTransaction;
using Storage = typename Entries::Storage;
struct StoreResult {
StorageID id = 0;
bool updated = false;
std::shared_ptr<typename Entries::Entry> oldEntry;
};
SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID);
SpecType retrieve(Storage &storage, StorageID storageID);
SpecType retrieve(Storage &storage, RWTxn *, const std::string &entryName);
SpecType retrieve(Storage &storage, const std::string &entryName);
StoreResult store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force);
StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr<Entry> &entry);
bool invalidate(Storage &storage, const std::string &entryName);
void clear(Storage &storage);
void clearCacheOnly(Storage &storage);
void setLimit(std::size_t limit);
private:
Entries m_entries;
std::mutex m_mutex;
};
} // namespace LibPkg
#endif // LIBPKG_DATA_STORAGE_GENERIC_H

155
libpkg/data/storageprivate.h
View File

@ -2,163 +2,10 @@
#define LIBPKG_DATA_STORAGE_PRIVATE_H
#include "./package.h"
#include "../lmdb-safe/lmdb-reflective.hh"
#include "../lmdb-safe/lmdb-safe.hh"
#include "../lmdb-safe/lmdb-typed.hh"
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index_container.hpp>
#include <memory>
#include <mutex>
#include "./storagegeneric.h"
namespace LibPkg {
using StorageID = std::uint32_t;
template <typename StorageType, typename EntryType> struct StorageCacheRef {
using Storage = StorageType;
explicit StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr<EntryType> &entry);
explicit StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName);
bool operator==(const StorageCacheRef &other) const;
const StorageType *relatedStorage = nullptr;
const std::string *entryName;
};
template <typename StorageType, typename EntryType>
inline StorageCacheRef<StorageType, EntryType>::StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr<EntryType> &entry)
: relatedStorage(&relatedStorage)
, entryName(&entry->name)
{
}
template <typename StorageType, typename EntryType>
inline StorageCacheRef<StorageType, EntryType>::StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName)
: relatedStorage(&relatedStorage)
, entryName(&entryName)
{
}
template <typename StorageType, typename EntryType>
inline bool StorageCacheRef<StorageType, EntryType>::operator==(const StorageCacheRef<StorageType, EntryType> &other) const
{
return relatedStorage == other.relatedStorage && *entryName == *other.entryName;
}
template <typename StorageRefType, typename EntryType> struct StorageCacheEntry {
using Ref = StorageRefType;
using Entry = EntryType;
using Storage = typename Ref::Storage;
explicit StorageCacheEntry(const StorageRefType &ref, StorageID id);
StorageRefType ref;
StorageID id;
std::shared_ptr<EntryType> entry;
};
template <typename StorageRefType, typename EntryType>
inline StorageCacheEntry<StorageRefType, EntryType>::StorageCacheEntry(const StorageRefType &ref, StorageID id)
: ref(ref)
, id(id)
{
}
template <typename StorageEntryType> struct StorageEntryByID {
struct result_type {
StorageID id = 0;
const typename StorageEntryType::Storage *storage = nullptr;
bool operator==(const result_type &other) const
{
return id == other.id && storage == other.storage;
}
};
result_type operator()(const StorageEntryType &storageEntry) const
{
return result_type{ storageEntry.id, storageEntry.ref.relatedStorage };
}
};
template <typename StorageEntryType> class StorageCacheEntries {
public:
using Ref = typename StorageEntryType::Ref;
using Entry = typename StorageEntryType::Entry;
using Storage = typename StorageEntryType::Storage;
using StorageEntry = StorageEntryType;
using ByID = StorageEntryByID<StorageEntry>;
using EntryList = boost::multi_index::multi_index_container<StorageEntry,
boost::multi_index::indexed_by<boost::multi_index::sequenced<>,
boost::multi_index::hashed_unique<boost::multi_index::tag<typename ByID::result_type>, ByID>,
boost::multi_index::hashed_unique<boost::multi_index::tag<Ref>, BOOST_MULTI_INDEX_MEMBER(StorageEntryType, Ref, ref)>>>;
using iterator = typename EntryList::iterator;
explicit StorageCacheEntries(std::size_t limit = 1000);
template <typename IndexType> StorageEntry *find(const IndexType &ref);
StorageEntry &insert(StorageEntry &&entry);
std::size_t erase(const Ref &ref);
std::size_t clear(const Storage &storage);
iterator begin();
iterator end();
void setLimit(std::size_t limit);
private:
EntryList m_entries;
std::size_t m_limit;
};
template <typename StorageEntryType>
inline StorageCacheEntries<StorageEntryType>::StorageCacheEntries(std::size_t limit)
: m_limit(limit)
{
}
template <typename StorageEntryType> inline std::size_t StorageCacheEntries<StorageEntryType>::erase(const Ref &ref)
{
return m_entries.template get<typename StorageEntryType::Ref>().erase(ref);
}
template <typename StorageEntryType> inline auto StorageCacheEntries<StorageEntryType>::begin() -> iterator
{
return m_entries.begin();
}
template <typename StorageEntryType> inline auto StorageCacheEntries<StorageEntryType>::end() -> iterator
{
return m_entries.end();
}
template <typename StorageEntriesType, typename StorageType, typename SpecType> struct StorageCache {
using Entries = StorageEntriesType;
using Entry = typename Entries::Entry;
using ROTxn = typename StorageType::ROTransaction;
using RWTxn = typename StorageType::RWTransaction;
using Storage = typename Entries::Storage;
struct StoreResult {
StorageID id = 0;
bool updated = false;
std::shared_ptr<typename Entries::Entry> oldEntry;
};
SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID);
SpecType retrieve(Storage &storage, StorageID storageID);
SpecType retrieve(Storage &storage, RWTxn *, const std::string &entryName);
SpecType retrieve(Storage &storage, const std::string &entryName);
StoreResult store(Storage &storage, const std::shared_ptr<Entry> &entry, bool force);
StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr<Entry> &entry);
bool invalidate(Storage &storage, const std::string &entryName);
void clear(Storage &storage);
void clearCacheOnly(Storage &storage);
void setLimit(std::size_t limit);
private:
Entries m_entries;
std::mutex m_mutex;
};
using PackageStorage = LMDBSafe::TypedDBI<Package, LMDBSafe::index_on<Package, std::string, &Package::name>>;
using DependencyStorage = LMDBSafe::TypedDBI<DatabaseDependency, LMDBSafe::index_on<Dependency, std::string, &DatabaseDependency::name>>;
using LibraryDependencyStorage

17
librepomgr/CMakeLists.txt
View File

@ -71,9 +71,6 @@ set(META_APP_DESCRIPTION "Library for managing custom Arch Linux repositories")
set(META_VERSION_MAJOR 0)
set(META_VERSION_MINOR 0)
set(META_VERSION_PATCH 1)
set(META_VERSION_CACHE 11)
set(META_VERSION_BUILD_ACTIONS_JSON 0)
set(META_VERSION_LIBRARY_DEPENDENCIES_JSON 0)
set(LINK_TESTS_AGAINST_APP_TARGET ON)
# find c++utilities
@ -89,9 +86,10 @@ set(Boost_USE_MULTITHREADED ON)
if (BOOST_STATIC_LINKAGE)
set(Boost_USE_STATIC_LIBS ON)
endif ()
set(BOOST_ARGS "REQUIRED;COMPONENTS;system;filesystem")
set(BOOST_ARGS "REQUIRED;COMPONENTS;system;filesystem;iostreams")
use_package(TARGET_NAME Boost::system PACKAGE_NAME Boost PACKAGE_ARGS "${BOOST_ARGS}")
use_package(TARGET_NAME Boost::filesystem PACKAGE_NAME Boost PACKAGE_ARGS "${BOOST_ARGS}")
use_package(TARGET_NAME Boost::iostreams PACKAGE_NAME Boost PACKAGE_ARGS "${BOOST_ARGS}")
option(BOOST_ASIO_IO_URING OFF "enable use of io_uring")
if (BOOST_ASIO_IO_URING)
list(APPEND META_PUBLIC_COMPILE_DEFINITIONS BOOST_ASIO_HAS_IO_URING BOOST_ASIO_DISABLE_EPOLL)
@ -106,6 +104,10 @@ use_reflective_rapidjson(VISIBILITY PUBLIC)
find_package(libpkg ${META_APP_VERSION} REQUIRED)
use_libpkg(VISIBILITY PUBLIC)
# find lmdb-safe
find_package(lmdb-safe${CONFIGURATION_PACKAGE_SUFFIX} REQUIRED)
use_lmdb_safe()
# link against crypto and SSL library from OpenSSL
use_openssl(VISIBILITY PUBLIC)
@ -115,13 +117,6 @@ list(APPEND PUBLIC_LIBRARIES pthread)
# apply basic configuration
include(BasicConfig)
# add cache version to config header
string(APPEND META_CUSTOM_CONFIG "#define ${META_PROJECT_VARNAME}_CACHE_VERSION \"${META_VERSION_CACHE}\"\n")
string(APPEND META_CUSTOM_CONFIG
"#define ${META_PROJECT_VARNAME}_BUILD_ACTIONS_JSON_VERSION \"${META_VERSION_BUILD_ACTIONS_JSON}\"\n")
string(APPEND META_CUSTOM_CONFIG
"#define ${META_PROJECT_VARNAME}_LIBRARY_DEPENDENCIES_JSON_VERSION \"${META_VERSION_LIBRARY_DEPENDENCIES_JSON}\"\n")
# trigger code generator for tests because the tests already contain structs to be (de)serialized
include(ReflectionGenerator)
add_reflection_generator_invocation(

81
librepomgr/buildactions/buildaction.cpp
View File

@ -204,6 +204,43 @@ BuildAction::BuildAction(IdType id, ServiceSetup *setup) noexcept
{
}
BuildAction &BuildAction::operator=(BuildAction &&other)
{
if (this == &other) {
return *this;
}
id = other.id;
taskName = std::move(other.taskName);
templateName = std::move(other.templateName);
directory = std::move(other.directory);
packageNames = std::move(other.packageNames);
sourceDbs = std::move(other.sourceDbs);
destinationDbs = std::move(other.destinationDbs);
settings = std::move(other.settings);
flags = other.flags;
type = other.type;
status = other.status;
result = other.result;
resultData = std::move(other.resultData);
output = std::move(other.output);
outputMimeType = std::move(other.outputMimeType);
logfiles = std::move(other.logfiles);
artefacts = std::move(other.artefacts);
created = other.created;
started = other.started;
finished = other.finished;
startAfter = std::move(other.startAfter);
m_log = std::move(other.m_log);
m_setup = other.m_setup;
m_aborted = false;
m_stopHandler = std::function<void(void)>();
m_concludeHandler = std::function<void(void)>();
m_ongoingProcesses.clear();
m_bufferingForSession.clear();
m_internalBuildAction = std::move(other.m_internalBuildAction);
return *this;
}
BuildAction::~BuildAction()
{
}
@ -223,10 +260,10 @@ bool BuildAction::haveSucceeded(const std::vector<std::shared_ptr<BuildAction>>
* the build action is setup-globally visible.
* \returns Returns immediately. The real work is done in a build action thread.
*/
void BuildAction::start(ServiceSetup &setup)
LibPkg::StorageID BuildAction::start(ServiceSetup &setup)
{
if (!isScheduled()) {
return;
return 0;
}
started = DateTime::gmtNow();
@ -236,8 +273,7 @@ void BuildAction::start(ServiceSetup &setup)
switch (type) {
case BuildActionType::Invalid:
resultData = "type is invalid";
conclude(BuildActionResult::Failure);
break;
return conclude(BuildActionResult::Failure);
case BuildActionType::RemovePackages:
post<RemovePackages>();
break;
@ -281,31 +317,17 @@ void BuildAction::start(ServiceSetup &setup)
break;
default:
resultData = "not implemented yet or invalid type";
conclude(BuildActionResult::Failure);
return conclude(BuildActionResult::Failure);
}
}
void BuildAction::startAfterOtherBuildActions(ServiceSetup &setup, const std::vector<std::shared_ptr<BuildAction>> &startsAfterBuildActions)
{
auto allSucceeded = true;
for (auto &previousBuildAction : startsAfterBuildActions) {
if (!previousBuildAction->hasSucceeded()) {
previousBuildAction->m_followUpActions.emplace_back(weak_from_this());
allSucceeded = false;
}
}
if (allSucceeded) {
start(setup);
}
// update in persistent storage and create entry in "running cache"
return m_setup->building.storeBuildAction(shared_from_this());
}
void BuildAction::assignStartAfter(const std::vector<std::shared_ptr<BuildAction>> &startsAfterBuildActions)
{
for (auto &previousBuildAction : startsAfterBuildActions) {
startAfter.emplace_back(previousBuildAction->id);
if (!previousBuildAction->hasSucceeded()) {
previousBuildAction->m_followUpActions.emplace_back(weak_from_this());
}
}
}
@ -333,7 +355,7 @@ template <typename Callback> void BuildAction::post(Callback &&codeToRun)
/*!
* \brief Internally called to conclude the build action.
*/
void BuildAction::conclude(BuildActionResult result)
LibPkg::StorageID BuildAction::conclude(BuildActionResult result)
{
// set fields accordingly
status = BuildActionStatus::Finished;
@ -354,19 +376,26 @@ void BuildAction::conclude(BuildActionResult result)
// start globally visible follow-up actions if succeeded
if (result == BuildActionResult::Success && m_setup) {
for (auto &maybeStillValidFollowUpAction : m_followUpActions) {
auto followUpAction = maybeStillValidFollowUpAction.lock();
if (followUpAction && followUpAction->isScheduled()
&& BuildAction::haveSucceeded(m_setup->building.getBuildActions(followUpAction->startAfter))) {
const auto followUps = m_setup->building.followUpBuildActions(id);
for (auto &followUpAction : followUps) {
if (followUpAction->isScheduled() && BuildAction::haveSucceeded(m_setup->building.getBuildActions(followUpAction->startAfter))) {
followUpAction->start(*m_setup);
}
}
// note: Not cleaning up the follow-up actions here because at some point I might implement recursive restarting.
}
// write build action to persistent storage
// TODO: should this also be done in the middle of the execution to have some "save points"?
auto id = LibPkg::StorageID();
if (m_setup && m_setup->building.hasStorage()) {
id = m_setup->building.storeBuildAction(shared_from_this());
}
if (m_concludeHandler) {
m_concludeHandler();
}
return id;
}
#ifdef LIBREPOMGR_DUMMY_BUILD_ACTION_ENABLED

14
librepomgr/buildactions/buildaction.h
View File

@ -173,6 +173,7 @@ public:
static constexpr IdType invalidId = std::numeric_limits<BuildAction::IdType>::max();
explicit BuildAction(IdType id = invalidId, ServiceSetup *setup = nullptr) noexcept;
BuildAction &operator=(BuildAction &&other);
~BuildAction();
bool isScheduled() const;
bool isExecuting() const;
@ -180,8 +181,7 @@ public:
bool hasSucceeded() const;
static bool haveSucceeded(const std::vector<std::shared_ptr<BuildAction>> &buildActions);
bool isAborted() const;
void start(ServiceSetup &setup);
void startAfterOtherBuildActions(ServiceSetup &setup, const std::vector<std::shared_ptr<BuildAction>> &startsAfterBuildActions);
LibPkg::StorageID start(ServiceSetup &setup);
void assignStartAfter(const std::vector<std::shared_ptr<BuildAction>> &startsAfterBuildActions);
void abort();
void appendOutput(std::string &&output);
@ -197,12 +197,13 @@ public:
void terminateOngoingBuildProcesses();
void streamFile(const WebAPI::Params &params, const std::string &filePath, std::string_view fileMimeType);
void streamOutput(const WebAPI::Params &params, std::size_t offset = 0);
ServiceSetup *setup();
protected:
private:
template <typename InternalBuildActionType> void post();
template <typename Callback> void post(Callback &&codeToRun);
void conclude(BuildActionResult result);
LibPkg::StorageID conclude(BuildActionResult result);
void continueStreamingExistingOutputToSession(std::shared_ptr<WebAPI::Session> session, OutputBufferingForSession &buffering,
const boost::system::error_code &error, std::size_t bytesTransferred);
void continueStreamingNewOutputToSession(std::shared_ptr<WebAPI::Session> session, OutputBufferingForSession &buffering,
@ -216,7 +217,6 @@ public:
std::string directory;
std::vector<std::string> packageNames;
std::vector<std::string> sourceDbs, destinationDbs;
std::vector<std::string> extraParams; // deprecated; remove at some point
std::unordered_map<std::string, std::string> settings;
BuildActionFlagType flags = noBuildActionFlags;
BuildActionType type = BuildActionType::Invalid;
@ -248,7 +248,6 @@ private:
std::mutex m_outputStreamingMutex;
std::unordered_map<std::shared_ptr<WebAPI::Session>, std::unique_ptr<OutputBufferingForSession>> m_bufferingForSession;
std::unique_ptr<InternalBuildAction> m_internalBuildAction;
std::vector<std::weak_ptr<BuildAction>> m_followUpActions;
};
inline bool BuildAction::isScheduled() const
@ -297,6 +296,11 @@ inline std::shared_ptr<BuildProcessSession> BuildAction::findBuildProcess(const
return i != m_ongoingProcesses.cend() ? i->second : nullptr;
}
inline ServiceSetup *BuildAction::setup()
{
return m_setup;
}
/*!
* \brief Appends the specified arguments to the build action's log but *not* to the overall service log.
*/

5
librepomgr/buildactions/buildactionfwd.h
View File

@ -1,13 +1,12 @@
#ifndef LIBREPOMGR_BUILD_ACTION_FWD_H
#define LIBREPOMGR_BUILD_ACTION_FWD_H
#include <memory>
#include <vector>
#include "../../libpkg/data/storagefwd.h"
namespace LibRepoMgr {
struct BuildAction;
using BuildActionIdType = std::vector<std::shared_ptr<BuildAction>>::size_type; // build actions are stored in a vector and the ID is used as index
using BuildActionIdType = std::size_t;
} // namespace LibRepoMgr

2
librepomgr/buildactions/buildactionprivate.h
View File

@ -276,7 +276,7 @@ template <typename... ChildArgs> void BuildProcessSession::launch(ChildArgs &&..
struct ProcessResult;
/// \brief The InternalBuildAction struct contains internal details (which are not serialized / accessible via the web API) and helpers.
/// \remarks This struct is mean to be inherited from when creating a new type build action like it is down by the classes below.
/// \remarks This struct is mean to be inherited from when creating a new type of build action like it is done by the classes below.
struct LIBREPOMGR_EXPORT InternalBuildAction {
InternalBuildAction(ServiceSetup &setup, const std::shared_ptr<BuildAction> &buildAction);

3
librepomgr/logcontext.h
View File

@ -15,12 +15,13 @@ struct BuildAction;
struct LIBREPOMGR_EXPORT LogContext {
explicit LogContext(BuildAction *buildAction = nullptr);
LogContext &operator=(const LogContext &) = delete;
LogContext &operator=(LogContext &&) = default;
template <typename... Args> LogContext &operator()(CppUtilities::EscapeCodes::Phrases phrase, Args &&...args);
template <typename... Args> LogContext &operator()(Args &&...args);
template <typename... Args> LogContext &operator()(std::string &&msg);
private:
BuildAction *const m_buildAction;
BuildAction *m_buildAction;
};
inline LogContext::LogContext(BuildAction *buildAction)

451
librepomgr/serversetup.cpp
View File

@ -7,6 +7,8 @@
#include "./webapi/server.h"
#include "../libpkg/data/storagegeneric.h"
#include "reflection/serversetup.h"
#include "resources/config.h"
@ -38,6 +40,24 @@ using namespace LibPkg;
namespace LibRepoMgr {
struct Storage {
using BuildActionStorage = LMDBSafe::TypedDBI<BuildAction>;
explicit Storage(const char *path);
private:
std::shared_ptr<LMDBSafe::MDBEnv> m_env;
public:
BuildActionStorage buildActions;
};
Storage::Storage(const char *path)
: m_env(LMDBSafe::getMDBEnv(path, MDB_NOSUBDIR, 0600, 2))
, buildActions(m_env, "buildactions")
{
}
static void deduplicateVector(std::vector<std::string> &vector)
{
std::unordered_set<std::string_view> visited;
@ -96,6 +116,16 @@ void ServiceSetup::WebServerSetup::applyConfig(const std::multimap<std::string,
}
}
ServiceSetup::BuildSetup::BuildSetup() = default;
ServiceSetup::BuildSetup::~BuildSetup() = default;
void ServiceSetup::BuildSetup::initStorage(const char *path)
{
if (!m_storage) {
m_storage = std::make_unique<Storage>(path);
}
}
void ServiceSetup::BuildSetup::applyConfig(const std::multimap<std::string, std::string> &multimap)
{
convertValue(multimap, "threads", threadCount);
@ -121,6 +151,7 @@ void ServiceSetup::BuildSetup::applyConfig(const std::multimap<std::string, std:
convertValue(multimap, "package_download_size_limit", packageDownloadSizeLimit);
convertValue(multimap, "test_files_dir", testFilesDir);
convertValue(multimap, "load_files_dbs", loadFilesDbs);
convertValue(multimap, "db_path", dbPath);
}
void ServiceSetup::BuildSetup::readPresets(const std::string &configFilePath, const std::string &presetsFileRelativePath)
@ -191,33 +222,166 @@ ServiceSetup::BuildSetup::Worker ServiceSetup::BuildSetup::allocateBuildWorker()
return Worker(*this);
}
BuildAction::IdType ServiceSetup::BuildSetup::allocateBuildActionID()
LibPkg::StorageID ServiceSetup::BuildSetup::allocateBuildActionID()
{
if (!invalidActions.empty()) {
const auto i = invalidActions.begin();
const auto id = *i;
invalidActions.erase(i);
return id;
}
const auto id = actions.size();
actions.emplace_back();
static const auto emptyBuildAction = BuildAction();
auto txn = m_storage->buildActions.getRWTransaction();
const auto id = txn.put(emptyBuildAction);
txn.commit();
return id;
}
std::vector<std::shared_ptr<BuildAction>> ServiceSetup::BuildSetup::getBuildActions(const std::vector<BuildAction::IdType> &ids)
std::shared_ptr<BuildAction> ServiceSetup::BuildSetup::getBuildAction(BuildActionIdType id)
{
if (auto i = m_runningActions.find(id); i != m_runningActions.end()) {
return i->second;
}
const auto res = std::make_shared<BuildAction>();
auto txn = m_storage->buildActions.getROTransaction();
return id <= std::numeric_limits<LibPkg::StorageID>::max() && txn.get(static_cast<LibPkg::StorageID>(id), *res) ? res : nullptr;
}
std::vector<std::shared_ptr<BuildAction>> ServiceSetup::BuildSetup::getBuildActions(const std::vector<BuildActionIdType> &ids)
{
auto buildAction = std::shared_ptr<BuildAction>();
auto buildActions = std::vector<std::shared_ptr<BuildAction>>();
buildActions.reserve(ids.size());
auto txn = m_storage->buildActions.getROTransaction();
for (const auto id : ids) {
if (id < actions.size()) {
if (auto &buildAction = actions[id]) {
buildActions.emplace_back(buildAction);
}
if (auto i = m_runningActions.find(id); i != m_runningActions.end()) {
buildActions.emplace_back(i->second);
continue;
}
if (id > std::numeric_limits<LibPkg::StorageID>::max()) {
continue;
}
if (!buildAction) {
buildAction = std::make_shared<BuildAction>();
}
if (txn.get(static_cast<LibPkg::StorageID>(id), *buildAction)) {
buildActions.emplace_back(std::move(buildAction));
}
}
return buildActions;
}
StorageID ServiceSetup::BuildSetup::storeBuildAction(const std::shared_ptr<BuildAction> &buildAction)
{
// update cache of running build actions
if (buildAction->isExecuting()) {
m_runningActions[buildAction->id] = buildAction;
} else {
m_runningActions.erase(buildAction->id);
}
// update index of follow-up actions for scheduled actions
// note: This would break if startAfter would be modified after the initial build action creation.
if (buildAction->isScheduled() && !buildAction->startAfter.empty()) {
const auto previousBuildActions = getBuildActions(buildAction->startAfter);
auto allSucceeded = false;
for (auto &previousBuildAction : previousBuildActions) {
if (!previousBuildAction->hasSucceeded()) {
m_followUpActions[previousBuildAction->id].emplace(buildAction->id);
allSucceeded = false;
}
}
// immediately start if all follow-up actions have succeeded
if (allSucceeded && buildAction->setup()) {
return buildAction->start(*buildAction->setup());
}
} else {
for (const auto id : buildAction->startAfter) {
if (const auto i = m_followUpActions.find(id); i != m_followUpActions.end()) {
auto &followUps = i->second;
followUps.erase(buildAction->id);
if (followUps.empty()) {
m_followUpActions.erase(i);
}
}
}
}
// update persistent storage
auto txn = m_storage->buildActions.getRWTransaction();
const auto id = txn.put(*buildAction, static_cast<LibPkg::StorageID>(buildAction->id)); // buildAction->id expected to be a valid StorageID or 0
txn.commit();
return id;
}
void ServiceSetup::BuildSetup::deleteBuildAction(const std::vector<std::shared_ptr<BuildAction>> &actions)
{
auto txn = m_storage->buildActions.getRWTransaction();
for (const auto &action : actions) {
// remove action from cache for running actions
m_runningActions.erase(action->id);
// remove any actions to start after the action to delete
m_followUpActions.erase(action->id);
// remove follow-up indexes for actions previous to the action to delete
for (auto i = m_followUpActions.begin(), end = m_followUpActions.end(); i != end;) {
auto &followUps = i->second;
followUps.erase(action->id);
if (followUps.empty()) {
i = m_followUpActions.erase(i);
} else {
++i;
}
}
// delete action from storage
if (action->id && action->id <= std::numeric_limits<LibPkg::StorageID>::max()) {
txn.del(static_cast<LibPkg::StorageID>(action->id));
}
}
txn.commit();
}
std::size_t ServiceSetup::BuildSetup::buildActionCount()
{
return m_storage->buildActions.getROTransaction().size();
}
void ServiceSetup::BuildSetup::forEachBuildAction(
std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func)
{
auto txn = m_storage->buildActions.getROTransaction();
count(txn.size());
for (auto i = txn.begin(); i != txn.end(); ++i) {
if (func(i.getID(), std::move(i.value()))) {
return;
}
}
}
void ServiceSetup::BuildSetup::forEachBuildAction(std::function<bool(LibPkg::StorageID, BuildAction &, bool &)> &&func)
{
auto txn = m_storage->buildActions.getRWTransaction();
for (auto i = txn.begin(); i != txn.end(); ++i) {
const auto running = m_runningActions.find(i.getID());
auto &action = running != m_runningActions.end() ? *running->second : i.value();
auto save = false;
const auto stop = func(i.getID(), action, save);
if (save) {
txn.put(action, i.getID());
}
if (stop) {
return;
}
}
}
std::vector<std::shared_ptr<BuildAction>> ServiceSetup::BuildSetup::followUpBuildActions(BuildActionIdType forId)
{
auto res = std::vector<std::shared_ptr<BuildAction>>();
const auto i = m_followUpActions.find(forId);
if (i == m_followUpActions.end()) {
return res;
}
res.reserve(i->second.size());
for (const auto followUpId : i->second) {
if (auto buildAction = getBuildAction(followUpId)) {
res.emplace_back(std::move(buildAction));
}
}
return res;
}
void ServiceSetup::loadConfigFiles(bool doFirstTimeSetup)
{
// read config file
@ -387,244 +551,29 @@ void ServiceSetup::printDatabases()
cerr << Phrases::SubMessage << "AUR (" << config.aur.packageCount() << " packages cached)" << Phrases::End;
}
std::string_view ServiceSetup::cacheFilePath() const
void ServiceSetup::restoreState()
{
return "cache-v" LIBREPOMGR_CACHE_VERSION ".bin";
}
RAPIDJSON_NAMESPACE::Document ServiceSetup::libraryDependenciesToJson()
{
namespace JR = ReflectiveRapidJSON::JsonReflector;
auto document = RAPIDJSON_NAMESPACE::Document(RAPIDJSON_NAMESPACE::kObjectType);
auto &alloc = document.GetAllocator();
for (auto &db : config.databases) {
auto dbValue = RAPIDJSON_NAMESPACE::Value(RAPIDJSON_NAMESPACE::Type::kObjectType);
db.allPackages([&](StorageID, const std::shared_ptr<Package> &package) {
if (!package->packageInfo) {
return false;
}
if (package->libdepends.empty() && package->libprovides.empty()) {
auto hasVersionedPythonOrPerlDep = false;
for (const auto &dependency : package->dependencies) {
if (dependency.mode == DependencyMode::Any || dependency.version.empty()
|| (dependency.name != "python" && dependency.name != "python2" && dependency.name != "perl")) {
return false;
}
hasVersionedPythonOrPerlDep = true;
break;
}
if (!hasVersionedPythonOrPerlDep) {
return false;
}
}
auto pkgValue = RAPIDJSON_NAMESPACE::Value(RAPIDJSON_NAMESPACE::Type::kObjectType);
auto pkgObj = pkgValue.GetObject();
JR::push(package->version, "v", pkgObj, alloc);
JR::push(package->packageInfo->buildDate, "t", pkgObj, alloc);
JR::push(package->dependencies, "d", pkgObj, alloc); // for versioned Python/Perl deps
JR::push(package->libdepends, "ld", pkgObj, alloc);
JR::push(package->libprovides, "lp", pkgObj, alloc);
dbValue.AddMember(RAPIDJSON_NAMESPACE::StringRef(package->name.data(), JR::rapidJsonSize(package->name.size())), pkgValue, alloc);
return false;
});
document.AddMember(RAPIDJSON_NAMESPACE::Value(db.name % '@' + db.arch, alloc), dbValue, alloc);
}
return document;
}
void ServiceSetup::restoreLibraryDependenciesFromJson(const string &json, ReflectiveRapidJSON::JsonDeserializationErrors *errors)
{
namespace JR = ReflectiveRapidJSON::JsonReflector;
const auto document = JR::parseJsonDocFromString(json.data(), json.size());
if (!document.IsObject()) {
errors->reportTypeMismatch<std::map<std::string, LibPkg::Database>>(document.GetType());
return;
}
// FIXME: be more error resilient here, e.g. set the following line and print list of errors instead of aborting on first error
// errors->throwOn = ReflectiveRapidJSON::JsonDeserializationErrors::ThrowOn::None;
const auto dbObj = document.GetObject();
for (const auto &dbEntry : dbObj) {
if (!dbEntry.value.IsObject()) {
errors->reportTypeMismatch<RAPIDJSON_NAMESPACE::kObjectType>(document.GetType());
continue;
}
auto *const db = config.findOrCreateDatabaseFromDenotation(std::string_view(dbEntry.name.GetString()));
const auto pkgsObj = dbEntry.value.GetObject();
for (const auto &pkgEntry : pkgsObj) {
if (!pkgEntry.value.IsObject()) {
errors->reportTypeMismatch<LibPkg::Package>(document.GetType());
continue;
}
const auto pkgObj = pkgEntry.value.GetObject();
auto name = std::string(pkgEntry.name.GetString());
auto [pkgID, pkg] = db->findPackageWithID(name);
if (pkg) {
// do not mess with already existing packages; this restoring stuff is supposed to be done before loading packages from DBs
continue;
}
pkg = std::make_shared<LibPkg::Package>();
pkg->name = std::move(name);
pkg->origin = PackageOrigin::CustomSource;
pkg->packageInfo = std::make_unique<LibPkg::PackageInfo>();
JR::pull(pkg->version, "v", pkgObj, errors);
JR::pull(pkg->packageInfo->buildDate, "t", pkgObj, errors);
JR::pull(pkg->dependencies, "d", pkgObj, errors); // for versioned Python/Perl deps
JR::pull(pkg->libdepends, "ld", pkgObj, errors);
JR::pull(pkg->libprovides, "lp", pkgObj, errors);
db->updatePackage(pkg);
}
}
}
std::size_t ServiceSetup::restoreState()
{
// clear old build actions before (There must not be any ongoing build actions when calling this function!)
building.actions.clear();
building.invalidActions.clear();
// restore configuration and maybe build actions from JSON file
const auto cacheFilePath = this->cacheFilePath();
std::size_t size = 0;
bool hasConfig = false, hasBuildActions = false;
try {
fstream cacheFile;
cacheFile.exceptions(ios_base::failbit | ios_base::badbit);
cacheFile.open(cacheFilePath.data(), ios_base::in | ios_base::binary);
ReflectiveRapidJSON::BinaryReflector::BinaryDeserializer deserializer(&cacheFile);
deserializer.read(config);
hasConfig = true;
if (!hasBuildActions) {
deserializer.read(building.actions);
hasBuildActions = true;
}
size = static_cast<std::uint64_t>(cacheFile.tellg());
cacheFile.close();
cerr << Phrases::SuccessMessage << "Restored cache file \"" << cacheFilePath << "\", " << dataSizeToString(size) << Phrases::EndFlush;
if (hasBuildActions) {
cerr << Phrases::SubMessage << "Restored build actions from cache file 😌" << Phrases::EndFlush;
}
} catch (const ConversionException &) {
cerr << Phrases::WarningMessage << "A conversion error occurred when restoring cache file \"" << cacheFilePath << "\"." << Phrases::EndFlush;
} catch (const ios_base::failure &) {
cerr << Phrases::WarningMessage << "An IO error occurred when restoring cache file \"" << cacheFilePath << "\"." << Phrases::EndFlush;
}
// open LMDB storage
cout << Phrases::InfoMessage << "Opening LMDB file: " << dbPath << " (max DBs: " << maxDbs << ')' << Phrases::EndFlush;
cout << Phrases::InfoMessage << "Opening config LMDB file: " << dbPath << " (max DBs: " << maxDbs << ')' << Phrases::EndFlush;
config.initStorage(dbPath.data(), maxDbs);
config.setPackageCacheLimit(packageCacheLimit);
// restore build actions from JSON file
if (!hasBuildActions) {
try {
if (!restoreJsonObject(building.actions, workingDirectory, "build-actions-v" LIBREPOMGR_BUILD_ACTIONS_JSON_VERSION,
RestoreJsonExistingFileHandling::Skip)
.empty()) {
cerr << Phrases::SuccessMessage << "Restored build actions from JSON file 😒" << Phrases::EndFlush;
}
} catch (const std::runtime_error &e) {
cerr << Phrases::ErrorMessage << e.what() << Phrases::EndFlush;
}
}
// restore provided/required libraries from JSON file
if (!hasConfig) {
try {
if (!restoreJsonObject(std::bind(&ServiceSetup::restoreLibraryDependenciesFromJson, this, std::placeholders::_1, std::placeholders::_2),
workingDirectory, "library-dependencies-v" LIBREPOMGR_LIBRARY_DEPENDENCIES_JSON_VERSION, RestoreJsonExistingFileHandling::Skip)
.empty()) {
cerr << Phrases::SuccessMessage << "Restored library dependencies from JSON file 😒" << Phrases::EndFlush;
}
} catch (const std::runtime_error &e) {
cerr << Phrases::ErrorMessage << e.what() << Phrases::EndFlush;
}
}
// determine invalid build actions
if (building.actions.empty()) {
return size;
}
auto newActionsSize = building.actions.size();
for (auto id = newActionsSize - 1;; --id) {
if (building.actions[id]) {
break;
}
newActionsSize = id;
if (!newActionsSize) {
break;
}
}
building.actions.resize(newActionsSize);
for (std::size_t buildActionId = 0u, buildActionsSize = building.actions.size(); buildActionId != buildActionsSize; ++buildActionId) {
if (!building.actions[buildActionId]) {
building.invalidActions.emplace(buildActionId);
}
}
cout << Phrases::InfoMessage << "Opening actions LMDB file: " << building.dbPath << Phrases::EndFlush;
building.initStorage(building.dbPath.data());
// ensure no build actions are considered running anymore and populate follow up actions
for (auto &action : building.actions) {
if (!action) {
continue;
}
for (const auto previousBuildActionID : action->startAfter) {
if (auto previousBuildAction = building.getBuildAction(previousBuildActionID)) {
previousBuildAction->m_followUpActions.emplace_back(action->weak_from_this());
building.forEachBuildAction([this](LibPkg::StorageID, BuildAction &buildAction, bool &save) {
if (buildAction.isExecuting()) {
buildAction.status = BuildActionStatus::Finished;
buildAction.result = BuildActionResult::Failure;
buildAction.resultData = "service crashed while exectuing";
save = true;
} else if (buildAction.isScheduled()) {
for (const auto previousBuildActionId : buildAction.startAfter) {
building.m_followUpActions[previousBuildActionId].emplace(buildAction.id);
}
}
if (action->isExecuting()) {
action->status = BuildActionStatus::Finished;
action->result = BuildActionResult::Failure;
action->resultData = "service crashed while exectuing";
}
}
return size;
}
std::size_t ServiceSetup::saveState()
{
// write cache file to be able to restore the service state when restarting the service efficiently
const auto cacheFilePath = this->cacheFilePath();
std::size_t size = 0;
try {
fstream cacheFile;
cacheFile.exceptions(ios_base::failbit | ios_base::badbit);
cacheFile.open(cacheFilePath.data(), ios_base::out | ios_base::trunc | ios_base::binary);
ReflectiveRapidJSON::BinaryReflector::BinarySerializer serializer(&cacheFile);
serializer.write(config);
serializer.write(building.actions);
size = static_cast<std::uint64_t>(cacheFile.tellp());
cacheFile.close();
cerr << Phrases::SuccessMessage << "Wrote cache file \"" << cacheFilePath << "\", " << dataSizeToString(size) << Phrases::EndFlush;
} catch (const ios_base::failure &) {
cerr << Phrases::WarningMessage << "An IO error occurred when dumping the cache file \"" << cacheFilePath << "\"." << Phrases::EndFlush;
}
// write build actions to a JSON file to be able to restore build actions even if the cache file can not be used due to version mismatch
// note: The JSON file's format is hopefully more stable.
try {
if (!dumpJsonObject(
building.actions, workingDirectory, "build-actions-v" LIBREPOMGR_BUILD_ACTIONS_JSON_VERSION, DumpJsonExistingFileHandling::Override)
.empty()) {
cerr << Phrases::SuccessMessage << "Wrote build actions to JSON file." << Phrases::EndFlush;
}
} catch (const std::runtime_error &e) {
cerr << Phrases::ErrorMessage << e.what() << Phrases::EndFlush;
}
// write provided/required libraries to a JSON file to be able to restore this information even if the cache file can not be used due to version
// mismatch
try {
if (!dumpJsonDocument(std::bind(&ServiceSetup::libraryDependenciesToJson, this), workingDirectory,
"library-dependencies-v" LIBREPOMGR_LIBRARY_DEPENDENCIES_JSON_VERSION, DumpJsonExistingFileHandling::Override)
.empty()) {
cerr << Phrases::SuccessMessage << "Wrote library dependencies to JSON file." << Phrases::EndFlush;
}
} catch (const std::runtime_error &e) {
cerr << Phrases::ErrorMessage << e.what() << Phrases::EndFlush;
}
return size;
return false;
});
}
void ServiceSetup::initStorage()
@ -674,14 +623,14 @@ void ServiceSetup::run()
#endif
}
for (auto &buildAction : building.actions) {
if (buildAction && buildAction->isExecuting()) {
buildAction->status = BuildActionStatus::Finished;
buildAction->result = BuildActionResult::Aborted;
building.forEachBuildAction([](LibPkg::StorageID, BuildAction &buildAction, bool &save) {
if (buildAction.isExecuting()) {
buildAction.status = BuildActionStatus::Finished;
buildAction.result = BuildActionResult::Aborted;
save = true;
}
}
saveState();
return false;
});
}
void ServiceSetup::Locks::clear()

58
librepomgr/serversetup.h
View File

@ -30,6 +30,7 @@ struct LIBREPOMGR_EXPORT ThreadPool {
};
struct ServiceStatus;
struct Storage;
struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
// the overall configuration (databases, packages, ...) used at various places
@ -46,6 +47,13 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
std::uint32_t maxDbs = 512;
std::size_t packageCacheLimit = 1000;
void loadConfigFiles(bool doFirstTimeSetup);
void printDatabases();
void restoreState();
void initStorage();
void run();
ServiceStatus computeStatus() const;
// variables relevant for the web server; only changed when (re)loading config
struct LIBREPOMGR_EXPORT WebServerSetup {
// only read by build actions and routes; changed when (re)loading config
@ -68,16 +76,17 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
// variables relevant for build actions and web server routes dealing with them
struct LIBREPOMGR_EXPORT BuildSetup : public LibPkg::Lockable {
friend void ServiceSetup::restoreState();
struct LIBREPOMGR_EXPORT Worker : private boost::asio::executor_work_guard<boost::asio::io_context::executor_type>, public ThreadPool {
explicit Worker(BuildSetup &setup);
~Worker();
BuildSetup &setup;
};
// read/written by build actions and routes
// -> acquire the build lock for these
std::vector<std::shared_ptr<BuildAction>> actions;
std::unordered_set<std::size_t> invalidActions;
explicit BuildSetup();
BuildSetup(BuildSetup &&) = delete;
~BuildSetup();
// fields which have their own locking
// -> acquire the object's lock
@ -113,13 +122,28 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
// never changed after startup
unsigned short threadCount = 4;
boost::asio::io_context ioContext;
std::string dbPath = "librepomgr.db";
void initStorage(const char *path);
bool hasStorage() const;
void applyConfig(const std::multimap<std::string, std::string> &multimap);
void readPresets(const std::string &configFilePath, const std::string &presetsFile);
Worker allocateBuildWorker();
BuildAction::IdType allocateBuildActionID();
std::shared_ptr<BuildAction> getBuildAction(BuildAction::IdType id);
std::vector<std::shared_ptr<BuildAction>> getBuildActions(const std::vector<BuildAction::IdType> &ids);
LibPkg::StorageID allocateBuildActionID();
std::shared_ptr<BuildAction> getBuildAction(BuildActionIdType id);
std::vector<std::shared_ptr<BuildAction>> getBuildActions(const std::vector<BuildActionIdType> &ids);
LibPkg::StorageID storeBuildAction(const std::shared_ptr<BuildAction> &buildAction);
void deleteBuildAction(const std::vector<std::shared_ptr<BuildAction>> &actions);
std::size_t buildActionCount();
std::size_t runningBuildActionCount();
void forEachBuildAction(std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func);
void forEachBuildAction(std::function<bool(LibPkg::StorageID, BuildAction &, bool &)> &&func);
std::vector<std::shared_ptr<BuildAction>> followUpBuildActions(BuildActionIdType forId);
private:
std::unordered_map<BuildActionIdType, std::shared_ptr<BuildAction>> m_runningActions;
std::unordered_map<BuildActionIdType, std::unordered_set<BuildActionIdType>> m_followUpActions;
std::unique_ptr<Storage> m_storage;
} building;
struct LIBREPOMGR_EXPORT Authentication : public LibPkg::Lockable {
@ -145,22 +169,16 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
std::shared_mutex m_cleanupMutex;
LockTable m_locksByName;
} locks;
void loadConfigFiles(bool doFirstTimeSetup);
void printDatabases();
std::string_view cacheFilePath() const;
RAPIDJSON_NAMESPACE::Document libraryDependenciesToJson();
void restoreLibraryDependenciesFromJson(const std::string &json, ReflectiveRapidJSON::JsonDeserializationErrors *errors);
std::size_t restoreState();
std::size_t saveState();
void initStorage();
void run();
ServiceStatus computeStatus() const;
};
inline std::shared_ptr<BuildAction> ServiceSetup::BuildSetup::getBuildAction(BuildAction::IdType id)
inline bool ServiceSetup::BuildSetup::hasStorage() const
{
return id < actions.size() ? actions[id] : nullptr;
return m_storage != nullptr;
}
inline std::size_t ServiceSetup::BuildSetup::runningBuildActionCount()
{
return m_runningActions.size();
}
inline GlobalLockable &ServiceSetup::Locks::namedLock(const std::string &lockName)

16
librepomgr/tests/buildactions.cpp
View File

@ -70,7 +70,7 @@ private:
void runBuildAction(const char *message, TimeSpan timeout = TimeSpan::fromSeconds(5));
template <typename InternalBuildActionType> InternalBuildActionType *internalBuildAction();
std::string m_dbFile;
std::string m_configDbFile, m_buildingDbFile;
ServiceSetup m_setup;
std::shared_ptr<BuildAction> m_buildAction;
std::filesystem::path m_workingDir;
@ -100,8 +100,14 @@ void BuildActionsTests::tearDown()
void BuildActionsTests::initStorage()
{
m_dbFile = workingCopyPath("test-build-actions.db", WorkingCopyMode::Cleanup);
m_setup.config.initStorage(m_dbFile.data());
if (!m_setup.config.storage()) {
m_configDbFile = workingCopyPath("test-build-actions-config.db", WorkingCopyMode::Cleanup);
m_setup.config.initStorage(m_configDbFile.data());
}
if (!m_setup.building.hasStorage()) {
m_buildingDbFile = workingCopyPath("test-build-actions-building.db", WorkingCopyMode::Cleanup);
m_setup.building.initStorage(m_buildingDbFile.data());
}
}
/*!
@ -131,9 +137,7 @@ void BuildActionsTests::loadBasicTestSetup()
*/
void BuildActionsTests::loadTestConfig()
{
if (!m_setup.config.storage()) {
initStorage();
}
initStorage();
m_setup.loadConfigFiles(false);
m_setup.building.workingDirectory = m_setup.workingDirectory + "/building";
m_setup.printDatabases();

73
librepomgr/tests/webapi.cpp
View File

@ -52,7 +52,7 @@ public:
void testPostingBuildActionsFromTask();
private:
std::string m_dbFile;
std::string m_configDbFile, m_buildingDbFile;
ServiceSetup m_setup;
boost::beast::error_code m_lastError;
string m_body;
@ -75,9 +75,11 @@ WebAPITests::WebAPITests()
void WebAPITests::setUp()
{
applicationInfo.version = APP_VERSION;
m_dbFile = workingCopyPath("test-webapi.db", WorkingCopyMode::Cleanup);
m_configDbFile = workingCopyPath("test-webapi-config.db", WorkingCopyMode::Cleanup);
m_buildingDbFile = workingCopyPath("test-webapi-building.db", WorkingCopyMode::Cleanup);
m_setup.webServer.port = randomPort();
m_setup.config.initStorage(m_dbFile.data());
m_setup.config.initStorage(m_configDbFile.data());
m_setup.building.initStorage(m_buildingDbFile.data());
}
void WebAPITests::tearDown()
@ -248,7 +250,8 @@ void WebAPITests::testPostingBuildActionsFromTask()
building.presets = decltype(building.presets)::fromJson(readFile(testFilePath("test-config/presets.json")));
CPPUNIT_ASSERT_MESSAGE("templates parsed from JSON", !building.presets.templates.empty());
CPPUNIT_ASSERT_MESSAGE("task parsed from JSON", building.presets.tasks.contains("foobarbaz"));
CPPUNIT_ASSERT_MESSAGE("no build actions present before", building.actions.empty());
CPPUNIT_ASSERT_EQUAL_MESSAGE("no build actions present before", 0_st, building.buildActionCount());
CPPUNIT_ASSERT_EQUAL_MESSAGE("no build actions running before", 0_st, building.runningBuildActionCount());
const auto response = invokeRouteHandler(&WebAPI::Routes::postBuildAction,
{
@ -260,28 +263,46 @@ void WebAPITests::testPostingBuildActionsFromTask()
const auto buildActions = parseBuildActions(response->body());
CPPUNIT_ASSERT_EQUAL_MESSAGE("expected number of build actions created", 5_st, buildActions.size());
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions actually present", 5_st, building.actions.size());
for (const auto &action : building.actions) {
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action->id, " not started yet"), BuildActionStatus::Created, action->status);
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action->id, " has no result yet"), BuildActionResult::None, action->result);
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action->id, " has task name assigned"), "foobarbaz"s, action->taskName);
}
CPPUNIT_ASSERT_EQUAL_MESSAGE("foo is 1st action", "foo"s, building.actions[0]->templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("foo has dir assigned", "foo"s, building.actions[0]->directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE("foo has correct deps", std::vector<BuildAction::IdType>{}, building.actions[0]->startAfter);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 is 2nd action", "bar-1"s, building.actions[1]->templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 has dir assigned", "bar"s, building.actions[1]->directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 has correct deps", std::vector<BuildAction::IdType>{ 0 }, building.actions[1]->startAfter);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 is 3rd action", "bar-2"s, building.actions[2]->templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 has dir assigned", "bar"s, building.actions[2]->directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 has correct deps", std::vector<BuildAction::IdType>{ 1 }, building.actions[2]->startAfter);
CPPUNIT_ASSERT_EQUAL_MESSAGE("baz is 4th action", "baz"s, building.actions[3]->templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("baz has dir assigned", "baz"s, building.actions[3]->directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE("baz has correct deps", std::vector<BuildAction::IdType>{ 0 }, building.actions[3]->startAfter);
CPPUNIT_ASSERT_EQUAL_MESSAGE("buz is 5th action", "buz"s, building.actions[4]->templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("buz has dir assigned", "buz"s, building.actions[4]->directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE(
"buz has correct deps", std::vector<BuildAction::IdType>{ 2 CPP_UTILITIES_PP_COMMA 3 }, building.actions[4]->startAfter);
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions actually present", 5_st, building.buildActionCount());
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions not started yet", 0_st, building.runningBuildActionCount());
building.forEachBuildAction([](std::size_t count) { CPPUNIT_ASSERT_EQUAL_MESSAGE("for-each loop returns correct size", 5_st, count); },
[](LibPkg::StorageID id, BuildAction &&action) {
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " not started yet"), BuildActionStatus::Created, action.status);
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has no result yet"), BuildActionResult::None, action.result);
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has task name assigned"), "foobarbaz"s, action.taskName);
switch (id) {
case 1:
CPPUNIT_ASSERT_EQUAL_MESSAGE("foo is 1st action", "foo"s, action.templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("foo has dir assigned", "foo"s, action.directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE("foo has correct deps", std::vector<BuildAction::IdType>{}, action.startAfter);
break;
case 2:
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 is 2nd action", "bar-1"s, action.templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 has dir assigned", "bar"s, action.directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 has correct deps", std::vector<BuildAction::IdType>{ 1 }, action.startAfter);
break;
case 3:
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 is 3rd action", "bar-2"s, action.templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 has dir assigned", "bar"s, action.directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 has correct deps", std::vector<BuildAction::IdType>{ 2 }, action.startAfter);
break;
case 4:
CPPUNIT_ASSERT_EQUAL_MESSAGE("baz is 4th action", "baz"s, action.templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("baz has dir assigned", "baz"s, action.directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE("baz has correct deps", std::vector<BuildAction::IdType>{ 1 }, action.startAfter);
break;
case 5:
CPPUNIT_ASSERT_EQUAL_MESSAGE("buz is 5th action", "buz"s, action.templateName);
CPPUNIT_ASSERT_EQUAL_MESSAGE("buz has dir assigned", "buz"s, action.directory);
CPPUNIT_ASSERT_EQUAL_MESSAGE(
"buz has correct deps", std::vector<BuildAction::IdType>{ 3 CPP_UTILITIES_PP_COMMA 4 }, action.startAfter);
break;
default:
CPPUNIT_FAIL(argsToString("build action with unexpected ID \"", id, "\" present"));
}
return false;
});
CPPUNIT_ASSERT_EQUAL_MESSAGE("response ok", boost::beast::http::status::ok, response->result());
}

10
librepomgr/webapi/routes.cpp
View File

@ -353,16 +353,6 @@ void postLoadPackages(const Params &params, ResponseHandler &&handler)
handler(makeText(params.request(), "packages loaded"));
}
void postDumpCacheFile(const Params &params, ResponseHandler &&handler)
{
auto configLock = params.setup.config.lockToRead();
auto buildLock = params.setup.building.lockToRead();
const auto size = params.setup.saveState();
buildLock.unlock();
configLock.unlock();
handler(makeText(params.request(), "cache file written (" % dataSizeToString(size) + ')'));
}
void postQuit(const Params &params, ResponseHandler &&handler)
{
cerr << Phrases::SuccessMessage << "Stopping via route /quit" << endl;

1
librepomgr/webapi/routes.h
View File

@ -22,7 +22,6 @@ LIBREPOMGR_EXPORT void getBuildActionOutput(const Params &params, ResponseHandle
LIBREPOMGR_EXPORT void getBuildActionLogFile(const Params &params, ResponseHandler &&handler);
LIBREPOMGR_EXPORT void getBuildActionArtefact(const Params &params, ResponseHandler &&handler);
LIBREPOMGR_EXPORT void postLoadPackages(const Params &params, ResponseHandler &&handler);
LIBREPOMGR_EXPORT void postDumpCacheFile(const Params &params, ResponseHandler &&handler);
LIBREPOMGR_EXPORT void postBuildAction(const Params &params, ResponseHandler &&handler);
LIBREPOMGR_EXPORT void postBuildActionsFromTask(const Params &params, ResponseHandler &&handler, const std::string &taskName,
const std::string &directory, const std::vector<BuildActionIdType> &startAfterIds, bool startImmediately);

171
librepomgr/webapi/routes_buildaction.cpp
View File

@ -24,33 +24,27 @@ void getBuildActions(const Params &params, ResponseHandler &&handler)
RAPIDJSON_NAMESPACE::Value::Array array(jsonDoc.GetArray());
auto buildActionLock = params.setup.building.lockToRead();
array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(params.setup.building.actions.size()), jsonDoc.GetAllocator());
auto configLock = params.setup.config.lockToRead(); // build actions might refer to "config things" like packages
for (const auto &buildAction : params.setup.building.actions) {
if (!buildAction) {
continue;
}
ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(*buildAction), array, jsonDoc.GetAllocator());
}
configLock.unlock();
params.setup.building.forEachBuildAction(
[&array, &jsonDoc](std::size_t count) { array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(count), jsonDoc.GetAllocator()); },
[&array, &jsonDoc](LibPkg::StorageID, BuildAction &&buildAction) {
ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(buildAction), array, jsonDoc.GetAllocator());
return false;
});
buildActionLock.unlock();
handler(makeJson(params.request(), jsonDoc, params.target.hasPrettyFlag()));
}
struct BuildActionSearchResult {
struct ActionRef {
std::uint64_t id;
BuildAction *action = nullptr;
};
std::vector<ActionRef> actions;
std::vector<BuildActionIdType> specifiedIds;
std::vector<std::shared_ptr<BuildAction>> actions;
std::variant<std::monostate, std::shared_lock<std::shared_mutex>, std::unique_lock<std::shared_mutex>> lock;
bool ok = false;
};
BuildActionSearchResult findBuildActions(const Params &params, ResponseHandler &&handler, bool acquireWriteLock = false, std::size_t maxIDs = 100)
{
BuildActionSearchResult result;
auto result = BuildActionSearchResult();
const auto idParams = params.target.decodeValues("id");
if (idParams.empty()) {
handler(makeBadRequest(params.request(), "need at least one build action ID"));
@ -61,10 +55,10 @@ BuildActionSearchResult findBuildActions(const Params &params, ResponseHandler &
return result;
}
result.actions.reserve(idParams.size());
result.specifiedIds.reserve(idParams.size());
for (const auto &idString : idParams) {
try {
result.actions.emplace_back(BuildActionSearchResult::ActionRef{ .id = stringToNumber<std::size_t>(idString) });
result.specifiedIds.emplace_back(stringToNumber<BuildActionIdType>(idString));
} catch (const ConversionException &) {
handler(makeBadRequest(params.request(), "all IDs must be unsigned integers"));
return result;
@ -77,17 +71,15 @@ BuildActionSearchResult findBuildActions(const Params &params, ResponseHandler &
result.lock = params.setup.building.lockToRead();
}
for (auto &actionRef : result.actions) {
if (actionRef.id >= params.setup.building.actions.size()) {
result.actions.reserve(result.specifiedIds.size());
for (const auto id : result.specifiedIds) {
auto action = params.setup.building.getBuildAction(id);
if (!action) {
result.lock = std::monostate{};
handler(makeBadRequest(params.request(), argsToString("no build action with specified ID \"", actionRef.id, "\" exists")));
return result;
}
actionRef.action = params.setup.building.actions[actionRef.id].get();
if (!actionRef.action) {
handler(makeBadRequest(params.request(), argsToString("no build action with specified ID \"", actionRef.id, "\" exists")));
handler(makeBadRequest(params.request(), argsToString("no build action with specified ID \"", id, "\" exists")));
return result;
}
result.actions.emplace_back(std::move(action));
}
result.ok = true;
return result;
@ -101,8 +93,8 @@ void getBuildActionDetails(const Params &params, ResponseHandler &&handler)
if (!buildActionsSearchResult.ok) {
return;
}
for (const auto &buildActionRef : buildActionsSearchResult.actions) {
ReflectiveRapidJSON::JsonReflector::push(*buildActionRef.action, array, jsonDoc.GetAllocator());
for (const auto &action : buildActionsSearchResult.actions) {
ReflectiveRapidJSON::JsonReflector::push(*action, array, jsonDoc.GetAllocator());
}
buildActionsSearchResult.lock = std::monostate{};
handler(makeJson(params.request(), jsonDoc, params.target.hasPrettyFlag()));
@ -128,7 +120,7 @@ void getBuildActionOutput(const Params &params, ResponseHandler &&handler)
if (!buildActionsSearchResult.ok) {
return;
}
auto buildAction = buildActionsSearchResult.actions.front().action;
auto &buildAction = buildActionsSearchResult.actions.front();
if (offset > buildAction->output.size()) {
buildActionsSearchResult.lock = std::monostate{};
handler(makeBadRequest(params.request(), "the offset must not exceed the output size"));
@ -163,8 +155,8 @@ static void getBuildActionFile(const Params &params, std::vector<std::string> Bu
return;
}
auto *const buildAction = buildActionsSearchResult.actions.front().action;
for (const auto &logFile : buildAction->*fileList) {
auto &buildAction = buildActionsSearchResult.actions.front();
for (const auto &logFile : (*buildAction).*fileList) {
if (name == logFile) {
buildAction->streamFile(params, name, "application/octet-stream");
return;
@ -218,7 +210,7 @@ void postBuildAction(const Params &params, ResponseHandler &&handler)
continue;
}
try {
startAfterIds.emplace_back(stringToNumber<BuildActionIdType>(startAfterIdValue));
startAfterIds.emplace_back(stringToNumber<LibPkg::StorageID>(startAfterIdValue));
} catch (const ConversionException &) {
handler(makeBadRequest(params.request(), "the specified ID to start after is not a valid build action ID"));
return;
@ -226,7 +218,7 @@ void postBuildAction(const Params &params, ResponseHandler &&handler)
}
if (startAfterAnotherAction) {
if (startAfterIds.empty()) {
handler(makeBadRequest(params.request(), "start condition is \"after\" but not exactly one \"start-after-id\" specified"));
handler(makeBadRequest(params.request(), "start condition is \"after\" but not at least one \"start-after-id\" specified"));
return;
}
} else if (!startAfterIdValues.empty() && (startAfterIdValues.size() > 1 || !startAfterIdValues.front().empty())) {
@ -275,10 +267,8 @@ void postBuildAction(const Params &params, ResponseHandler &&handler)
// initialize build action
auto buildLock = params.setup.building.lockToWrite();
const auto id = params.setup.building.allocateBuildActionID();
auto startsAfterBuildActions = params.setup.building.getBuildActions(startAfterIds);
const auto startNow = startImmediately || (!startsAfterBuildActions.empty() && BuildAction::haveSucceeded(startsAfterBuildActions));
buildLock.unlock();
auto buildAction = std::make_shared<BuildAction>(id);
auto buildAction = std::make_shared<BuildAction>(id, &params.setup);
if (!directories.empty()) {
buildAction->directory = move(directories.front());
}
@ -293,17 +283,13 @@ void postBuildAction(const Params &params, ResponseHandler &&handler)
// serialize build action
const auto response = buildAction->toJsonDocument();
// start build action immediately (no locking required because build action is not part of setup-global list yet)
if (startNow) {
buildAction->start(params.setup);
}
// add build action to setup-global list and to "follow-up actions" of the build action this one should be started after
// start build action immediately or just add to setup-global list for now
auto buildLock2 = params.setup.building.lockToWrite();
if (!startNow && !startsAfterBuildActions.empty()) {
buildAction->startAfterOtherBuildActions(params.setup, startsAfterBuildActions);
if (startImmediately) {
buildAction->start(params.setup);
} else {
params.setup.building.storeBuildAction(buildAction);
}
params.setup.building.actions[id] = std::move(buildAction);
buildLock2.unlock();
handler(makeJson(params.request(), response));
@ -502,7 +488,7 @@ void postBuildActionsFromTask(const Params &params, ResponseHandler &&handler, c
// add build actions to setup-global list
buildLock = building.lockToWrite(buildReadLock);
for (auto &newAction : allocatedActions) {
building.actions[newAction->id] = std::move(newAction);
building.storeBuildAction(newAction);
}
buildLock.unlock();
@ -515,41 +501,16 @@ void deleteBuildActions(const Params &params, ResponseHandler &&handler)
if (!buildActionsSearchResult.ok) {
return;
}
for (const auto &actionRef : buildActionsSearchResult.actions) {
if (!actionRef.action->isExecuting()) {
for (const auto &action : buildActionsSearchResult.actions) {
if (!action->isExecuting()) {
continue;
}
buildActionsSearchResult.lock = std::monostate{};
handler(
makeBadRequest(params.request(), argsToString("can not delete \"", actionRef.id, "\"; it is still being executed; no actions altered")));
makeBadRequest(params.request(), argsToString("can not delete \"", action->id, "\"; it is still being executed; no actions altered")));
return;
}
auto &actions = params.setup.building.actions;
auto &invalidActions = params.setup.building.invalidActions;
for (auto &actionRef : buildActionsSearchResult.actions) {
for (auto &maybeFollowUpAction : actionRef.action->m_followUpActions) {
if (auto followUpAction = maybeFollowUpAction.lock()) {
auto &startAfter = followUpAction->startAfter;
startAfter.erase(std::remove(startAfter.begin(), startAfter.end(), actionRef.id));
}
}
actions[actionRef.id] = nullptr;
invalidActions.emplace(actionRef.id);
}
if (!actions.empty()) {
auto newActionsSize = actions.size();
for (auto id = newActionsSize - 1;; --id) {
if (actions[id]) {
break;
}
newActionsSize = id;
invalidActions.erase(id);
if (!newActionsSize) {
break;
}
}
actions.resize(newActionsSize);
}
params.setup.building.deleteBuildAction(buildActionsSearchResult.actions);
buildActionsSearchResult.lock = std::monostate{};
handler(makeText(params.request(), "ok"));
}
@ -570,49 +531,32 @@ void postCloneBuildActions(const Params &params, ResponseHandler &&handler)
if (!buildActionsSearchResult.ok) {
return;
}
for (const auto &actionRef : buildActionsSearchResult.actions) {
if (actionRef.action->isDone()) {
for (const auto &action : buildActionsSearchResult.actions) {
if (action->isDone()) {
continue;
}
buildActionsSearchResult.lock = std::monostate{};
handler(makeBadRequest(
params.request(), argsToString("can not clone \"", actionRef.id, "\"; it is still scheduled or executed; no actions altered")));
params.request(), argsToString("can not clone \"", action->id, "\"; it is still scheduled or executed; no actions altered")));
return;
}
std::vector<BuildAction::IdType> cloneIds;
auto cloneIds = std::vector<BuildAction::IdType>();
cloneIds.reserve(buildActionsSearchResult.actions.size());
for (const auto &actionRef : buildActionsSearchResult.actions) {
const auto orig = actionRef.action;
for (const auto &orig : buildActionsSearchResult.actions) {
const auto id = params.setup.building.allocateBuildActionID();
auto clone = make_shared<BuildAction>(id);
clone->directory = orig->directory;
clone->packageNames = orig->packageNames;
clone->sourceDbs = orig->sourceDbs;
clone->destinationDbs = orig->destinationDbs;
clone->extraParams = orig->extraParams;
clone->settings = orig->settings;
clone->flags = orig->flags;
clone->type = orig->type;
// transfer any follow-up actions which haven't already started yet from the original build action to the new one
// TODO: It would be cool to have a "recursive flag" which would allow restarting follow-ups.
clone->m_followUpActions.reserve(orig->m_followUpActions.size());
for (auto &maybeOrigFollowUp : orig->m_followUpActions) {
auto origFollowUp = maybeOrigFollowUp.lock();
if (!origFollowUp || !origFollowUp->isScheduled()) {
continue;
}
for (auto &startAfterId : origFollowUp->startAfter) {
if (startAfterId == orig->id) {
startAfterId = id;
}
}
clone->m_followUpActions.emplace_back(origFollowUp);
}
orig->m_followUpActions.clear();
clone->startAfter = orig->startAfter;
if (startImmediately) {
clone->start(params.setup);
}
params.setup.building.actions[id] = move(clone);
params.setup.building.storeBuildAction(std::move(clone));
cloneIds.emplace_back(id);
}
buildActionsSearchResult.lock = std::monostate{};
@ -625,17 +569,17 @@ void postStartBuildActions(const Params &params, ResponseHandler &&handler)
if (!buildActionsSearchResult.ok) {
return;
}
for (const auto &actionRef : buildActionsSearchResult.actions) {
if (actionRef.action->isScheduled()) {
for (const auto &action : buildActionsSearchResult.actions) {
if (action->isScheduled()) {
continue;
}
buildActionsSearchResult.lock = std::monostate{};
handler(
makeBadRequest(params.request(), argsToString("can not start \"", actionRef.id, "\"; it has already been started; no actions altered")));
makeBadRequest(params.request(), argsToString("can not start \"", action->id, "\"; it has already been started; no actions altered")));
return;
}
for (auto &actionRef : buildActionsSearchResult.actions) {
actionRef.action->start(params.setup);
for (auto &action : buildActionsSearchResult.actions) {
action->start(params.setup);
}
buildActionsSearchResult.lock = std::monostate{};
handler(makeText(params.request(), "ok"));
@ -647,27 +591,28 @@ void postStopBuildActions(const Params &params, ResponseHandler &&handler)
if (!buildActionsSearchResult.ok) {
return;
}
for (const auto &actionRef : buildActionsSearchResult.actions) {
if (actionRef.action->isExecuting()) {
for (const auto &action : buildActionsSearchResult.actions) {
if (action->isExecuting()) {
continue;
}
buildActionsSearchResult.lock = std::monostate{};
handler(makeBadRequest(
params.request(), argsToString("can not stop/decline \"", actionRef.id, "\"; it is not being executed; no actions altered")));
params.request(), argsToString("can not stop/decline \"", action->id, "\"; it is not being executed; no actions altered")));
return;
}
for (auto &actionRef : buildActionsSearchResult.actions) {
actionRef.action->abort();
if (actionRef.action->status == BuildActionStatus::Running) {
for (auto &action : buildActionsSearchResult.actions) {
action->abort();
if (action->status == BuildActionStatus::Running) {
// can not immediately stop a running action; the action needs to terminate itself acknowledging the aborted flag
continue;
}
actionRef.action->status = BuildActionStatus::Finished;
if (actionRef.action->status == BuildActionStatus::AwaitingConfirmation) {
actionRef.action->result = BuildActionResult::ConfirmationDeclined;
if (action->status == BuildActionStatus::AwaitingConfirmation) {
action->result = BuildActionResult::ConfirmationDeclined;
} else {
actionRef.action->result = BuildActionResult::Aborted;
action->result = BuildActionResult::Aborted;
}
action->status = BuildActionStatus::Finished;
params.setup.building.storeBuildAction(action);
}
buildActionsSearchResult.lock = std::monostate{};
handler(makeText(params.request(), "ok"));

1
librepomgr/webapi/server.cpp
View File

@ -40,7 +40,6 @@ const Router Server::s_router = {
{ { http::verb::post, "/api/v0/build-action/clone" }, Route{&Routes::postCloneBuildActions, UserPermissions::ModifyBuildActions} },
{ { http::verb::post, "/api/v0/build-action/start" }, Route{&Routes::postStartBuildActions, UserPermissions::ModifyBuildActions} },
{ { http::verb::post, "/api/v0/build-action/stop" }, Route{&Routes::postStopBuildActions, UserPermissions::ModifyBuildActions} },
{ { http::verb::post, "/api/v0/dump/cache-file" }, Route{&Routes::postDumpCacheFile, UserPermissions::PerformAdminActions} },
{ { http::verb::post, "/api/v0/quit" }, Route{&Routes::postQuit, UserPermissions::PerformAdminActions} },
};
// clang-format on

4
srv/static/index.html
View File

@ -124,10 +124,6 @@
<h2>
Build actions
<span class="heading-actions" id="build-action-toolbar">
<a href="#" title="Save state manually"
class="icon-link"
data-action="/dump/cache-file"
data-method="POST"><img src="img/icon/content-save.svg" alt="Save state manually" class="icon" /></a>
<a href="#" title="Stop service"
class="icon-link"
data-action="/quit"