From 557fd1a738bb6d2b98059bb95d761a5fd25ea9d7 Mon Sep 17 00:00:00 2001 From: Martchus Date: Sat, 5 Feb 2022 22:09:52 +0100 Subject: [PATCH] lmdb: Use lmdb to store build actions --- 3rdparty/lmdb-safe | 2 +- README.md | 11 +- libpkg/CMakeLists.txt | 1 + libpkg/data/storagegeneric.h | 162 +++++++ libpkg/data/storageprivate.h | 155 +------ librepomgr/CMakeLists.txt | 17 +- librepomgr/buildactions/buildaction.cpp | 81 ++-- librepomgr/buildactions/buildaction.h | 14 +- librepomgr/buildactions/buildactionfwd.h | 5 +- librepomgr/buildactions/buildactionprivate.h | 2 +- librepomgr/logcontext.h | 3 +- librepomgr/serversetup.cpp | 451 ++++++++----------- librepomgr/serversetup.h | 58 ++- librepomgr/tests/buildactions.cpp | 16 +- librepomgr/tests/webapi.cpp | 73 +-- librepomgr/webapi/routes.cpp | 10 - librepomgr/webapi/routes.h | 1 - librepomgr/webapi/routes_buildaction.cpp | 171 +++---- librepomgr/webapi/server.cpp | 1 - srv/static/index.html | 4 - 20 files changed, 597 insertions(+), 641 deletions(-) create mode 100644 libpkg/data/storagegeneric.h diff --git a/3rdparty/lmdb-safe b/3rdparty/lmdb-safe index fbcdc05..1517bdc 160000 --- a/3rdparty/lmdb-safe +++ b/3rdparty/lmdb-safe @@ -1 +1 @@ -Subproject commit fbcdc05e9c389ec93d5eb05e92ece188aeb48239 +Subproject commit 1517bdce2931353f6d929fe3ad9bd03ce10a66ab diff --git a/README.md b/README.md index f347626..e26471f 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ repositories. It is built on top of the official tools provided by the `pacman` At this point the project is rather raw and there are many things left to implement and to improve (checkout the TODOs section below). Currently builds are exclusively triggered -manually (although the API brings the possibility for automation), scalability is limited -through the partial use of an in-memory database. Builds are only conducted on one host. +manually (although the API brings the possibility for automation) and are only conducted +on one host (although one can simply setup the build service on multiple hosts). On the upside, this project can easily be used to together with other build scripts. It doesn't care if your repository's DB file is updated by another application; just be sure @@ -333,11 +333,8 @@ editing the presets JSON file (e.g. `/etc/buildservice-git/presets.json` in the only `pkgrel` changes or when building from VCS sources or when some sources just remain the same). ## TODOs and further ideas for improvement -* [ ] Use persistent, non-in-memory database (e.g. lmdb or leveldb) to improve scalability - * [x] Use db for packages - * [ ] Use db for build actions * [ ] Allow triggering tasks automatically/periodically -* [ ] Allow to run `makechrootpkg` on a remote host (e.g. via SSH) to work can be spread across multiple hosts +* [ ] Allow to run `makechrootpkg` on a remote host (e.g. via SSH) so work can be spread across multiple hosts * [ ] More advanced search options * [ ] Refresh build action details on the web UI automatically while an action is pending * [ ] Fix permission error when stopping a process @@ -351,7 +348,7 @@ editing the presets JSON file (e.g. `/etc/buildservice-git/presets.json` in the * Start process producing lots of output very fast * Let different clients connect and disconnect fast * [ ] Improve test coverage -* [ ] Add fancy graphs for dependencies on the web UI +* [ ] Add fancy graphs for dependencies of build actions on the web UI * [ ] Include `xterm.js` via JavaScript modules (blocked by https://github.com/xtermjs/xterm.js/issues/2878) ## Build instructions and dependencies diff --git a/libpkg/CMakeLists.txt b/libpkg/CMakeLists.txt index 9ff9cf5..83259d6 100644 --- a/libpkg/CMakeLists.txt +++ b/libpkg/CMakeLists.txt @@ -19,6 +19,7 @@ set(SRC_FILES data/database.cpp data/config.cpp data/lockable.cpp + data/storagegeneric.h data/storageprivate.h data/storage.cpp algo/search.cpp diff --git a/libpkg/data/storagegeneric.h b/libpkg/data/storagegeneric.h new file mode 100644 index 0000000..975acf5 --- /dev/null +++ b/libpkg/data/storagegeneric.h @@ -0,0 +1,162 @@ +#ifndef LIBPKG_DATA_STORAGE_GENERIC_H +#define LIBPKG_DATA_STORAGE_GENERIC_H + +#include "../lmdb-safe/lmdb-reflective.hh" +#include "../lmdb-safe/lmdb-safe.hh" +#include "../lmdb-safe/lmdb-typed.hh" + +#include +#include +#include +#include + +#include +#include + +namespace LibPkg { + +using StorageID = std::uint32_t; + +template struct StorageCacheRef { + using Storage = StorageType; + explicit StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr &entry); + explicit StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName); + bool operator==(const StorageCacheRef &other) const; + const StorageType *relatedStorage = nullptr; + const std::string *entryName; +}; + +template +inline StorageCacheRef::StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr &entry) + : relatedStorage(&relatedStorage) + , entryName(&entry->name) +{ +} + +template +inline StorageCacheRef::StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName) + : relatedStorage(&relatedStorage) + , entryName(&entryName) +{ +} + +template +inline bool StorageCacheRef::operator==(const StorageCacheRef &other) const +{ + return relatedStorage == other.relatedStorage && *entryName == *other.entryName; +} + +template struct StorageCacheEntry { + using Ref = StorageRefType; + using Entry = EntryType; + using Storage = typename Ref::Storage; + explicit StorageCacheEntry(const StorageRefType &ref, StorageID id); + StorageRefType ref; + StorageID id; + std::shared_ptr entry; +}; + +template +inline StorageCacheEntry::StorageCacheEntry(const StorageRefType &ref, StorageID id) + : ref(ref) + , id(id) +{ +} + +template struct StorageEntryByID { + struct result_type { + StorageID id = 0; + const typename StorageEntryType::Storage *storage = nullptr; + + bool operator==(const result_type &other) const + { + return id == other.id && storage == other.storage; + } + }; + + result_type operator()(const StorageEntryType &storageEntry) const + { + return result_type{ storageEntry.id, storageEntry.ref.relatedStorage }; + } +}; + +template class StorageCacheEntries { +public: + using Ref = typename StorageEntryType::Ref; + using Entry = typename StorageEntryType::Entry; + using Storage = typename StorageEntryType::Storage; + using StorageEntry = StorageEntryType; + using ByID = StorageEntryByID; + using EntryList = boost::multi_index::multi_index_container, + boost::multi_index::hashed_unique, ByID>, + boost::multi_index::hashed_unique, BOOST_MULTI_INDEX_MEMBER(StorageEntryType, Ref, ref)>>>; + using iterator = typename EntryList::iterator; + + explicit StorageCacheEntries(std::size_t limit = 1000); + + template StorageEntry *find(const IndexType &ref); + StorageEntry &insert(StorageEntry &&entry); + std::size_t erase(const Ref &ref); + std::size_t clear(const Storage &storage); + iterator begin(); + iterator end(); + void setLimit(std::size_t limit); + +private: + EntryList m_entries; + std::size_t m_limit; +}; + +template +inline StorageCacheEntries::StorageCacheEntries(std::size_t limit) + : m_limit(limit) +{ +} + +template inline std::size_t StorageCacheEntries::erase(const Ref &ref) +{ + return m_entries.template get().erase(ref); +} + +template inline auto StorageCacheEntries::begin() -> iterator +{ + return m_entries.begin(); +} + +template inline auto StorageCacheEntries::end() -> iterator +{ + return m_entries.end(); +} + +template struct StorageCache { + using Entries = StorageEntriesType; + using Entry = typename Entries::Entry; + using ROTxn = typename StorageType::ROTransaction; + using RWTxn = typename StorageType::RWTransaction; + using Storage = typename Entries::Storage; + struct StoreResult { + StorageID id = 0; + bool updated = false; + std::shared_ptr oldEntry; + }; + + SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID); + SpecType retrieve(Storage &storage, StorageID storageID); + SpecType retrieve(Storage &storage, RWTxn *, const std::string &entryName); + SpecType retrieve(Storage &storage, const std::string &entryName); + StoreResult store(Storage &storage, const std::shared_ptr &entry, bool force); + StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr &entry); + bool invalidate(Storage &storage, const std::string &entryName); + void clear(Storage &storage); + void clearCacheOnly(Storage &storage); + void setLimit(std::size_t limit); + +private: + Entries m_entries; + std::mutex m_mutex; +}; + +} // namespace LibPkg + +#endif // LIBPKG_DATA_STORAGE_GENERIC_H diff --git a/libpkg/data/storageprivate.h b/libpkg/data/storageprivate.h index 0612859..0be0433 100644 --- a/libpkg/data/storageprivate.h +++ b/libpkg/data/storageprivate.h @@ -2,163 +2,10 @@ #define LIBPKG_DATA_STORAGE_PRIVATE_H #include "./package.h" - -#include "../lmdb-safe/lmdb-reflective.hh" -#include "../lmdb-safe/lmdb-safe.hh" -#include "../lmdb-safe/lmdb-typed.hh" - -#include -#include -#include -#include - -#include -#include +#include "./storagegeneric.h" namespace LibPkg { -using StorageID = std::uint32_t; - -template struct StorageCacheRef { - using Storage = StorageType; - explicit StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr &entry); - explicit StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName); - bool operator==(const StorageCacheRef &other) const; - const StorageType *relatedStorage = nullptr; - const std::string *entryName; -}; - -template -inline StorageCacheRef::StorageCacheRef(const StorageType &relatedStorage, const std::shared_ptr &entry) - : relatedStorage(&relatedStorage) - , entryName(&entry->name) -{ -} - -template -inline StorageCacheRef::StorageCacheRef(const StorageType &relatedStorage, const std::string &entryName) - : relatedStorage(&relatedStorage) - , entryName(&entryName) -{ -} - -template -inline bool StorageCacheRef::operator==(const StorageCacheRef &other) const -{ - return relatedStorage == other.relatedStorage && *entryName == *other.entryName; -} - -template struct StorageCacheEntry { - using Ref = StorageRefType; - using Entry = EntryType; - using Storage = typename Ref::Storage; - explicit StorageCacheEntry(const StorageRefType &ref, StorageID id); - StorageRefType ref; - StorageID id; - std::shared_ptr entry; -}; - -template -inline StorageCacheEntry::StorageCacheEntry(const StorageRefType &ref, StorageID id) - : ref(ref) - , id(id) -{ -} - -template struct StorageEntryByID { - struct result_type { - StorageID id = 0; - const typename StorageEntryType::Storage *storage = nullptr; - - bool operator==(const result_type &other) const - { - return id == other.id && storage == other.storage; - } - }; - - result_type operator()(const StorageEntryType &storageEntry) const - { - return result_type{ storageEntry.id, storageEntry.ref.relatedStorage }; - } -}; - -template class StorageCacheEntries { -public: - using Ref = typename StorageEntryType::Ref; - using Entry = typename StorageEntryType::Entry; - using Storage = typename StorageEntryType::Storage; - using StorageEntry = StorageEntryType; - using ByID = StorageEntryByID; - using EntryList = boost::multi_index::multi_index_container, - boost::multi_index::hashed_unique, ByID>, - boost::multi_index::hashed_unique, BOOST_MULTI_INDEX_MEMBER(StorageEntryType, Ref, ref)>>>; - using iterator = typename EntryList::iterator; - - explicit StorageCacheEntries(std::size_t limit = 1000); - - template StorageEntry *find(const IndexType &ref); - StorageEntry &insert(StorageEntry &&entry); - std::size_t erase(const Ref &ref); - std::size_t clear(const Storage &storage); - iterator begin(); - iterator end(); - void setLimit(std::size_t limit); - -private: - EntryList m_entries; - std::size_t m_limit; -}; - -template -inline StorageCacheEntries::StorageCacheEntries(std::size_t limit) - : m_limit(limit) -{ -} - -template inline std::size_t StorageCacheEntries::erase(const Ref &ref) -{ - return m_entries.template get().erase(ref); -} - -template inline auto StorageCacheEntries::begin() -> iterator -{ - return m_entries.begin(); -} - -template inline auto StorageCacheEntries::end() -> iterator -{ - return m_entries.end(); -} - -template struct StorageCache { - using Entries = StorageEntriesType; - using Entry = typename Entries::Entry; - using ROTxn = typename StorageType::ROTransaction; - using RWTxn = typename StorageType::RWTransaction; - using Storage = typename Entries::Storage; - struct StoreResult { - StorageID id = 0; - bool updated = false; - std::shared_ptr oldEntry; - }; - - SpecType retrieve(Storage &storage, ROTxn *, StorageID storageID); - SpecType retrieve(Storage &storage, StorageID storageID); - SpecType retrieve(Storage &storage, RWTxn *, const std::string &entryName); - SpecType retrieve(Storage &storage, const std::string &entryName); - StoreResult store(Storage &storage, const std::shared_ptr &entry, bool force); - StoreResult store(Storage &storage, RWTxn &txn, const std::shared_ptr &entry); - bool invalidate(Storage &storage, const std::string &entryName); - void clear(Storage &storage); - void clearCacheOnly(Storage &storage); - void setLimit(std::size_t limit); - -private: - Entries m_entries; - std::mutex m_mutex; -}; - using PackageStorage = LMDBSafe::TypedDBI>; using DependencyStorage = LMDBSafe::TypedDBI>; using LibraryDependencyStorage diff --git a/librepomgr/CMakeLists.txt b/librepomgr/CMakeLists.txt index eca1057..c34a917 100644 --- a/librepomgr/CMakeLists.txt +++ b/librepomgr/CMakeLists.txt @@ -71,9 +71,6 @@ set(META_APP_DESCRIPTION "Library for managing custom Arch Linux repositories") set(META_VERSION_MAJOR 0) set(META_VERSION_MINOR 0) set(META_VERSION_PATCH 1) -set(META_VERSION_CACHE 11) -set(META_VERSION_BUILD_ACTIONS_JSON 0) -set(META_VERSION_LIBRARY_DEPENDENCIES_JSON 0) set(LINK_TESTS_AGAINST_APP_TARGET ON) # find c++utilities @@ -89,9 +86,10 @@ set(Boost_USE_MULTITHREADED ON) if (BOOST_STATIC_LINKAGE) set(Boost_USE_STATIC_LIBS ON) endif () -set(BOOST_ARGS "REQUIRED;COMPONENTS;system;filesystem") +set(BOOST_ARGS "REQUIRED;COMPONENTS;system;filesystem;iostreams") use_package(TARGET_NAME Boost::system PACKAGE_NAME Boost PACKAGE_ARGS "${BOOST_ARGS}") use_package(TARGET_NAME Boost::filesystem PACKAGE_NAME Boost PACKAGE_ARGS "${BOOST_ARGS}") +use_package(TARGET_NAME Boost::iostreams PACKAGE_NAME Boost PACKAGE_ARGS "${BOOST_ARGS}") option(BOOST_ASIO_IO_URING OFF "enable use of io_uring") if (BOOST_ASIO_IO_URING) list(APPEND META_PUBLIC_COMPILE_DEFINITIONS BOOST_ASIO_HAS_IO_URING BOOST_ASIO_DISABLE_EPOLL) @@ -106,6 +104,10 @@ use_reflective_rapidjson(VISIBILITY PUBLIC) find_package(libpkg ${META_APP_VERSION} REQUIRED) use_libpkg(VISIBILITY PUBLIC) +# find lmdb-safe +find_package(lmdb-safe${CONFIGURATION_PACKAGE_SUFFIX} REQUIRED) +use_lmdb_safe() + # link against crypto and SSL library from OpenSSL use_openssl(VISIBILITY PUBLIC) @@ -115,13 +117,6 @@ list(APPEND PUBLIC_LIBRARIES pthread) # apply basic configuration include(BasicConfig) -# add cache version to config header -string(APPEND META_CUSTOM_CONFIG "#define ${META_PROJECT_VARNAME}_CACHE_VERSION \"${META_VERSION_CACHE}\"\n") -string(APPEND META_CUSTOM_CONFIG - "#define ${META_PROJECT_VARNAME}_BUILD_ACTIONS_JSON_VERSION \"${META_VERSION_BUILD_ACTIONS_JSON}\"\n") -string(APPEND META_CUSTOM_CONFIG - "#define ${META_PROJECT_VARNAME}_LIBRARY_DEPENDENCIES_JSON_VERSION \"${META_VERSION_LIBRARY_DEPENDENCIES_JSON}\"\n") - # trigger code generator for tests because the tests already contain structs to be (de)serialized include(ReflectionGenerator) add_reflection_generator_invocation( diff --git a/librepomgr/buildactions/buildaction.cpp b/librepomgr/buildactions/buildaction.cpp index 76717fd..cc398c9 100644 --- a/librepomgr/buildactions/buildaction.cpp +++ b/librepomgr/buildactions/buildaction.cpp @@ -204,6 +204,43 @@ BuildAction::BuildAction(IdType id, ServiceSetup *setup) noexcept { } +BuildAction &BuildAction::operator=(BuildAction &&other) +{ + if (this == &other) { + return *this; + } + id = other.id; + taskName = std::move(other.taskName); + templateName = std::move(other.templateName); + directory = std::move(other.directory); + packageNames = std::move(other.packageNames); + sourceDbs = std::move(other.sourceDbs); + destinationDbs = std::move(other.destinationDbs); + settings = std::move(other.settings); + flags = other.flags; + type = other.type; + status = other.status; + result = other.result; + resultData = std::move(other.resultData); + output = std::move(other.output); + outputMimeType = std::move(other.outputMimeType); + logfiles = std::move(other.logfiles); + artefacts = std::move(other.artefacts); + created = other.created; + started = other.started; + finished = other.finished; + startAfter = std::move(other.startAfter); + m_log = std::move(other.m_log); + m_setup = other.m_setup; + m_aborted = false; + m_stopHandler = std::function(); + m_concludeHandler = std::function(); + m_ongoingProcesses.clear(); + m_bufferingForSession.clear(); + m_internalBuildAction = std::move(other.m_internalBuildAction); + return *this; +} + BuildAction::~BuildAction() { } @@ -223,10 +260,10 @@ bool BuildAction::haveSucceeded(const std::vector> * the build action is setup-globally visible. * \returns Returns immediately. The real work is done in a build action thread. */ -void BuildAction::start(ServiceSetup &setup) +LibPkg::StorageID BuildAction::start(ServiceSetup &setup) { if (!isScheduled()) { - return; + return 0; } started = DateTime::gmtNow(); @@ -236,8 +273,7 @@ void BuildAction::start(ServiceSetup &setup) switch (type) { case BuildActionType::Invalid: resultData = "type is invalid"; - conclude(BuildActionResult::Failure); - break; + return conclude(BuildActionResult::Failure); case BuildActionType::RemovePackages: post(); break; @@ -281,31 +317,17 @@ void BuildAction::start(ServiceSetup &setup) break; default: resultData = "not implemented yet or invalid type"; - conclude(BuildActionResult::Failure); + return conclude(BuildActionResult::Failure); } -} -void BuildAction::startAfterOtherBuildActions(ServiceSetup &setup, const std::vector> &startsAfterBuildActions) -{ - auto allSucceeded = true; - for (auto &previousBuildAction : startsAfterBuildActions) { - if (!previousBuildAction->hasSucceeded()) { - previousBuildAction->m_followUpActions.emplace_back(weak_from_this()); - allSucceeded = false; - } - } - if (allSucceeded) { - start(setup); - } + // update in persistent storage and create entry in "running cache" + return m_setup->building.storeBuildAction(shared_from_this()); } void BuildAction::assignStartAfter(const std::vector> &startsAfterBuildActions) { for (auto &previousBuildAction : startsAfterBuildActions) { startAfter.emplace_back(previousBuildAction->id); - if (!previousBuildAction->hasSucceeded()) { - previousBuildAction->m_followUpActions.emplace_back(weak_from_this()); - } } } @@ -333,7 +355,7 @@ template void BuildAction::post(Callback &&codeToRun) /*! * \brief Internally called to conclude the build action. */ -void BuildAction::conclude(BuildActionResult result) +LibPkg::StorageID BuildAction::conclude(BuildActionResult result) { // set fields accordingly status = BuildActionStatus::Finished; @@ -354,19 +376,26 @@ void BuildAction::conclude(BuildActionResult result) // start globally visible follow-up actions if succeeded if (result == BuildActionResult::Success && m_setup) { - for (auto &maybeStillValidFollowUpAction : m_followUpActions) { - auto followUpAction = maybeStillValidFollowUpAction.lock(); - if (followUpAction && followUpAction->isScheduled() - && BuildAction::haveSucceeded(m_setup->building.getBuildActions(followUpAction->startAfter))) { + const auto followUps = m_setup->building.followUpBuildActions(id); + for (auto &followUpAction : followUps) { + if (followUpAction->isScheduled() && BuildAction::haveSucceeded(m_setup->building.getBuildActions(followUpAction->startAfter))) { followUpAction->start(*m_setup); } } // note: Not cleaning up the follow-up actions here because at some point I might implement recursive restarting. } + // write build action to persistent storage + // TODO: should this also be done in the middle of the execution to have some "save points"? + auto id = LibPkg::StorageID(); + if (m_setup && m_setup->building.hasStorage()) { + id = m_setup->building.storeBuildAction(shared_from_this()); + } + if (m_concludeHandler) { m_concludeHandler(); } + return id; } #ifdef LIBREPOMGR_DUMMY_BUILD_ACTION_ENABLED diff --git a/librepomgr/buildactions/buildaction.h b/librepomgr/buildactions/buildaction.h index f7e6295..1b2ab98 100644 --- a/librepomgr/buildactions/buildaction.h +++ b/librepomgr/buildactions/buildaction.h @@ -173,6 +173,7 @@ public: static constexpr IdType invalidId = std::numeric_limits::max(); explicit BuildAction(IdType id = invalidId, ServiceSetup *setup = nullptr) noexcept; + BuildAction &operator=(BuildAction &&other); ~BuildAction(); bool isScheduled() const; bool isExecuting() const; @@ -180,8 +181,7 @@ public: bool hasSucceeded() const; static bool haveSucceeded(const std::vector> &buildActions); bool isAborted() const; - void start(ServiceSetup &setup); - void startAfterOtherBuildActions(ServiceSetup &setup, const std::vector> &startsAfterBuildActions); + LibPkg::StorageID start(ServiceSetup &setup); void assignStartAfter(const std::vector> &startsAfterBuildActions); void abort(); void appendOutput(std::string &&output); @@ -197,12 +197,13 @@ public: void terminateOngoingBuildProcesses(); void streamFile(const WebAPI::Params ¶ms, const std::string &filePath, std::string_view fileMimeType); void streamOutput(const WebAPI::Params ¶ms, std::size_t offset = 0); + ServiceSetup *setup(); protected: private: template void post(); template void post(Callback &&codeToRun); - void conclude(BuildActionResult result); + LibPkg::StorageID conclude(BuildActionResult result); void continueStreamingExistingOutputToSession(std::shared_ptr session, OutputBufferingForSession &buffering, const boost::system::error_code &error, std::size_t bytesTransferred); void continueStreamingNewOutputToSession(std::shared_ptr session, OutputBufferingForSession &buffering, @@ -216,7 +217,6 @@ public: std::string directory; std::vector packageNames; std::vector sourceDbs, destinationDbs; - std::vector extraParams; // deprecated; remove at some point std::unordered_map settings; BuildActionFlagType flags = noBuildActionFlags; BuildActionType type = BuildActionType::Invalid; @@ -248,7 +248,6 @@ private: std::mutex m_outputStreamingMutex; std::unordered_map, std::unique_ptr> m_bufferingForSession; std::unique_ptr m_internalBuildAction; - std::vector> m_followUpActions; }; inline bool BuildAction::isScheduled() const @@ -297,6 +296,11 @@ inline std::shared_ptr BuildAction::findBuildProcess(const return i != m_ongoingProcesses.cend() ? i->second : nullptr; } +inline ServiceSetup *BuildAction::setup() +{ + return m_setup; +} + /*! * \brief Appends the specified arguments to the build action's log but *not* to the overall service log. */ diff --git a/librepomgr/buildactions/buildactionfwd.h b/librepomgr/buildactions/buildactionfwd.h index 1049acd..1b9d535 100644 --- a/librepomgr/buildactions/buildactionfwd.h +++ b/librepomgr/buildactions/buildactionfwd.h @@ -1,13 +1,12 @@ #ifndef LIBREPOMGR_BUILD_ACTION_FWD_H #define LIBREPOMGR_BUILD_ACTION_FWD_H -#include -#include +#include "../../libpkg/data/storagefwd.h" namespace LibRepoMgr { struct BuildAction; -using BuildActionIdType = std::vector>::size_type; // build actions are stored in a vector and the ID is used as index +using BuildActionIdType = std::size_t; } // namespace LibRepoMgr diff --git a/librepomgr/buildactions/buildactionprivate.h b/librepomgr/buildactions/buildactionprivate.h index 0a638e7..3c6a121 100644 --- a/librepomgr/buildactions/buildactionprivate.h +++ b/librepomgr/buildactions/buildactionprivate.h @@ -276,7 +276,7 @@ template void BuildProcessSession::launch(ChildArgs &&.. struct ProcessResult; /// \brief The InternalBuildAction struct contains internal details (which are not serialized / accessible via the web API) and helpers. -/// \remarks This struct is mean to be inherited from when creating a new type build action like it is down by the classes below. +/// \remarks This struct is mean to be inherited from when creating a new type of build action like it is done by the classes below. struct LIBREPOMGR_EXPORT InternalBuildAction { InternalBuildAction(ServiceSetup &setup, const std::shared_ptr &buildAction); diff --git a/librepomgr/logcontext.h b/librepomgr/logcontext.h index 3cdfe6a..6e8da11 100644 --- a/librepomgr/logcontext.h +++ b/librepomgr/logcontext.h @@ -15,12 +15,13 @@ struct BuildAction; struct LIBREPOMGR_EXPORT LogContext { explicit LogContext(BuildAction *buildAction = nullptr); LogContext &operator=(const LogContext &) = delete; + LogContext &operator=(LogContext &&) = default; template LogContext &operator()(CppUtilities::EscapeCodes::Phrases phrase, Args &&...args); template LogContext &operator()(Args &&...args); template LogContext &operator()(std::string &&msg); private: - BuildAction *const m_buildAction; + BuildAction *m_buildAction; }; inline LogContext::LogContext(BuildAction *buildAction) diff --git a/librepomgr/serversetup.cpp b/librepomgr/serversetup.cpp index f597523..f32b99c 100644 --- a/librepomgr/serversetup.cpp +++ b/librepomgr/serversetup.cpp @@ -7,6 +7,8 @@ #include "./webapi/server.h" +#include "../libpkg/data/storagegeneric.h" + #include "reflection/serversetup.h" #include "resources/config.h" @@ -38,6 +40,24 @@ using namespace LibPkg; namespace LibRepoMgr { +struct Storage { + using BuildActionStorage = LMDBSafe::TypedDBI; + + explicit Storage(const char *path); + +private: + std::shared_ptr m_env; + +public: + BuildActionStorage buildActions; +}; + +Storage::Storage(const char *path) + : m_env(LMDBSafe::getMDBEnv(path, MDB_NOSUBDIR, 0600, 2)) + , buildActions(m_env, "buildactions") +{ +} + static void deduplicateVector(std::vector &vector) { std::unordered_set visited; @@ -96,6 +116,16 @@ void ServiceSetup::WebServerSetup::applyConfig(const std::multimap(path); + } +} + void ServiceSetup::BuildSetup::applyConfig(const std::multimap &multimap) { convertValue(multimap, "threads", threadCount); @@ -121,6 +151,7 @@ void ServiceSetup::BuildSetup::applyConfig(const std::multimapbuildActions.getRWTransaction(); + const auto id = txn.put(emptyBuildAction); + txn.commit(); return id; } -std::vector> ServiceSetup::BuildSetup::getBuildActions(const std::vector &ids) +std::shared_ptr ServiceSetup::BuildSetup::getBuildAction(BuildActionIdType id) { + if (auto i = m_runningActions.find(id); i != m_runningActions.end()) { + return i->second; + } + const auto res = std::make_shared(); + auto txn = m_storage->buildActions.getROTransaction(); + return id <= std::numeric_limits::max() && txn.get(static_cast(id), *res) ? res : nullptr; +} + +std::vector> ServiceSetup::BuildSetup::getBuildActions(const std::vector &ids) +{ + auto buildAction = std::shared_ptr(); auto buildActions = std::vector>(); buildActions.reserve(ids.size()); + auto txn = m_storage->buildActions.getROTransaction(); for (const auto id : ids) { - if (id < actions.size()) { - if (auto &buildAction = actions[id]) { - buildActions.emplace_back(buildAction); - } + if (auto i = m_runningActions.find(id); i != m_runningActions.end()) { + buildActions.emplace_back(i->second); + continue; + } + if (id > std::numeric_limits::max()) { + continue; + } + if (!buildAction) { + buildAction = std::make_shared(); + } + if (txn.get(static_cast(id), *buildAction)) { + buildActions.emplace_back(std::move(buildAction)); } } return buildActions; } +StorageID ServiceSetup::BuildSetup::storeBuildAction(const std::shared_ptr &buildAction) +{ + // update cache of running build actions + if (buildAction->isExecuting()) { + m_runningActions[buildAction->id] = buildAction; + } else { + m_runningActions.erase(buildAction->id); + } + // update index of follow-up actions for scheduled actions + // note: This would break if startAfter would be modified after the initial build action creation. + if (buildAction->isScheduled() && !buildAction->startAfter.empty()) { + const auto previousBuildActions = getBuildActions(buildAction->startAfter); + auto allSucceeded = false; + for (auto &previousBuildAction : previousBuildActions) { + if (!previousBuildAction->hasSucceeded()) { + m_followUpActions[previousBuildAction->id].emplace(buildAction->id); + allSucceeded = false; + } + } + // immediately start if all follow-up actions have succeeded + if (allSucceeded && buildAction->setup()) { + return buildAction->start(*buildAction->setup()); + } + } else { + for (const auto id : buildAction->startAfter) { + if (const auto i = m_followUpActions.find(id); i != m_followUpActions.end()) { + auto &followUps = i->second; + followUps.erase(buildAction->id); + if (followUps.empty()) { + m_followUpActions.erase(i); + } + } + } + } + // update persistent storage + auto txn = m_storage->buildActions.getRWTransaction(); + const auto id = txn.put(*buildAction, static_cast(buildAction->id)); // buildAction->id expected to be a valid StorageID or 0 + txn.commit(); + return id; +} + +void ServiceSetup::BuildSetup::deleteBuildAction(const std::vector> &actions) +{ + auto txn = m_storage->buildActions.getRWTransaction(); + for (const auto &action : actions) { + // remove action from cache for running actions + m_runningActions.erase(action->id); + // remove any actions to start after the action to delete + m_followUpActions.erase(action->id); + // remove follow-up indexes for actions previous to the action to delete + for (auto i = m_followUpActions.begin(), end = m_followUpActions.end(); i != end;) { + auto &followUps = i->second; + followUps.erase(action->id); + if (followUps.empty()) { + i = m_followUpActions.erase(i); + } else { + ++i; + } + } + // delete action from storage + if (action->id && action->id <= std::numeric_limits::max()) { + txn.del(static_cast(action->id)); + } + } + txn.commit(); +} + +std::size_t ServiceSetup::BuildSetup::buildActionCount() +{ + return m_storage->buildActions.getROTransaction().size(); +} + +void ServiceSetup::BuildSetup::forEachBuildAction( + std::function count, std::function &&func) +{ + auto txn = m_storage->buildActions.getROTransaction(); + count(txn.size()); + for (auto i = txn.begin(); i != txn.end(); ++i) { + if (func(i.getID(), std::move(i.value()))) { + return; + } + } +} + +void ServiceSetup::BuildSetup::forEachBuildAction(std::function &&func) +{ + auto txn = m_storage->buildActions.getRWTransaction(); + for (auto i = txn.begin(); i != txn.end(); ++i) { + const auto running = m_runningActions.find(i.getID()); + auto &action = running != m_runningActions.end() ? *running->second : i.value(); + auto save = false; + const auto stop = func(i.getID(), action, save); + if (save) { + txn.put(action, i.getID()); + } + if (stop) { + return; + } + } +} + +std::vector> ServiceSetup::BuildSetup::followUpBuildActions(BuildActionIdType forId) +{ + auto res = std::vector>(); + const auto i = m_followUpActions.find(forId); + if (i == m_followUpActions.end()) { + return res; + } + res.reserve(i->second.size()); + for (const auto followUpId : i->second) { + if (auto buildAction = getBuildAction(followUpId)) { + res.emplace_back(std::move(buildAction)); + } + } + return res; +} + void ServiceSetup::loadConfigFiles(bool doFirstTimeSetup) { // read config file @@ -387,244 +551,29 @@ void ServiceSetup::printDatabases() cerr << Phrases::SubMessage << "AUR (" << config.aur.packageCount() << " packages cached)" << Phrases::End; } -std::string_view ServiceSetup::cacheFilePath() const +void ServiceSetup::restoreState() { - return "cache-v" LIBREPOMGR_CACHE_VERSION ".bin"; -} - -RAPIDJSON_NAMESPACE::Document ServiceSetup::libraryDependenciesToJson() -{ - namespace JR = ReflectiveRapidJSON::JsonReflector; - auto document = RAPIDJSON_NAMESPACE::Document(RAPIDJSON_NAMESPACE::kObjectType); - auto &alloc = document.GetAllocator(); - for (auto &db : config.databases) { - auto dbValue = RAPIDJSON_NAMESPACE::Value(RAPIDJSON_NAMESPACE::Type::kObjectType); - db.allPackages([&](StorageID, const std::shared_ptr &package) { - if (!package->packageInfo) { - return false; - } - if (package->libdepends.empty() && package->libprovides.empty()) { - auto hasVersionedPythonOrPerlDep = false; - for (const auto &dependency : package->dependencies) { - if (dependency.mode == DependencyMode::Any || dependency.version.empty() - || (dependency.name != "python" && dependency.name != "python2" && dependency.name != "perl")) { - return false; - } - hasVersionedPythonOrPerlDep = true; - break; - } - if (!hasVersionedPythonOrPerlDep) { - return false; - } - } - auto pkgValue = RAPIDJSON_NAMESPACE::Value(RAPIDJSON_NAMESPACE::Type::kObjectType); - auto pkgObj = pkgValue.GetObject(); - JR::push(package->version, "v", pkgObj, alloc); - JR::push(package->packageInfo->buildDate, "t", pkgObj, alloc); - JR::push(package->dependencies, "d", pkgObj, alloc); // for versioned Python/Perl deps - JR::push(package->libdepends, "ld", pkgObj, alloc); - JR::push(package->libprovides, "lp", pkgObj, alloc); - dbValue.AddMember(RAPIDJSON_NAMESPACE::StringRef(package->name.data(), JR::rapidJsonSize(package->name.size())), pkgValue, alloc); - return false; - }); - document.AddMember(RAPIDJSON_NAMESPACE::Value(db.name % '@' + db.arch, alloc), dbValue, alloc); - } - return document; -} - -void ServiceSetup::restoreLibraryDependenciesFromJson(const string &json, ReflectiveRapidJSON::JsonDeserializationErrors *errors) -{ - namespace JR = ReflectiveRapidJSON::JsonReflector; - const auto document = JR::parseJsonDocFromString(json.data(), json.size()); - if (!document.IsObject()) { - errors->reportTypeMismatch>(document.GetType()); - return; - } - // FIXME: be more error resilient here, e.g. set the following line and print list of errors instead of aborting on first error - // errors->throwOn = ReflectiveRapidJSON::JsonDeserializationErrors::ThrowOn::None; - const auto dbObj = document.GetObject(); - for (const auto &dbEntry : dbObj) { - if (!dbEntry.value.IsObject()) { - errors->reportTypeMismatch(document.GetType()); - continue; - } - auto *const db = config.findOrCreateDatabaseFromDenotation(std::string_view(dbEntry.name.GetString())); - const auto pkgsObj = dbEntry.value.GetObject(); - for (const auto &pkgEntry : pkgsObj) { - if (!pkgEntry.value.IsObject()) { - errors->reportTypeMismatch(document.GetType()); - continue; - } - const auto pkgObj = pkgEntry.value.GetObject(); - auto name = std::string(pkgEntry.name.GetString()); - auto [pkgID, pkg] = db->findPackageWithID(name); - if (pkg) { - // do not mess with already existing packages; this restoring stuff is supposed to be done before loading packages from DBs - continue; - } - pkg = std::make_shared(); - pkg->name = std::move(name); - pkg->origin = PackageOrigin::CustomSource; - pkg->packageInfo = std::make_unique(); - JR::pull(pkg->version, "v", pkgObj, errors); - JR::pull(pkg->packageInfo->buildDate, "t", pkgObj, errors); - JR::pull(pkg->dependencies, "d", pkgObj, errors); // for versioned Python/Perl deps - JR::pull(pkg->libdepends, "ld", pkgObj, errors); - JR::pull(pkg->libprovides, "lp", pkgObj, errors); - db->updatePackage(pkg); - } - } -} - -std::size_t ServiceSetup::restoreState() -{ - // clear old build actions before (There must not be any ongoing build actions when calling this function!) - building.actions.clear(); - building.invalidActions.clear(); - - // restore configuration and maybe build actions from JSON file - const auto cacheFilePath = this->cacheFilePath(); - std::size_t size = 0; - bool hasConfig = false, hasBuildActions = false; - try { - fstream cacheFile; - cacheFile.exceptions(ios_base::failbit | ios_base::badbit); - cacheFile.open(cacheFilePath.data(), ios_base::in | ios_base::binary); - ReflectiveRapidJSON::BinaryReflector::BinaryDeserializer deserializer(&cacheFile); - deserializer.read(config); - hasConfig = true; - if (!hasBuildActions) { - deserializer.read(building.actions); - hasBuildActions = true; - } - size = static_cast(cacheFile.tellg()); - cacheFile.close(); - cerr << Phrases::SuccessMessage << "Restored cache file \"" << cacheFilePath << "\", " << dataSizeToString(size) << Phrases::EndFlush; - if (hasBuildActions) { - cerr << Phrases::SubMessage << "Restored build actions from cache file 😌" << Phrases::EndFlush; - } - } catch (const ConversionException &) { - cerr << Phrases::WarningMessage << "A conversion error occurred when restoring cache file \"" << cacheFilePath << "\"." << Phrases::EndFlush; - } catch (const ios_base::failure &) { - cerr << Phrases::WarningMessage << "An IO error occurred when restoring cache file \"" << cacheFilePath << "\"." << Phrases::EndFlush; - } - // open LMDB storage - cout << Phrases::InfoMessage << "Opening LMDB file: " << dbPath << " (max DBs: " << maxDbs << ')' << Phrases::EndFlush; + cout << Phrases::InfoMessage << "Opening config LMDB file: " << dbPath << " (max DBs: " << maxDbs << ')' << Phrases::EndFlush; config.initStorage(dbPath.data(), maxDbs); config.setPackageCacheLimit(packageCacheLimit); - - // restore build actions from JSON file - if (!hasBuildActions) { - try { - if (!restoreJsonObject(building.actions, workingDirectory, "build-actions-v" LIBREPOMGR_BUILD_ACTIONS_JSON_VERSION, - RestoreJsonExistingFileHandling::Skip) - .empty()) { - cerr << Phrases::SuccessMessage << "Restored build actions from JSON file 😒" << Phrases::EndFlush; - } - } catch (const std::runtime_error &e) { - cerr << Phrases::ErrorMessage << e.what() << Phrases::EndFlush; - } - } - - // restore provided/required libraries from JSON file - if (!hasConfig) { - try { - if (!restoreJsonObject(std::bind(&ServiceSetup::restoreLibraryDependenciesFromJson, this, std::placeholders::_1, std::placeholders::_2), - workingDirectory, "library-dependencies-v" LIBREPOMGR_LIBRARY_DEPENDENCIES_JSON_VERSION, RestoreJsonExistingFileHandling::Skip) - .empty()) { - cerr << Phrases::SuccessMessage << "Restored library dependencies from JSON file 😒" << Phrases::EndFlush; - } - } catch (const std::runtime_error &e) { - cerr << Phrases::ErrorMessage << e.what() << Phrases::EndFlush; - } - } - - // determine invalid build actions - if (building.actions.empty()) { - return size; - } - auto newActionsSize = building.actions.size(); - for (auto id = newActionsSize - 1;; --id) { - if (building.actions[id]) { - break; - } - newActionsSize = id; - if (!newActionsSize) { - break; - } - } - building.actions.resize(newActionsSize); - for (std::size_t buildActionId = 0u, buildActionsSize = building.actions.size(); buildActionId != buildActionsSize; ++buildActionId) { - if (!building.actions[buildActionId]) { - building.invalidActions.emplace(buildActionId); - } - } + cout << Phrases::InfoMessage << "Opening actions LMDB file: " << building.dbPath << Phrases::EndFlush; + building.initStorage(building.dbPath.data()); // ensure no build actions are considered running anymore and populate follow up actions - for (auto &action : building.actions) { - if (!action) { - continue; - } - for (const auto previousBuildActionID : action->startAfter) { - if (auto previousBuildAction = building.getBuildAction(previousBuildActionID)) { - previousBuildAction->m_followUpActions.emplace_back(action->weak_from_this()); + building.forEachBuildAction([this](LibPkg::StorageID, BuildAction &buildAction, bool &save) { + if (buildAction.isExecuting()) { + buildAction.status = BuildActionStatus::Finished; + buildAction.result = BuildActionResult::Failure; + buildAction.resultData = "service crashed while exectuing"; + save = true; + } else if (buildAction.isScheduled()) { + for (const auto previousBuildActionId : buildAction.startAfter) { + building.m_followUpActions[previousBuildActionId].emplace(buildAction.id); } } - if (action->isExecuting()) { - action->status = BuildActionStatus::Finished; - action->result = BuildActionResult::Failure; - action->resultData = "service crashed while exectuing"; - } - } - - return size; -} - -std::size_t ServiceSetup::saveState() -{ - // write cache file to be able to restore the service state when restarting the service efficiently - const auto cacheFilePath = this->cacheFilePath(); - std::size_t size = 0; - try { - fstream cacheFile; - cacheFile.exceptions(ios_base::failbit | ios_base::badbit); - cacheFile.open(cacheFilePath.data(), ios_base::out | ios_base::trunc | ios_base::binary); - ReflectiveRapidJSON::BinaryReflector::BinarySerializer serializer(&cacheFile); - serializer.write(config); - serializer.write(building.actions); - size = static_cast(cacheFile.tellp()); - cacheFile.close(); - cerr << Phrases::SuccessMessage << "Wrote cache file \"" << cacheFilePath << "\", " << dataSizeToString(size) << Phrases::EndFlush; - } catch (const ios_base::failure &) { - cerr << Phrases::WarningMessage << "An IO error occurred when dumping the cache file \"" << cacheFilePath << "\"." << Phrases::EndFlush; - } - - // write build actions to a JSON file to be able to restore build actions even if the cache file can not be used due to version mismatch - // note: The JSON file's format is hopefully more stable. - try { - if (!dumpJsonObject( - building.actions, workingDirectory, "build-actions-v" LIBREPOMGR_BUILD_ACTIONS_JSON_VERSION, DumpJsonExistingFileHandling::Override) - .empty()) { - cerr << Phrases::SuccessMessage << "Wrote build actions to JSON file." << Phrases::EndFlush; - } - } catch (const std::runtime_error &e) { - cerr << Phrases::ErrorMessage << e.what() << Phrases::EndFlush; - } - - // write provided/required libraries to a JSON file to be able to restore this information even if the cache file can not be used due to version - // mismatch - try { - if (!dumpJsonDocument(std::bind(&ServiceSetup::libraryDependenciesToJson, this), workingDirectory, - "library-dependencies-v" LIBREPOMGR_LIBRARY_DEPENDENCIES_JSON_VERSION, DumpJsonExistingFileHandling::Override) - .empty()) { - cerr << Phrases::SuccessMessage << "Wrote library dependencies to JSON file." << Phrases::EndFlush; - } - } catch (const std::runtime_error &e) { - cerr << Phrases::ErrorMessage << e.what() << Phrases::EndFlush; - } - - return size; + return false; + }); } void ServiceSetup::initStorage() @@ -674,14 +623,14 @@ void ServiceSetup::run() #endif } - for (auto &buildAction : building.actions) { - if (buildAction && buildAction->isExecuting()) { - buildAction->status = BuildActionStatus::Finished; - buildAction->result = BuildActionResult::Aborted; + building.forEachBuildAction([](LibPkg::StorageID, BuildAction &buildAction, bool &save) { + if (buildAction.isExecuting()) { + buildAction.status = BuildActionStatus::Finished; + buildAction.result = BuildActionResult::Aborted; + save = true; } - } - - saveState(); + return false; + }); } void ServiceSetup::Locks::clear() diff --git a/librepomgr/serversetup.h b/librepomgr/serversetup.h index 5274c61..cd7d9e2 100644 --- a/librepomgr/serversetup.h +++ b/librepomgr/serversetup.h @@ -30,6 +30,7 @@ struct LIBREPOMGR_EXPORT ThreadPool { }; struct ServiceStatus; +struct Storage; struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { // the overall configuration (databases, packages, ...) used at various places @@ -46,6 +47,13 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { std::uint32_t maxDbs = 512; std::size_t packageCacheLimit = 1000; + void loadConfigFiles(bool doFirstTimeSetup); + void printDatabases(); + void restoreState(); + void initStorage(); + void run(); + ServiceStatus computeStatus() const; + // variables relevant for the web server; only changed when (re)loading config struct LIBREPOMGR_EXPORT WebServerSetup { // only read by build actions and routes; changed when (re)loading config @@ -68,16 +76,17 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { // variables relevant for build actions and web server routes dealing with them struct LIBREPOMGR_EXPORT BuildSetup : public LibPkg::Lockable { + friend void ServiceSetup::restoreState(); + struct LIBREPOMGR_EXPORT Worker : private boost::asio::executor_work_guard, public ThreadPool { explicit Worker(BuildSetup &setup); ~Worker(); BuildSetup &setup; }; - // read/written by build actions and routes - // -> acquire the build lock for these - std::vector> actions; - std::unordered_set invalidActions; + explicit BuildSetup(); + BuildSetup(BuildSetup &&) = delete; + ~BuildSetup(); // fields which have their own locking // -> acquire the object's lock @@ -113,13 +122,28 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { // never changed after startup unsigned short threadCount = 4; boost::asio::io_context ioContext; + std::string dbPath = "librepomgr.db"; + void initStorage(const char *path); + bool hasStorage() const; void applyConfig(const std::multimap &multimap); void readPresets(const std::string &configFilePath, const std::string &presetsFile); Worker allocateBuildWorker(); - BuildAction::IdType allocateBuildActionID(); - std::shared_ptr getBuildAction(BuildAction::IdType id); - std::vector> getBuildActions(const std::vector &ids); + LibPkg::StorageID allocateBuildActionID(); + std::shared_ptr getBuildAction(BuildActionIdType id); + std::vector> getBuildActions(const std::vector &ids); + LibPkg::StorageID storeBuildAction(const std::shared_ptr &buildAction); + void deleteBuildAction(const std::vector> &actions); + std::size_t buildActionCount(); + std::size_t runningBuildActionCount(); + void forEachBuildAction(std::function count, std::function &&func); + void forEachBuildAction(std::function &&func); + std::vector> followUpBuildActions(BuildActionIdType forId); + + private: + std::unordered_map> m_runningActions; + std::unordered_map> m_followUpActions; + std::unique_ptr m_storage; } building; struct LIBREPOMGR_EXPORT Authentication : public LibPkg::Lockable { @@ -145,22 +169,16 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { std::shared_mutex m_cleanupMutex; LockTable m_locksByName; } locks; - - void loadConfigFiles(bool doFirstTimeSetup); - void printDatabases(); - std::string_view cacheFilePath() const; - RAPIDJSON_NAMESPACE::Document libraryDependenciesToJson(); - void restoreLibraryDependenciesFromJson(const std::string &json, ReflectiveRapidJSON::JsonDeserializationErrors *errors); - std::size_t restoreState(); - std::size_t saveState(); - void initStorage(); - void run(); - ServiceStatus computeStatus() const; }; -inline std::shared_ptr ServiceSetup::BuildSetup::getBuildAction(BuildAction::IdType id) +inline bool ServiceSetup::BuildSetup::hasStorage() const { - return id < actions.size() ? actions[id] : nullptr; + return m_storage != nullptr; +} + +inline std::size_t ServiceSetup::BuildSetup::runningBuildActionCount() +{ + return m_runningActions.size(); } inline GlobalLockable &ServiceSetup::Locks::namedLock(const std::string &lockName) diff --git a/librepomgr/tests/buildactions.cpp b/librepomgr/tests/buildactions.cpp index e3a6d6d..750154c 100644 --- a/librepomgr/tests/buildactions.cpp +++ b/librepomgr/tests/buildactions.cpp @@ -70,7 +70,7 @@ private: void runBuildAction(const char *message, TimeSpan timeout = TimeSpan::fromSeconds(5)); template InternalBuildActionType *internalBuildAction(); - std::string m_dbFile; + std::string m_configDbFile, m_buildingDbFile; ServiceSetup m_setup; std::shared_ptr m_buildAction; std::filesystem::path m_workingDir; @@ -100,8 +100,14 @@ void BuildActionsTests::tearDown() void BuildActionsTests::initStorage() { - m_dbFile = workingCopyPath("test-build-actions.db", WorkingCopyMode::Cleanup); - m_setup.config.initStorage(m_dbFile.data()); + if (!m_setup.config.storage()) { + m_configDbFile = workingCopyPath("test-build-actions-config.db", WorkingCopyMode::Cleanup); + m_setup.config.initStorage(m_configDbFile.data()); + } + if (!m_setup.building.hasStorage()) { + m_buildingDbFile = workingCopyPath("test-build-actions-building.db", WorkingCopyMode::Cleanup); + m_setup.building.initStorage(m_buildingDbFile.data()); + } } /*! @@ -131,9 +137,7 @@ void BuildActionsTests::loadBasicTestSetup() */ void BuildActionsTests::loadTestConfig() { - if (!m_setup.config.storage()) { - initStorage(); - } + initStorage(); m_setup.loadConfigFiles(false); m_setup.building.workingDirectory = m_setup.workingDirectory + "/building"; m_setup.printDatabases(); diff --git a/librepomgr/tests/webapi.cpp b/librepomgr/tests/webapi.cpp index 3a7c57d..310d3a0 100644 --- a/librepomgr/tests/webapi.cpp +++ b/librepomgr/tests/webapi.cpp @@ -52,7 +52,7 @@ public: void testPostingBuildActionsFromTask(); private: - std::string m_dbFile; + std::string m_configDbFile, m_buildingDbFile; ServiceSetup m_setup; boost::beast::error_code m_lastError; string m_body; @@ -75,9 +75,11 @@ WebAPITests::WebAPITests() void WebAPITests::setUp() { applicationInfo.version = APP_VERSION; - m_dbFile = workingCopyPath("test-webapi.db", WorkingCopyMode::Cleanup); + m_configDbFile = workingCopyPath("test-webapi-config.db", WorkingCopyMode::Cleanup); + m_buildingDbFile = workingCopyPath("test-webapi-building.db", WorkingCopyMode::Cleanup); m_setup.webServer.port = randomPort(); - m_setup.config.initStorage(m_dbFile.data()); + m_setup.config.initStorage(m_configDbFile.data()); + m_setup.building.initStorage(m_buildingDbFile.data()); } void WebAPITests::tearDown() @@ -248,7 +250,8 @@ void WebAPITests::testPostingBuildActionsFromTask() building.presets = decltype(building.presets)::fromJson(readFile(testFilePath("test-config/presets.json"))); CPPUNIT_ASSERT_MESSAGE("templates parsed from JSON", !building.presets.templates.empty()); CPPUNIT_ASSERT_MESSAGE("task parsed from JSON", building.presets.tasks.contains("foobarbaz")); - CPPUNIT_ASSERT_MESSAGE("no build actions present before", building.actions.empty()); + CPPUNIT_ASSERT_EQUAL_MESSAGE("no build actions present before", 0_st, building.buildActionCount()); + CPPUNIT_ASSERT_EQUAL_MESSAGE("no build actions running before", 0_st, building.runningBuildActionCount()); const auto response = invokeRouteHandler(&WebAPI::Routes::postBuildAction, { @@ -260,28 +263,46 @@ void WebAPITests::testPostingBuildActionsFromTask() const auto buildActions = parseBuildActions(response->body()); CPPUNIT_ASSERT_EQUAL_MESSAGE("expected number of build actions created", 5_st, buildActions.size()); - CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions actually present", 5_st, building.actions.size()); - for (const auto &action : building.actions) { - CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action->id, " not started yet"), BuildActionStatus::Created, action->status); - CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action->id, " has no result yet"), BuildActionResult::None, action->result); - CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action->id, " has task name assigned"), "foobarbaz"s, action->taskName); - } - CPPUNIT_ASSERT_EQUAL_MESSAGE("foo is 1st action", "foo"s, building.actions[0]->templateName); - CPPUNIT_ASSERT_EQUAL_MESSAGE("foo has dir assigned", "foo"s, building.actions[0]->directory); - CPPUNIT_ASSERT_EQUAL_MESSAGE("foo has correct deps", std::vector{}, building.actions[0]->startAfter); - CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 is 2nd action", "bar-1"s, building.actions[1]->templateName); - CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 has dir assigned", "bar"s, building.actions[1]->directory); - CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 has correct deps", std::vector{ 0 }, building.actions[1]->startAfter); - CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 is 3rd action", "bar-2"s, building.actions[2]->templateName); - CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 has dir assigned", "bar"s, building.actions[2]->directory); - CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 has correct deps", std::vector{ 1 }, building.actions[2]->startAfter); - CPPUNIT_ASSERT_EQUAL_MESSAGE("baz is 4th action", "baz"s, building.actions[3]->templateName); - CPPUNIT_ASSERT_EQUAL_MESSAGE("baz has dir assigned", "baz"s, building.actions[3]->directory); - CPPUNIT_ASSERT_EQUAL_MESSAGE("baz has correct deps", std::vector{ 0 }, building.actions[3]->startAfter); - CPPUNIT_ASSERT_EQUAL_MESSAGE("buz is 5th action", "buz"s, building.actions[4]->templateName); - CPPUNIT_ASSERT_EQUAL_MESSAGE("buz has dir assigned", "buz"s, building.actions[4]->directory); - CPPUNIT_ASSERT_EQUAL_MESSAGE( - "buz has correct deps", std::vector{ 2 CPP_UTILITIES_PP_COMMA 3 }, building.actions[4]->startAfter); + CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions actually present", 5_st, building.buildActionCount()); + CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions not started yet", 0_st, building.runningBuildActionCount()); + building.forEachBuildAction([](std::size_t count) { CPPUNIT_ASSERT_EQUAL_MESSAGE("for-each loop returns correct size", 5_st, count); }, + [](LibPkg::StorageID id, BuildAction &&action) { + CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " not started yet"), BuildActionStatus::Created, action.status); + CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has no result yet"), BuildActionResult::None, action.result); + CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has task name assigned"), "foobarbaz"s, action.taskName); + switch (id) { + case 1: + CPPUNIT_ASSERT_EQUAL_MESSAGE("foo is 1st action", "foo"s, action.templateName); + CPPUNIT_ASSERT_EQUAL_MESSAGE("foo has dir assigned", "foo"s, action.directory); + CPPUNIT_ASSERT_EQUAL_MESSAGE("foo has correct deps", std::vector{}, action.startAfter); + break; + case 2: + CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 is 2nd action", "bar-1"s, action.templateName); + CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 has dir assigned", "bar"s, action.directory); + CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-1 has correct deps", std::vector{ 1 }, action.startAfter); + break; + case 3: + CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 is 3rd action", "bar-2"s, action.templateName); + CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 has dir assigned", "bar"s, action.directory); + CPPUNIT_ASSERT_EQUAL_MESSAGE("bar-2 has correct deps", std::vector{ 2 }, action.startAfter); + break; + case 4: + CPPUNIT_ASSERT_EQUAL_MESSAGE("baz is 4th action", "baz"s, action.templateName); + CPPUNIT_ASSERT_EQUAL_MESSAGE("baz has dir assigned", "baz"s, action.directory); + CPPUNIT_ASSERT_EQUAL_MESSAGE("baz has correct deps", std::vector{ 1 }, action.startAfter); + break; + case 5: + CPPUNIT_ASSERT_EQUAL_MESSAGE("buz is 5th action", "buz"s, action.templateName); + CPPUNIT_ASSERT_EQUAL_MESSAGE("buz has dir assigned", "buz"s, action.directory); + CPPUNIT_ASSERT_EQUAL_MESSAGE( + "buz has correct deps", std::vector{ 3 CPP_UTILITIES_PP_COMMA 4 }, action.startAfter); + break; + default: + CPPUNIT_FAIL(argsToString("build action with unexpected ID \"", id, "\" present")); + } + + return false; + }); CPPUNIT_ASSERT_EQUAL_MESSAGE("response ok", boost::beast::http::status::ok, response->result()); } diff --git a/librepomgr/webapi/routes.cpp b/librepomgr/webapi/routes.cpp index 48c9167..25ad144 100644 --- a/librepomgr/webapi/routes.cpp +++ b/librepomgr/webapi/routes.cpp @@ -353,16 +353,6 @@ void postLoadPackages(const Params ¶ms, ResponseHandler &&handler) handler(makeText(params.request(), "packages loaded")); } -void postDumpCacheFile(const Params ¶ms, ResponseHandler &&handler) -{ - auto configLock = params.setup.config.lockToRead(); - auto buildLock = params.setup.building.lockToRead(); - const auto size = params.setup.saveState(); - buildLock.unlock(); - configLock.unlock(); - handler(makeText(params.request(), "cache file written (" % dataSizeToString(size) + ')')); -} - void postQuit(const Params ¶ms, ResponseHandler &&handler) { cerr << Phrases::SuccessMessage << "Stopping via route /quit" << endl; diff --git a/librepomgr/webapi/routes.h b/librepomgr/webapi/routes.h index a780903..2312855 100644 --- a/librepomgr/webapi/routes.h +++ b/librepomgr/webapi/routes.h @@ -22,7 +22,6 @@ LIBREPOMGR_EXPORT void getBuildActionOutput(const Params ¶ms, ResponseHandle LIBREPOMGR_EXPORT void getBuildActionLogFile(const Params ¶ms, ResponseHandler &&handler); LIBREPOMGR_EXPORT void getBuildActionArtefact(const Params ¶ms, ResponseHandler &&handler); LIBREPOMGR_EXPORT void postLoadPackages(const Params ¶ms, ResponseHandler &&handler); -LIBREPOMGR_EXPORT void postDumpCacheFile(const Params ¶ms, ResponseHandler &&handler); LIBREPOMGR_EXPORT void postBuildAction(const Params ¶ms, ResponseHandler &&handler); LIBREPOMGR_EXPORT void postBuildActionsFromTask(const Params ¶ms, ResponseHandler &&handler, const std::string &taskName, const std::string &directory, const std::vector &startAfterIds, bool startImmediately); diff --git a/librepomgr/webapi/routes_buildaction.cpp b/librepomgr/webapi/routes_buildaction.cpp index 334a3f1..c9b9fb1 100644 --- a/librepomgr/webapi/routes_buildaction.cpp +++ b/librepomgr/webapi/routes_buildaction.cpp @@ -24,33 +24,27 @@ void getBuildActions(const Params ¶ms, ResponseHandler &&handler) RAPIDJSON_NAMESPACE::Value::Array array(jsonDoc.GetArray()); auto buildActionLock = params.setup.building.lockToRead(); - array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(params.setup.building.actions.size()), jsonDoc.GetAllocator()); - auto configLock = params.setup.config.lockToRead(); // build actions might refer to "config things" like packages - for (const auto &buildAction : params.setup.building.actions) { - if (!buildAction) { - continue; - } - ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(*buildAction), array, jsonDoc.GetAllocator()); - } - configLock.unlock(); + params.setup.building.forEachBuildAction( + [&array, &jsonDoc](std::size_t count) { array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(count), jsonDoc.GetAllocator()); }, + [&array, &jsonDoc](LibPkg::StorageID, BuildAction &&buildAction) { + ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(buildAction), array, jsonDoc.GetAllocator()); + return false; + }); buildActionLock.unlock(); handler(makeJson(params.request(), jsonDoc, params.target.hasPrettyFlag())); } struct BuildActionSearchResult { - struct ActionRef { - std::uint64_t id; - BuildAction *action = nullptr; - }; - std::vector actions; + std::vector specifiedIds; + std::vector> actions; std::variant, std::unique_lock> lock; bool ok = false; }; BuildActionSearchResult findBuildActions(const Params ¶ms, ResponseHandler &&handler, bool acquireWriteLock = false, std::size_t maxIDs = 100) { - BuildActionSearchResult result; + auto result = BuildActionSearchResult(); const auto idParams = params.target.decodeValues("id"); if (idParams.empty()) { handler(makeBadRequest(params.request(), "need at least one build action ID")); @@ -61,10 +55,10 @@ BuildActionSearchResult findBuildActions(const Params ¶ms, ResponseHandler & return result; } - result.actions.reserve(idParams.size()); + result.specifiedIds.reserve(idParams.size()); for (const auto &idString : idParams) { try { - result.actions.emplace_back(BuildActionSearchResult::ActionRef{ .id = stringToNumber(idString) }); + result.specifiedIds.emplace_back(stringToNumber(idString)); } catch (const ConversionException &) { handler(makeBadRequest(params.request(), "all IDs must be unsigned integers")); return result; @@ -77,17 +71,15 @@ BuildActionSearchResult findBuildActions(const Params ¶ms, ResponseHandler & result.lock = params.setup.building.lockToRead(); } - for (auto &actionRef : result.actions) { - if (actionRef.id >= params.setup.building.actions.size()) { + result.actions.reserve(result.specifiedIds.size()); + for (const auto id : result.specifiedIds) { + auto action = params.setup.building.getBuildAction(id); + if (!action) { result.lock = std::monostate{}; - handler(makeBadRequest(params.request(), argsToString("no build action with specified ID \"", actionRef.id, "\" exists"))); - return result; - } - actionRef.action = params.setup.building.actions[actionRef.id].get(); - if (!actionRef.action) { - handler(makeBadRequest(params.request(), argsToString("no build action with specified ID \"", actionRef.id, "\" exists"))); + handler(makeBadRequest(params.request(), argsToString("no build action with specified ID \"", id, "\" exists"))); return result; } + result.actions.emplace_back(std::move(action)); } result.ok = true; return result; @@ -101,8 +93,8 @@ void getBuildActionDetails(const Params ¶ms, ResponseHandler &&handler) if (!buildActionsSearchResult.ok) { return; } - for (const auto &buildActionRef : buildActionsSearchResult.actions) { - ReflectiveRapidJSON::JsonReflector::push(*buildActionRef.action, array, jsonDoc.GetAllocator()); + for (const auto &action : buildActionsSearchResult.actions) { + ReflectiveRapidJSON::JsonReflector::push(*action, array, jsonDoc.GetAllocator()); } buildActionsSearchResult.lock = std::monostate{}; handler(makeJson(params.request(), jsonDoc, params.target.hasPrettyFlag())); @@ -128,7 +120,7 @@ void getBuildActionOutput(const Params ¶ms, ResponseHandler &&handler) if (!buildActionsSearchResult.ok) { return; } - auto buildAction = buildActionsSearchResult.actions.front().action; + auto &buildAction = buildActionsSearchResult.actions.front(); if (offset > buildAction->output.size()) { buildActionsSearchResult.lock = std::monostate{}; handler(makeBadRequest(params.request(), "the offset must not exceed the output size")); @@ -163,8 +155,8 @@ static void getBuildActionFile(const Params ¶ms, std::vector Bu return; } - auto *const buildAction = buildActionsSearchResult.actions.front().action; - for (const auto &logFile : buildAction->*fileList) { + auto &buildAction = buildActionsSearchResult.actions.front(); + for (const auto &logFile : (*buildAction).*fileList) { if (name == logFile) { buildAction->streamFile(params, name, "application/octet-stream"); return; @@ -218,7 +210,7 @@ void postBuildAction(const Params ¶ms, ResponseHandler &&handler) continue; } try { - startAfterIds.emplace_back(stringToNumber(startAfterIdValue)); + startAfterIds.emplace_back(stringToNumber(startAfterIdValue)); } catch (const ConversionException &) { handler(makeBadRequest(params.request(), "the specified ID to start after is not a valid build action ID")); return; @@ -226,7 +218,7 @@ void postBuildAction(const Params ¶ms, ResponseHandler &&handler) } if (startAfterAnotherAction) { if (startAfterIds.empty()) { - handler(makeBadRequest(params.request(), "start condition is \"after\" but not exactly one \"start-after-id\" specified")); + handler(makeBadRequest(params.request(), "start condition is \"after\" but not at least one \"start-after-id\" specified")); return; } } else if (!startAfterIdValues.empty() && (startAfterIdValues.size() > 1 || !startAfterIdValues.front().empty())) { @@ -275,10 +267,8 @@ void postBuildAction(const Params ¶ms, ResponseHandler &&handler) // initialize build action auto buildLock = params.setup.building.lockToWrite(); const auto id = params.setup.building.allocateBuildActionID(); - auto startsAfterBuildActions = params.setup.building.getBuildActions(startAfterIds); - const auto startNow = startImmediately || (!startsAfterBuildActions.empty() && BuildAction::haveSucceeded(startsAfterBuildActions)); buildLock.unlock(); - auto buildAction = std::make_shared(id); + auto buildAction = std::make_shared(id, ¶ms.setup); if (!directories.empty()) { buildAction->directory = move(directories.front()); } @@ -293,17 +283,13 @@ void postBuildAction(const Params ¶ms, ResponseHandler &&handler) // serialize build action const auto response = buildAction->toJsonDocument(); - // start build action immediately (no locking required because build action is not part of setup-global list yet) - if (startNow) { - buildAction->start(params.setup); - } - - // add build action to setup-global list and to "follow-up actions" of the build action this one should be started after + // start build action immediately or just add to setup-global list for now auto buildLock2 = params.setup.building.lockToWrite(); - if (!startNow && !startsAfterBuildActions.empty()) { - buildAction->startAfterOtherBuildActions(params.setup, startsAfterBuildActions); + if (startImmediately) { + buildAction->start(params.setup); + } else { + params.setup.building.storeBuildAction(buildAction); } - params.setup.building.actions[id] = std::move(buildAction); buildLock2.unlock(); handler(makeJson(params.request(), response)); @@ -502,7 +488,7 @@ void postBuildActionsFromTask(const Params ¶ms, ResponseHandler &&handler, c // add build actions to setup-global list buildLock = building.lockToWrite(buildReadLock); for (auto &newAction : allocatedActions) { - building.actions[newAction->id] = std::move(newAction); + building.storeBuildAction(newAction); } buildLock.unlock(); @@ -515,41 +501,16 @@ void deleteBuildActions(const Params ¶ms, ResponseHandler &&handler) if (!buildActionsSearchResult.ok) { return; } - for (const auto &actionRef : buildActionsSearchResult.actions) { - if (!actionRef.action->isExecuting()) { + for (const auto &action : buildActionsSearchResult.actions) { + if (!action->isExecuting()) { continue; } buildActionsSearchResult.lock = std::monostate{}; handler( - makeBadRequest(params.request(), argsToString("can not delete \"", actionRef.id, "\"; it is still being executed; no actions altered"))); + makeBadRequest(params.request(), argsToString("can not delete \"", action->id, "\"; it is still being executed; no actions altered"))); return; } - auto &actions = params.setup.building.actions; - auto &invalidActions = params.setup.building.invalidActions; - for (auto &actionRef : buildActionsSearchResult.actions) { - for (auto &maybeFollowUpAction : actionRef.action->m_followUpActions) { - if (auto followUpAction = maybeFollowUpAction.lock()) { - auto &startAfter = followUpAction->startAfter; - startAfter.erase(std::remove(startAfter.begin(), startAfter.end(), actionRef.id)); - } - } - actions[actionRef.id] = nullptr; - invalidActions.emplace(actionRef.id); - } - if (!actions.empty()) { - auto newActionsSize = actions.size(); - for (auto id = newActionsSize - 1;; --id) { - if (actions[id]) { - break; - } - newActionsSize = id; - invalidActions.erase(id); - if (!newActionsSize) { - break; - } - } - actions.resize(newActionsSize); - } + params.setup.building.deleteBuildAction(buildActionsSearchResult.actions); buildActionsSearchResult.lock = std::monostate{}; handler(makeText(params.request(), "ok")); } @@ -570,49 +531,32 @@ void postCloneBuildActions(const Params ¶ms, ResponseHandler &&handler) if (!buildActionsSearchResult.ok) { return; } - for (const auto &actionRef : buildActionsSearchResult.actions) { - if (actionRef.action->isDone()) { + for (const auto &action : buildActionsSearchResult.actions) { + if (action->isDone()) { continue; } buildActionsSearchResult.lock = std::monostate{}; handler(makeBadRequest( - params.request(), argsToString("can not clone \"", actionRef.id, "\"; it is still scheduled or executed; no actions altered"))); + params.request(), argsToString("can not clone \"", action->id, "\"; it is still scheduled or executed; no actions altered"))); return; } - std::vector cloneIds; + auto cloneIds = std::vector(); cloneIds.reserve(buildActionsSearchResult.actions.size()); - for (const auto &actionRef : buildActionsSearchResult.actions) { - const auto orig = actionRef.action; + for (const auto &orig : buildActionsSearchResult.actions) { const auto id = params.setup.building.allocateBuildActionID(); auto clone = make_shared(id); clone->directory = orig->directory; clone->packageNames = orig->packageNames; clone->sourceDbs = orig->sourceDbs; clone->destinationDbs = orig->destinationDbs; - clone->extraParams = orig->extraParams; clone->settings = orig->settings; clone->flags = orig->flags; clone->type = orig->type; - // transfer any follow-up actions which haven't already started yet from the original build action to the new one - // TODO: It would be cool to have a "recursive flag" which would allow restarting follow-ups. - clone->m_followUpActions.reserve(orig->m_followUpActions.size()); - for (auto &maybeOrigFollowUp : orig->m_followUpActions) { - auto origFollowUp = maybeOrigFollowUp.lock(); - if (!origFollowUp || !origFollowUp->isScheduled()) { - continue; - } - for (auto &startAfterId : origFollowUp->startAfter) { - if (startAfterId == orig->id) { - startAfterId = id; - } - } - clone->m_followUpActions.emplace_back(origFollowUp); - } - orig->m_followUpActions.clear(); + clone->startAfter = orig->startAfter; if (startImmediately) { clone->start(params.setup); } - params.setup.building.actions[id] = move(clone); + params.setup.building.storeBuildAction(std::move(clone)); cloneIds.emplace_back(id); } buildActionsSearchResult.lock = std::monostate{}; @@ -625,17 +569,17 @@ void postStartBuildActions(const Params ¶ms, ResponseHandler &&handler) if (!buildActionsSearchResult.ok) { return; } - for (const auto &actionRef : buildActionsSearchResult.actions) { - if (actionRef.action->isScheduled()) { + for (const auto &action : buildActionsSearchResult.actions) { + if (action->isScheduled()) { continue; } buildActionsSearchResult.lock = std::monostate{}; handler( - makeBadRequest(params.request(), argsToString("can not start \"", actionRef.id, "\"; it has already been started; no actions altered"))); + makeBadRequest(params.request(), argsToString("can not start \"", action->id, "\"; it has already been started; no actions altered"))); return; } - for (auto &actionRef : buildActionsSearchResult.actions) { - actionRef.action->start(params.setup); + for (auto &action : buildActionsSearchResult.actions) { + action->start(params.setup); } buildActionsSearchResult.lock = std::monostate{}; handler(makeText(params.request(), "ok")); @@ -647,27 +591,28 @@ void postStopBuildActions(const Params ¶ms, ResponseHandler &&handler) if (!buildActionsSearchResult.ok) { return; } - for (const auto &actionRef : buildActionsSearchResult.actions) { - if (actionRef.action->isExecuting()) { + for (const auto &action : buildActionsSearchResult.actions) { + if (action->isExecuting()) { continue; } buildActionsSearchResult.lock = std::monostate{}; handler(makeBadRequest( - params.request(), argsToString("can not stop/decline \"", actionRef.id, "\"; it is not being executed; no actions altered"))); + params.request(), argsToString("can not stop/decline \"", action->id, "\"; it is not being executed; no actions altered"))); return; } - for (auto &actionRef : buildActionsSearchResult.actions) { - actionRef.action->abort(); - if (actionRef.action->status == BuildActionStatus::Running) { + for (auto &action : buildActionsSearchResult.actions) { + action->abort(); + if (action->status == BuildActionStatus::Running) { // can not immediately stop a running action; the action needs to terminate itself acknowledging the aborted flag continue; } - actionRef.action->status = BuildActionStatus::Finished; - if (actionRef.action->status == BuildActionStatus::AwaitingConfirmation) { - actionRef.action->result = BuildActionResult::ConfirmationDeclined; + if (action->status == BuildActionStatus::AwaitingConfirmation) { + action->result = BuildActionResult::ConfirmationDeclined; } else { - actionRef.action->result = BuildActionResult::Aborted; + action->result = BuildActionResult::Aborted; } + action->status = BuildActionStatus::Finished; + params.setup.building.storeBuildAction(action); } buildActionsSearchResult.lock = std::monostate{}; handler(makeText(params.request(), "ok")); diff --git a/librepomgr/webapi/server.cpp b/librepomgr/webapi/server.cpp index 65bf4ab..dc89d9b 100644 --- a/librepomgr/webapi/server.cpp +++ b/librepomgr/webapi/server.cpp @@ -40,7 +40,6 @@ const Router Server::s_router = { { { http::verb::post, "/api/v0/build-action/clone" }, Route{&Routes::postCloneBuildActions, UserPermissions::ModifyBuildActions} }, { { http::verb::post, "/api/v0/build-action/start" }, Route{&Routes::postStartBuildActions, UserPermissions::ModifyBuildActions} }, { { http::verb::post, "/api/v0/build-action/stop" }, Route{&Routes::postStopBuildActions, UserPermissions::ModifyBuildActions} }, - { { http::verb::post, "/api/v0/dump/cache-file" }, Route{&Routes::postDumpCacheFile, UserPermissions::PerformAdminActions} }, { { http::verb::post, "/api/v0/quit" }, Route{&Routes::postQuit, UserPermissions::PerformAdminActions} }, }; // clang-format on diff --git a/srv/static/index.html b/srv/static/index.html index cf73037..5682638 100644 --- a/srv/static/index.html +++ b/srv/static/index.html @@ -124,10 +124,6 @@

Build actions - Save state manually