Speed up returning build actions table by avoiding deserializing whole obj

This commit is contained in:
Martchus 2022-05-31 20:54:02 +02:00
parent 645bb0fe54
commit 55c7c62c7c
6 changed files with 43 additions and 73 deletions

View File

@ -198,7 +198,7 @@ bool InternalBuildAction::reportAbortedIfAborted()
}
BuildAction::BuildAction(IdType id, ServiceSetup *setup) noexcept
: id(id)
: BuildActionBase(id)
, m_log(this)
, m_setup(setup)
, m_stopHandler(std::bind(&BuildAction::terminateOngoingBuildProcesses, this))

View File

@ -158,8 +158,35 @@ struct LIBREPOMGR_EXPORT BuildActionBase : public ReflectiveRapidJSON::JsonSeria
public ReflectiveRapidJSON::BinarySerializable<BuildActionBase, 1> {
using IdType = BuildActionIdType;
static constexpr IdType invalidId = std::numeric_limits<BuildActionBase::IdType>::max();
explicit BuildActionBase(IdType id = invalidId);
bool isScheduled() const;
bool isExecuting() const;
bool isDone() const;
bool hasSucceeded() const;
IdType id;
BuildActionType type = BuildActionType::Invalid;
std::string taskName;
std::string templateName;
BuildActionStatus status = BuildActionStatus::Created;
BuildActionResult result = BuildActionResult::None;
CppUtilities::DateTime created = CppUtilities::DateTime::gmtNow();
CppUtilities::DateTime started;
CppUtilities::DateTime finished;
std::vector<IdType> startAfter;
std::string directory;
std::vector<std::string> sourceDbs, destinationDbs;
std::vector<std::string> packageNames;
BuildActionFlagType flags = noBuildActionFlags;
std::unordered_map<std::string, std::string> settings;
};
inline BuildActionBase::BuildActionBase(IdType id)
: id(id)
{
}
struct LIBREPOMGR_EXPORT BuildAction : public BuildActionBase,
public std::enable_shared_from_this<BuildAction>,
public ReflectiveRapidJSON::JsonSerializable<BuildAction>,
@ -178,10 +205,6 @@ public:
explicit BuildAction(IdType id = invalidId, ServiceSetup *setup = nullptr) noexcept;
BuildAction &operator=(BuildAction &&other);
~BuildAction();
bool isScheduled() const;
bool isExecuting() const;
bool isDone() const;
bool hasSucceeded() const;
static bool haveSucceeded(const std::vector<std::shared_ptr<BuildAction>> &buildActions);
bool isAborted() const;
const std::atomic_bool &aborted() const;
@ -216,22 +239,6 @@ private:
LibPkg::StorageID conclude(BuildActionResult result);
public:
IdType id;
BuildActionType type = BuildActionType::Invalid;
std::string taskName;
std::string templateName;
BuildActionStatus status = BuildActionStatus::Created;
BuildActionResult result = BuildActionResult::None;
CppUtilities::DateTime created = CppUtilities::DateTime::gmtNow();
CppUtilities::DateTime started;
CppUtilities::DateTime finished;
std::vector<IdType> startAfter;
std::string directory;
std::vector<std::string> sourceDbs, destinationDbs;
std::vector<std::string> packageNames;
BuildActionFlagType flags = noBuildActionFlags;
std::unordered_map<std::string, std::string> settings;
std::vector<std::string> logfiles;
std::vector<std::string> artefacts;
std::variant<std::string, std::vector<std::string>, LibPkg::LicenseResult, LibPkg::PackageUpdates, BuildPreparation, BuildProgress,
@ -251,22 +258,22 @@ private:
std::unique_ptr<InternalBuildAction> m_internalBuildAction;
};
inline bool BuildAction::isScheduled() const
inline bool BuildActionBase::isScheduled() const
{
return status == BuildActionStatus::Created || status == BuildActionStatus::AwaitingConfirmation;
}
inline bool BuildAction::isExecuting() const
inline bool BuildActionBase::isExecuting() const
{
return status == BuildActionStatus::Enqueued || status == BuildActionStatus::Running;
}
inline bool BuildAction::isDone() const
inline bool BuildActionBase::isDone() const
{
return status == BuildActionStatus::Finished;
}
inline bool BuildAction::hasSucceeded() const
inline bool BuildActionBase::hasSucceeded() const
{
return isDone() && result == BuildActionResult::Success;
}
@ -331,44 +338,6 @@ template <typename... Args> inline void BuildAction::appendOutput(CppUtilities::
appendOutput(std::move(CppUtilities::argsToString(CppUtilities::EscapeCodes::formattedPhraseString(phrase), std::forward<Args>(args)...)));
}
struct LIBREPOMGR_EXPORT BuildActionBasicInfo : public ReflectiveRapidJSON::JsonSerializable<BuildActionBasicInfo> {
explicit BuildActionBasicInfo(const BuildAction &buildAction)
: id(buildAction.id)
, taskName(buildAction.taskName)
, templateName(buildAction.templateName)
, directory(buildAction.directory)
, packageNames(buildAction.packageNames)
, sourceDbs(buildAction.sourceDbs)
, destinationDbs(buildAction.destinationDbs)
, startAfter(buildAction.startAfter)
, settings(buildAction.settings)
, flags(buildAction.flags)
, type(buildAction.type)
, status(buildAction.status)
, result(buildAction.result)
, created(buildAction.created)
, started(buildAction.started)
, finished(buildAction.finished)
{
}
const BuildAction::IdType id;
const std::string &taskName;
const std::string &templateName;
const std::string &directory;
const std::vector<std::string> &packageNames;
const std::vector<std::string> &sourceDbs, &destinationDbs;
const std::vector<BuildAction::IdType> &startAfter;
const std::unordered_map<std::string, std::string> settings;
const BuildActionFlagType flags = noBuildActionFlags;
const BuildActionType type;
const BuildActionStatus status;
const BuildActionResult result;
const CppUtilities::DateTime created;
const CppUtilities::DateTime started;
const CppUtilities::DateTime finished;
};
} // namespace LibRepoMgr
#endif // LIBREPOMGR_BUILD_ACTION_H

View File

@ -397,14 +397,14 @@ void ServiceSetup::BuildSetup::rebuildDb()
}
void ServiceSetup::BuildSetup::forEachBuildAction(
std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func, std::size_t limit, std::size_t start)
std::function<void(std::size_t)> count, ServiceSetup::BuildSetup::BuildActionVisitorBase &&func, std::size_t limit, std::size_t start)
{
auto txn = m_storage->buildActions.getROTransaction();
const auto total = txn.size();
count(std::min(limit, total));
const auto reverse = start == std::numeric_limits<std::size_t>::max();
for (auto i = reverse ? txn.rbegin()
: txn.lower_bound(static_cast<LibPkg::StorageID>(
for (auto i = reverse ? txn.rbegin<decltype(txn)::DirectStorage, BuildActionBase>()
: txn.lower_bound<decltype(txn)::DirectStorage, BuildActionBase>(static_cast<LibPkg::StorageID>(
start > std::numeric_limits<LibPkg::StorageID>::max() ? std::numeric_limits<LibPkg::StorageID>::max() : start));
i != txn.end() && limit; reverse ? --i : ++i, --limit) {
if (func(i.getID(), std::move(i.value()))) {
@ -413,7 +413,7 @@ void ServiceSetup::BuildSetup::forEachBuildAction(
}
}
void ServiceSetup::BuildSetup::forEachBuildAction(std::function<bool(LibPkg::StorageID, BuildAction &, bool &)> &&func)
void ServiceSetup::BuildSetup::forEachBuildAction(ServiceSetup::BuildSetup::BuildActionVisitorWriteable &&func)
{
auto txn = m_storage->buildActions.getRWTransaction();
for (auto i = txn.begin(); i != txn.end(); ++i) {

View File

@ -145,9 +145,10 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
std::size_t buildActionCount();
std::size_t runningBuildActionCount() const;
void rebuildDb();
void forEachBuildAction(std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func,
std::size_t limit, std::size_t start);
void forEachBuildAction(std::function<bool(LibPkg::StorageID, BuildAction &, bool &)> &&func);
using BuildActionVisitorBase = std::function<bool(LibPkg::StorageID, BuildActionBase &&)>;
void forEachBuildAction(std::function<void(std::size_t)> count, BuildActionVisitorBase &&func, std::size_t limit, std::size_t start);
using BuildActionVisitorWriteable = std::function<bool(LibPkg::StorageID, BuildAction &, bool &)>;
void forEachBuildAction(BuildActionVisitorWriteable &&func);
std::vector<std::shared_ptr<BuildAction>> followUpBuildActions(BuildActionIdType forId);
private:

View File

@ -267,7 +267,7 @@ void WebAPITests::testPostingBuildActionsFromTask()
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions actually present", 5_st, building.buildActionCount());
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions not started yet", 0_st, building.runningBuildActionCount());
building.forEachBuildAction([](std::size_t count) { CPPUNIT_ASSERT_EQUAL_MESSAGE("for-each loop returns correct size", 5_st, count); },
[](LibPkg::StorageID id, BuildAction &&action) {
[](LibPkg::StorageID id, BuildActionBase &&action) {
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " not started yet"), BuildActionStatus::Created, action.status);
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has no result yet"), BuildActionResult::None, action.result);
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has task name assigned"), "foobarbaz"s, action.taskName);

View File

@ -36,8 +36,8 @@ void getBuildActions(const Params &params, ResponseHandler &&handler)
auto buildActionLock = params.setup.building.lockToRead();
params.setup.building.forEachBuildAction(
[&array, &jsonDoc](std::size_t count) { array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(count), jsonDoc.GetAllocator()); },
[&array, &jsonDoc, limit](LibPkg::StorageID, BuildAction &&buildAction) {
ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(buildAction), array, jsonDoc.GetAllocator());
[&array, &jsonDoc, limit](LibPkg::StorageID, BuildActionBase &&buildActionBase) {
ReflectiveRapidJSON::JsonReflector::push(buildActionBase, array, jsonDoc.GetAllocator());
return array.Size() >= limit;
},
limit, start);