Speed up returning build actions table by avoiding deserializing whole obj
This commit is contained in:
parent
645bb0fe54
commit
55c7c62c7c
|
@ -198,7 +198,7 @@ bool InternalBuildAction::reportAbortedIfAborted()
|
||||||
}
|
}
|
||||||
|
|
||||||
BuildAction::BuildAction(IdType id, ServiceSetup *setup) noexcept
|
BuildAction::BuildAction(IdType id, ServiceSetup *setup) noexcept
|
||||||
: id(id)
|
: BuildActionBase(id)
|
||||||
, m_log(this)
|
, m_log(this)
|
||||||
, m_setup(setup)
|
, m_setup(setup)
|
||||||
, m_stopHandler(std::bind(&BuildAction::terminateOngoingBuildProcesses, this))
|
, m_stopHandler(std::bind(&BuildAction::terminateOngoingBuildProcesses, this))
|
||||||
|
|
|
@ -158,8 +158,35 @@ struct LIBREPOMGR_EXPORT BuildActionBase : public ReflectiveRapidJSON::JsonSeria
|
||||||
public ReflectiveRapidJSON::BinarySerializable<BuildActionBase, 1> {
|
public ReflectiveRapidJSON::BinarySerializable<BuildActionBase, 1> {
|
||||||
using IdType = BuildActionIdType;
|
using IdType = BuildActionIdType;
|
||||||
static constexpr IdType invalidId = std::numeric_limits<BuildActionBase::IdType>::max();
|
static constexpr IdType invalidId = std::numeric_limits<BuildActionBase::IdType>::max();
|
||||||
|
explicit BuildActionBase(IdType id = invalidId);
|
||||||
|
|
||||||
|
bool isScheduled() const;
|
||||||
|
bool isExecuting() const;
|
||||||
|
bool isDone() const;
|
||||||
|
bool hasSucceeded() const;
|
||||||
|
|
||||||
|
IdType id;
|
||||||
|
BuildActionType type = BuildActionType::Invalid;
|
||||||
|
std::string taskName;
|
||||||
|
std::string templateName;
|
||||||
|
BuildActionStatus status = BuildActionStatus::Created;
|
||||||
|
BuildActionResult result = BuildActionResult::None;
|
||||||
|
CppUtilities::DateTime created = CppUtilities::DateTime::gmtNow();
|
||||||
|
CppUtilities::DateTime started;
|
||||||
|
CppUtilities::DateTime finished;
|
||||||
|
std::vector<IdType> startAfter;
|
||||||
|
std::string directory;
|
||||||
|
std::vector<std::string> sourceDbs, destinationDbs;
|
||||||
|
std::vector<std::string> packageNames;
|
||||||
|
BuildActionFlagType flags = noBuildActionFlags;
|
||||||
|
std::unordered_map<std::string, std::string> settings;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
inline BuildActionBase::BuildActionBase(IdType id)
|
||||||
|
: id(id)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
struct LIBREPOMGR_EXPORT BuildAction : public BuildActionBase,
|
struct LIBREPOMGR_EXPORT BuildAction : public BuildActionBase,
|
||||||
public std::enable_shared_from_this<BuildAction>,
|
public std::enable_shared_from_this<BuildAction>,
|
||||||
public ReflectiveRapidJSON::JsonSerializable<BuildAction>,
|
public ReflectiveRapidJSON::JsonSerializable<BuildAction>,
|
||||||
|
@ -178,10 +205,6 @@ public:
|
||||||
explicit BuildAction(IdType id = invalidId, ServiceSetup *setup = nullptr) noexcept;
|
explicit BuildAction(IdType id = invalidId, ServiceSetup *setup = nullptr) noexcept;
|
||||||
BuildAction &operator=(BuildAction &&other);
|
BuildAction &operator=(BuildAction &&other);
|
||||||
~BuildAction();
|
~BuildAction();
|
||||||
bool isScheduled() const;
|
|
||||||
bool isExecuting() const;
|
|
||||||
bool isDone() const;
|
|
||||||
bool hasSucceeded() const;
|
|
||||||
static bool haveSucceeded(const std::vector<std::shared_ptr<BuildAction>> &buildActions);
|
static bool haveSucceeded(const std::vector<std::shared_ptr<BuildAction>> &buildActions);
|
||||||
bool isAborted() const;
|
bool isAborted() const;
|
||||||
const std::atomic_bool &aborted() const;
|
const std::atomic_bool &aborted() const;
|
||||||
|
@ -216,22 +239,6 @@ private:
|
||||||
LibPkg::StorageID conclude(BuildActionResult result);
|
LibPkg::StorageID conclude(BuildActionResult result);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
IdType id;
|
|
||||||
BuildActionType type = BuildActionType::Invalid;
|
|
||||||
std::string taskName;
|
|
||||||
std::string templateName;
|
|
||||||
BuildActionStatus status = BuildActionStatus::Created;
|
|
||||||
BuildActionResult result = BuildActionResult::None;
|
|
||||||
CppUtilities::DateTime created = CppUtilities::DateTime::gmtNow();
|
|
||||||
CppUtilities::DateTime started;
|
|
||||||
CppUtilities::DateTime finished;
|
|
||||||
std::vector<IdType> startAfter;
|
|
||||||
std::string directory;
|
|
||||||
std::vector<std::string> sourceDbs, destinationDbs;
|
|
||||||
std::vector<std::string> packageNames;
|
|
||||||
BuildActionFlagType flags = noBuildActionFlags;
|
|
||||||
std::unordered_map<std::string, std::string> settings;
|
|
||||||
|
|
||||||
std::vector<std::string> logfiles;
|
std::vector<std::string> logfiles;
|
||||||
std::vector<std::string> artefacts;
|
std::vector<std::string> artefacts;
|
||||||
std::variant<std::string, std::vector<std::string>, LibPkg::LicenseResult, LibPkg::PackageUpdates, BuildPreparation, BuildProgress,
|
std::variant<std::string, std::vector<std::string>, LibPkg::LicenseResult, LibPkg::PackageUpdates, BuildPreparation, BuildProgress,
|
||||||
|
@ -251,22 +258,22 @@ private:
|
||||||
std::unique_ptr<InternalBuildAction> m_internalBuildAction;
|
std::unique_ptr<InternalBuildAction> m_internalBuildAction;
|
||||||
};
|
};
|
||||||
|
|
||||||
inline bool BuildAction::isScheduled() const
|
inline bool BuildActionBase::isScheduled() const
|
||||||
{
|
{
|
||||||
return status == BuildActionStatus::Created || status == BuildActionStatus::AwaitingConfirmation;
|
return status == BuildActionStatus::Created || status == BuildActionStatus::AwaitingConfirmation;
|
||||||
}
|
}
|
||||||
|
|
||||||
inline bool BuildAction::isExecuting() const
|
inline bool BuildActionBase::isExecuting() const
|
||||||
{
|
{
|
||||||
return status == BuildActionStatus::Enqueued || status == BuildActionStatus::Running;
|
return status == BuildActionStatus::Enqueued || status == BuildActionStatus::Running;
|
||||||
}
|
}
|
||||||
|
|
||||||
inline bool BuildAction::isDone() const
|
inline bool BuildActionBase::isDone() const
|
||||||
{
|
{
|
||||||
return status == BuildActionStatus::Finished;
|
return status == BuildActionStatus::Finished;
|
||||||
}
|
}
|
||||||
|
|
||||||
inline bool BuildAction::hasSucceeded() const
|
inline bool BuildActionBase::hasSucceeded() const
|
||||||
{
|
{
|
||||||
return isDone() && result == BuildActionResult::Success;
|
return isDone() && result == BuildActionResult::Success;
|
||||||
}
|
}
|
||||||
|
@ -331,44 +338,6 @@ template <typename... Args> inline void BuildAction::appendOutput(CppUtilities::
|
||||||
appendOutput(std::move(CppUtilities::argsToString(CppUtilities::EscapeCodes::formattedPhraseString(phrase), std::forward<Args>(args)...)));
|
appendOutput(std::move(CppUtilities::argsToString(CppUtilities::EscapeCodes::formattedPhraseString(phrase), std::forward<Args>(args)...)));
|
||||||
}
|
}
|
||||||
|
|
||||||
struct LIBREPOMGR_EXPORT BuildActionBasicInfo : public ReflectiveRapidJSON::JsonSerializable<BuildActionBasicInfo> {
|
|
||||||
explicit BuildActionBasicInfo(const BuildAction &buildAction)
|
|
||||||
: id(buildAction.id)
|
|
||||||
, taskName(buildAction.taskName)
|
|
||||||
, templateName(buildAction.templateName)
|
|
||||||
, directory(buildAction.directory)
|
|
||||||
, packageNames(buildAction.packageNames)
|
|
||||||
, sourceDbs(buildAction.sourceDbs)
|
|
||||||
, destinationDbs(buildAction.destinationDbs)
|
|
||||||
, startAfter(buildAction.startAfter)
|
|
||||||
, settings(buildAction.settings)
|
|
||||||
, flags(buildAction.flags)
|
|
||||||
, type(buildAction.type)
|
|
||||||
, status(buildAction.status)
|
|
||||||
, result(buildAction.result)
|
|
||||||
, created(buildAction.created)
|
|
||||||
, started(buildAction.started)
|
|
||||||
, finished(buildAction.finished)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
const BuildAction::IdType id;
|
|
||||||
const std::string &taskName;
|
|
||||||
const std::string &templateName;
|
|
||||||
const std::string &directory;
|
|
||||||
const std::vector<std::string> &packageNames;
|
|
||||||
const std::vector<std::string> &sourceDbs, &destinationDbs;
|
|
||||||
const std::vector<BuildAction::IdType> &startAfter;
|
|
||||||
const std::unordered_map<std::string, std::string> settings;
|
|
||||||
const BuildActionFlagType flags = noBuildActionFlags;
|
|
||||||
const BuildActionType type;
|
|
||||||
const BuildActionStatus status;
|
|
||||||
const BuildActionResult result;
|
|
||||||
const CppUtilities::DateTime created;
|
|
||||||
const CppUtilities::DateTime started;
|
|
||||||
const CppUtilities::DateTime finished;
|
|
||||||
};
|
|
||||||
|
|
||||||
} // namespace LibRepoMgr
|
} // namespace LibRepoMgr
|
||||||
|
|
||||||
#endif // LIBREPOMGR_BUILD_ACTION_H
|
#endif // LIBREPOMGR_BUILD_ACTION_H
|
||||||
|
|
|
@ -397,14 +397,14 @@ void ServiceSetup::BuildSetup::rebuildDb()
|
||||||
}
|
}
|
||||||
|
|
||||||
void ServiceSetup::BuildSetup::forEachBuildAction(
|
void ServiceSetup::BuildSetup::forEachBuildAction(
|
||||||
std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func, std::size_t limit, std::size_t start)
|
std::function<void(std::size_t)> count, ServiceSetup::BuildSetup::BuildActionVisitorBase &&func, std::size_t limit, std::size_t start)
|
||||||
{
|
{
|
||||||
auto txn = m_storage->buildActions.getROTransaction();
|
auto txn = m_storage->buildActions.getROTransaction();
|
||||||
const auto total = txn.size();
|
const auto total = txn.size();
|
||||||
count(std::min(limit, total));
|
count(std::min(limit, total));
|
||||||
const auto reverse = start == std::numeric_limits<std::size_t>::max();
|
const auto reverse = start == std::numeric_limits<std::size_t>::max();
|
||||||
for (auto i = reverse ? txn.rbegin()
|
for (auto i = reverse ? txn.rbegin<decltype(txn)::DirectStorage, BuildActionBase>()
|
||||||
: txn.lower_bound(static_cast<LibPkg::StorageID>(
|
: txn.lower_bound<decltype(txn)::DirectStorage, BuildActionBase>(static_cast<LibPkg::StorageID>(
|
||||||
start > std::numeric_limits<LibPkg::StorageID>::max() ? std::numeric_limits<LibPkg::StorageID>::max() : start));
|
start > std::numeric_limits<LibPkg::StorageID>::max() ? std::numeric_limits<LibPkg::StorageID>::max() : start));
|
||||||
i != txn.end() && limit; reverse ? --i : ++i, --limit) {
|
i != txn.end() && limit; reverse ? --i : ++i, --limit) {
|
||||||
if (func(i.getID(), std::move(i.value()))) {
|
if (func(i.getID(), std::move(i.value()))) {
|
||||||
|
@ -413,7 +413,7 @@ void ServiceSetup::BuildSetup::forEachBuildAction(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ServiceSetup::BuildSetup::forEachBuildAction(std::function<bool(LibPkg::StorageID, BuildAction &, bool &)> &&func)
|
void ServiceSetup::BuildSetup::forEachBuildAction(ServiceSetup::BuildSetup::BuildActionVisitorWriteable &&func)
|
||||||
{
|
{
|
||||||
auto txn = m_storage->buildActions.getRWTransaction();
|
auto txn = m_storage->buildActions.getRWTransaction();
|
||||||
for (auto i = txn.begin(); i != txn.end(); ++i) {
|
for (auto i = txn.begin(); i != txn.end(); ++i) {
|
||||||
|
|
|
@ -145,9 +145,10 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
|
||||||
std::size_t buildActionCount();
|
std::size_t buildActionCount();
|
||||||
std::size_t runningBuildActionCount() const;
|
std::size_t runningBuildActionCount() const;
|
||||||
void rebuildDb();
|
void rebuildDb();
|
||||||
void forEachBuildAction(std::function<void(std::size_t)> count, std::function<bool(LibPkg::StorageID, BuildAction &&)> &&func,
|
using BuildActionVisitorBase = std::function<bool(LibPkg::StorageID, BuildActionBase &&)>;
|
||||||
std::size_t limit, std::size_t start);
|
void forEachBuildAction(std::function<void(std::size_t)> count, BuildActionVisitorBase &&func, std::size_t limit, std::size_t start);
|
||||||
void forEachBuildAction(std::function<bool(LibPkg::StorageID, BuildAction &, bool &)> &&func);
|
using BuildActionVisitorWriteable = std::function<bool(LibPkg::StorageID, BuildAction &, bool &)>;
|
||||||
|
void forEachBuildAction(BuildActionVisitorWriteable &&func);
|
||||||
std::vector<std::shared_ptr<BuildAction>> followUpBuildActions(BuildActionIdType forId);
|
std::vector<std::shared_ptr<BuildAction>> followUpBuildActions(BuildActionIdType forId);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
|
@ -267,7 +267,7 @@ void WebAPITests::testPostingBuildActionsFromTask()
|
||||||
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions actually present", 5_st, building.buildActionCount());
|
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions actually present", 5_st, building.buildActionCount());
|
||||||
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions not started yet", 0_st, building.runningBuildActionCount());
|
CPPUNIT_ASSERT_EQUAL_MESSAGE("build actions not started yet", 0_st, building.runningBuildActionCount());
|
||||||
building.forEachBuildAction([](std::size_t count) { CPPUNIT_ASSERT_EQUAL_MESSAGE("for-each loop returns correct size", 5_st, count); },
|
building.forEachBuildAction([](std::size_t count) { CPPUNIT_ASSERT_EQUAL_MESSAGE("for-each loop returns correct size", 5_st, count); },
|
||||||
[](LibPkg::StorageID id, BuildAction &&action) {
|
[](LibPkg::StorageID id, BuildActionBase &&action) {
|
||||||
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " not started yet"), BuildActionStatus::Created, action.status);
|
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " not started yet"), BuildActionStatus::Created, action.status);
|
||||||
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has no result yet"), BuildActionResult::None, action.result);
|
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has no result yet"), BuildActionResult::None, action.result);
|
||||||
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has task name assigned"), "foobarbaz"s, action.taskName);
|
CPPUNIT_ASSERT_EQUAL_MESSAGE(argsToString("build action ", action.id, " has task name assigned"), "foobarbaz"s, action.taskName);
|
||||||
|
|
|
@ -36,8 +36,8 @@ void getBuildActions(const Params ¶ms, ResponseHandler &&handler)
|
||||||
auto buildActionLock = params.setup.building.lockToRead();
|
auto buildActionLock = params.setup.building.lockToRead();
|
||||||
params.setup.building.forEachBuildAction(
|
params.setup.building.forEachBuildAction(
|
||||||
[&array, &jsonDoc](std::size_t count) { array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(count), jsonDoc.GetAllocator()); },
|
[&array, &jsonDoc](std::size_t count) { array.Reserve(ReflectiveRapidJSON::JsonReflector::rapidJsonSize(count), jsonDoc.GetAllocator()); },
|
||||||
[&array, &jsonDoc, limit](LibPkg::StorageID, BuildAction &&buildAction) {
|
[&array, &jsonDoc, limit](LibPkg::StorageID, BuildActionBase &&buildActionBase) {
|
||||||
ReflectiveRapidJSON::JsonReflector::push(BuildActionBasicInfo(buildAction), array, jsonDoc.GetAllocator());
|
ReflectiveRapidJSON::JsonReflector::push(buildActionBase, array, jsonDoc.GetAllocator());
|
||||||
return array.Size() >= limit;
|
return array.Size() >= limit;
|
||||||
},
|
},
|
||||||
limit, start);
|
limit, start);
|
||||||
|
|
Loading…
Reference in New Issue