Martchus
/
arch-repo-manager
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor to optimize deserialization of base-data 

* Break backwards compatibility
* Allow to deserialize only base-data of packages and build
  actions to potentially speed up showing tables
* Speed up package search in many cases by only deserializing
  base-data (unless details are actually wanted)
This commit is contained in:
Martchus 2022-05-15 03:24:10 +02:00
parent ae44624989
commit 68d67f543f
20 changed files with 467 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
3rdparty/lmdb-safe vendored

@ -1 +1 @@
Subproject commit 2f5eb32a33130c4deb139118ebbd2dfa634d69bd
Subproject commit 9a9bed0f462244192e7f8524077c0daef64047c6

34
libpkg/algo/search.cpp
View File

@ -200,6 +200,29 @@ void Config::packages(std::string_view dbName, std::string_view dbArch, const st
}
}
void Config::packages(std::string_view dbName, std::string_view dbArch, const std::string &packageName, const DatabaseVisitor &databaseVisitor,
const PackageVisitorBase &visitor)
{
// don't allow to iterate though all packages
if (packageName.empty()) {
return;
}
auto basePackage = std::make_shared<PackageBase>();
for (auto &db : databases) {
if ((!dbName.empty() && dbName != db.name) || (!dbArch.empty() && dbArch != db.arch) || (databaseVisitor && databaseVisitor(db))) {
continue;
}
if (!basePackage) {
basePackage = std::make_shared<PackageBase>();
} else {
basePackage->clear();
}
if (const auto id = db.findBasePackageWithID(packageName, *basePackage)) {
visitor(db, id, std::move(basePackage));
}
}
}
void Config::packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByName &visitor)
{
for (auto &db : databases) {
@ -211,6 +234,17 @@ void Config::packagesByName(const DatabaseVisitor &databaseVisitor, const Packag
}
}
void Config::packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByNameBase &visitor)
{
for (auto &db : databases) {
if (databaseVisitor && databaseVisitor(db)) {
continue;
}
db.allPackagesByName(
[&](std::string_view packageName, const std::function<StorageID(PackageBase&)> &getPackage) { return visitor(db, packageName, getPackage); });
}
}
void Config::providingPackages(const Dependency &dependency, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor)
{
for (auto &db : databases) {

5
libpkg/data/config.h
View File

@ -107,10 +107,12 @@ constexpr bool operator&(BuildOrderOptions lhs, BuildOrderOptions rhs)
struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::BinarySerializable<Config> {
using DatabaseVisitor = std::function<bool(Database &)>;
using PackageVisitorBase = std::function<bool(Database &, StorageID, std::shared_ptr<PackageBase> &&)>; // package is invalidated/reused unless moved from!!!
using PackageVisitorMove
= std::function<bool(Database &, StorageID, std::shared_ptr<Package> &&)>; // package is invalidated/reused unless moved from!!!
using PackageVisitorConst = std::function<bool(Database &, StorageID, const std::shared_ptr<Package> &)>;
using PackageVisitorByName = std::function<bool(Database &, std::string_view, const std::function<PackageSpec(void)> &)>;
using PackageVisitorByNameBase = std::function<bool(Database &, std::string_view, const std::function<StorageID(PackageBase&)> &)>;
explicit Config();
~Config();
@ -164,7 +166,10 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar
// package iteration
void packages(std::string_view dbName, std::string_view dbArch, const std::string &packageName, const DatabaseVisitor &databaseVisitor,
const PackageVisitorConst &visitor);
void packages(std::string_view dbName, std::string_view dbArch, const std::string &packageName, const DatabaseVisitor &databaseVisitor,
const PackageVisitorBase &visitor);
void packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByName &visitor);
void packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByNameBase &visitor);
void providingPackages(const Dependency &dependency, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor);
void providingPackages(const std::string &libraryName, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor);

81
libpkg/data/database.cpp
View File

@ -299,12 +299,33 @@ void Database::allPackages(const PackageVisitorMove &visitor)
}
}
void Database::allPackages(const PackageVisitorBase &visitor)
{
auto txn = m_storage->packages.getROTransaction();
for (auto i = txn.begin<std::shared_ptr, PackageBase>(); i != txn.end(); ++i) {
if (visitor(i.getID(), std::move(i.getPointer()))) {
return;
}
}
}
void LibPkg::Database::allPackagesByName(const PackageVisitorByName &visitor)
{
auto txn = m_storage->packages.getROTransaction();
for (auto i = txn.begin_idx<0, std::shared_ptr>(); i != txn.end(); ++i) {
const auto packageName = i.getKey().get<string_view>();
if (visitor(packageName, [this, &txn, &i]() { return m_storage->packageCache.retrieve(*m_storage, &txn, i.value()); })) {
if (visitor(packageName, [this, &txn, &i] { return m_storage->packageCache.retrieve(*m_storage, &txn, i.value()); })) {
return;
}
}
}
void LibPkg::Database::allPackagesByName(const PackageVisitorByNameBase &visitor)
{
auto txn = m_storage->packages.getROTransaction();
for (auto i = txn.begin_idx<0, std::shared_ptr>(); i != txn.end(); ++i) {
const auto packageName = i.getKey().get<string_view>();
if (visitor(packageName, [&txn, &i] (PackageBase &pkg) { return txn.get(i.value(), pkg); })) {
return;
}
}
@ -393,6 +414,12 @@ PackageSpec Database::findPackageWithID(const std::string &packageName)
return m_storage->packageCache.retrieve(*m_storage, packageName);
}
StorageID Database::findBasePackageWithID(const std::string &packageName, PackageBase &basePackage)
{
auto txn = m_storage->packages.getROTransaction();
return txn.get<0, PackageBase>(packageName, basePackage);
}
void Database::removePackage(const std::string &packageName)
{
const auto lock = std::unique_lock(m_storage->updateMutex);
@ -504,7 +531,7 @@ std::unordered_map<PackageSpec, UnresolvedDependencies> Database::detectUnresolv
// add packages to list of unresolved packages
for (const auto &affectedPackageID : requiredDep.relevantPackages) {
const auto affectedPackage = findPackage(affectedPackageID);
unresolvedPackages[PackageSpec(affectedPackageID, affectedPackage)].deps.emplace_back(requiredDep);
unresolvedPackages[GenericPackageSpec(affectedPackageID, affectedPackage)].deps.emplace_back(requiredDep);
}
}
@ -545,7 +572,7 @@ std::unordered_map<PackageSpec, UnresolvedDependencies> Database::detectUnresolv
// add packages to list of unresolved packages
for (const auto &affectedPackageID : requiredLib.relevantPackages) {
const auto affectedPackage = findPackage(affectedPackageID);
unresolvedPackages[PackageSpec(affectedPackageID, affectedPackage)].libs.emplace_back(requiredLib.name);
unresolvedPackages[GenericPackageSpec(affectedPackageID, affectedPackage)].libs.emplace_back(requiredLib.name);
}
}
@ -801,7 +828,7 @@ PackageUpdater::~PackageUpdater()
{
}
LibPkg::PackageSpec LibPkg::PackageUpdater::findPackageWithID(const std::string &packageName)
PackageSpec LibPkg::PackageUpdater::findPackageWithID(const std::string &packageName)
{
return m_database.m_storage->packageCache.retrieve(*m_database.m_storage, &m_d->packagesTxn, packageName);
}
@ -901,14 +928,14 @@ void push<LibPkg::PackageSearchResult>(
push(pkg->timestamp, "timestamp", obj, allocator);
push(pkg->version, "version", obj, allocator);
push(pkg->description, "description", obj, allocator);
if (const auto &pkgInfo = pkg->packageInfo) {
push(pkgInfo->arch, "arch", obj, allocator);
push(pkgInfo->buildDate, "buildDate", obj, allocator);
if (!pkg->arch.empty()) {
push(pkg->arch, "arch", obj, allocator);
}
if (!pkg->buildDate.isNull()) {
push(pkg->buildDate, "buildDate", obj, allocator);
}
if (!pkg->archs.empty()) {
push(pkg->archs, "archs", obj, allocator);
} else if (const auto &srcInfo = pkg->sourceInfo) {
push(srcInfo->archs, "archs", obj, allocator);
}
if (const auto *const dbInfo = std::get_if<LibPkg::DatabaseInfo>(&reflectable.db)) {
if (!dbInfo->name.empty()) {
@ -946,18 +973,42 @@ void pull<LibPkg::PackageSearchResult>(LibPkg::PackageSearchResult &reflectable,
ReflectiveRapidJSON::JsonReflector::pull(pkg->timestamp, "timestamp", obj, errors);
ReflectiveRapidJSON::JsonReflector::pull(pkg->version, "version", obj, errors);
ReflectiveRapidJSON::JsonReflector::pull(pkg->description, "description", obj, errors);
auto &pkgInfo = pkg->packageInfo;
if (!pkgInfo) {
pkgInfo = make_unique<LibPkg::PackageInfo>();
}
ReflectiveRapidJSON::JsonReflector::pull(pkgInfo->arch, "arch", obj, errors);
ReflectiveRapidJSON::JsonReflector::pull(pkgInfo->buildDate, "buildDate", obj, errors);
ReflectiveRapidJSON::JsonReflector::pull(pkg->arch, "arch", obj, errors);
ReflectiveRapidJSON::JsonReflector::pull(pkg->buildDate, "buildDate", obj, errors);
ReflectiveRapidJSON::JsonReflector::pull(pkg->archs, "archs", obj, errors);
auto &dbInfo = reflectable.db.emplace<LibPkg::DatabaseInfo>();
ReflectiveRapidJSON::JsonReflector::pull(dbInfo.name, "db", obj, errors);
ReflectiveRapidJSON::JsonReflector::pull(dbInfo.arch, "dbArch", obj, errors);
}
template <>
void push<LibPkg::PackageBaseSearchResult>(
const LibPkg::PackageBaseSearchResult &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)
{
// serialize PackageBaseSearchResult object in accordance with PackageSearchResult
value.SetObject();
auto obj = value.GetObject();
if (auto &pkg = reflectable.pkg) {
push(*pkg, obj, allocator);
}
if (const auto &db = reflectable.db) {
push(db->name, "db", obj, allocator);
if (!db->arch.empty()) {
push(db->arch, "dbArch", obj, allocator);
}
}
}
template <>
void pull<LibPkg::PackageBaseSearchResult>(LibPkg::PackageBaseSearchResult &reflectable,
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors)
{
CPP_UTILITIES_UNUSED(reflectable)
CPP_UTILITIES_UNUSED(value)
CPP_UTILITIES_UNUSED(errors)
throw std::logic_error("Attempt to deserialize LibPkg::PackageBaseSearchResult");
}
template <>
void push<LibPkg::AtomicDateTime>(
const LibPkg::AtomicDateTime &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)

36
libpkg/data/database.h
View File

@ -42,6 +42,20 @@ struct LIBPKG_EXPORT PackageSearchResult {
StorageID id;
};
struct LIBPKG_EXPORT PackageBaseSearchResult {
PackageBaseSearchResult() = default;
PackageBaseSearchResult(const Database &database, const PackageBase &package, StorageID id);
/// \brief The related database.
/// \remarks
/// - The find functions always uses Database* and it is guaranteed to be never nullptr.
/// - The deserialization functions always use DatabaseInfo and the values might be empty if the source was empty.
/// - The serialization functions can cope with both alternatives.
const Database *db = nullptr;
const PackageBase *pkg = nullptr;
StorageID id = 0;
};
/*!
* \brief The DatabaseUsage enum specifies the usage of a database within pacman.
*/
@ -131,9 +145,11 @@ struct AtomicDateTime : public std::atomic<CppUtilities::DateTime> {
};
struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Database>, public ReflectiveRapidJSON::BinarySerializable<Database> {
using PackageVisitorBase = std::function<bool(StorageID, std::shared_ptr<PackageBase> &&)>; // package is invalidated/reused unless moved from!!!
using PackageVisitorMove = std::function<bool(StorageID, std::shared_ptr<Package> &&)>; // package is invalidated/reused unless moved from!!!
using PackageVisitorConst = std::function<bool(StorageID, const std::shared_ptr<Package> &)>;
using PackageVisitorByName = std::function<bool(std::string_view, const std::function<PackageSpec(void)> &)>;
using PackageVisitorByNameBase = std::function<bool(std::string_view, const std::function<StorageID(PackageBase&)> &)>;
friend struct PackageUpdater;
@ -153,7 +169,9 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);
std::vector<std::shared_ptr<Package>> findPackages(const std::function<bool(const Database &, const Package &)> &pred);
void allPackages(const PackageVisitorMove &visitor);
void allPackages(const PackageVisitorBase &visitor);
void allPackagesByName(const PackageVisitorByName &visitor);
void allPackagesByName(const PackageVisitorByNameBase &visitor);
std::size_t packageCount() const;
void providingPackages(const Dependency &dependency, bool reverse, const PackageVisitorConst &visitor);
void providingPackages(const std::string &libraryName, bool reverse, const PackageVisitorConst &visitor);
@ -162,6 +180,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
std::shared_ptr<Package> findPackage(StorageID packageID);
std::shared_ptr<Package> findPackage(const std::string &packageName);
PackageSpec findPackageWithID(const std::string &packageName);
StorageID findBasePackageWithID(const std::string &packageName, PackageBase &basePackage);
void removePackage(const std::string &packageName);
StorageID updatePackage(const std::shared_ptr<Package> &package);
StorageID forceUpdatePackage(const std::shared_ptr<Package> &package);
@ -233,6 +252,13 @@ inline bool PackageSearchResult::operator==(const PackageSearchResult &other) co
return ((!*db1 && !*db2) || (*db1 && *db2 && (**db1).name == (**db2).name)) && pkg == other.pkg;
}
inline PackageBaseSearchResult::PackageBaseSearchResult(const Database &database, const PackageBase &package, StorageID id)
: db(&database)
, pkg(&package)
, id(id)
{
}
} // namespace LibPkg
namespace std {
@ -247,7 +273,7 @@ template <> struct hash<LibPkg::PackageSearchResult> {
} else if (const auto *const db = std::get<LibPkg::Database *>(res.db)) {
dbName = &db->name;
}
return ((hash<string>()(dbName ? *dbName : string()) ^ (hash<std::shared_ptr<LibPkg::Package>>()(res.pkg) << 1)) >> 1);
return ((hash<string>()(dbName ? *dbName : string()) ^ (hash<std::shared_ptr<LibPkg::PackageBase>>()(res.pkg) << 1)) >> 1);
}
};
@ -265,6 +291,14 @@ template <>
LIBPKG_EXPORT void pull<LibPkg::PackageSearchResult>(LibPkg::PackageSearchResult &reflectable,
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors);
// declare custom (de)serialization for PackageBaseSearchResult
template <>
LIBPKG_EXPORT void push<LibPkg::PackageBaseSearchResult>(
const LibPkg::PackageBaseSearchResult &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator);
template <>
LIBPKG_EXPORT void pull<LibPkg::PackageBaseSearchResult>(LibPkg::PackageBaseSearchResult &reflectable,
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors);
// declare custom (de)serialization for AtomicDateTime
template <>
LIBPKG_EXPORT void push<LibPkg::AtomicDateTime>(

50
libpkg/data/mrulist.h Normal file
View File

@ -0,0 +1,50 @@
#ifndef LIBPKG_DATA_MRU_LIST_H
#define LIBPKG_DATA_MRU_LIST_H
#include <boost/multi_index/hashed_index.hpp>
#include <boost/multi_index/identity.hpp>
#include <boost/multi_index/sequenced_index.hpp>
#include <boost/multi_index_container.hpp>
namespace LibPkg {
template <typename Item> class MostRecentUseList {
using ItemList = boost::multi_index::multi_index_container<Item,
boost::multi_index::indexed_by<boost::multi_index::sequenced<>, boost::multi_index::hashed_unique<boost::multi_index::identity<Item>>>>;
public:
using item_type = Item;
using iterator = typename ItemList::iterator;
explicit MostRecentUseList(std::size_t limit = 1000)
: m_limit(limit)
{
}
void insert(const item_type &item)
{
auto [i, newItem] = std::pair<iterator, bool>(m_itemList.push_front(item));
if (!newItem) {
m_itemList.relocate(m_itemList.begin(), i);
} else if (m_itemList.size() > m_limit) {
m_itemList.pop_back();
}
}
iterator begin()
{
return m_itemList.begin();
}
iterator end()
{
return m_itemList.end();
}
private:
ItemList m_itemList;
std::size_t m_limit;
};
} // namespace LibPkg
#endif // LIBPKG_DATA_MRU_LIST_H

125
libpkg/data/package.cpp
View File

@ -163,7 +163,7 @@ ostream &operator<<(ostream &o, const PackageVersionPartComparison &res)
return o;
}
PackageVersionComparison Package::compareVersion(const Package &other) const
PackageVersionComparison PackageBase::compareVersion(const PackageBase &other) const
{
return PackageVersion::fromString(version).compare(PackageVersion::fromString(other.version));
}
@ -331,7 +331,7 @@ string LibPkg::Package::computeFileName(const char *extension) const
return argsToString(name, '-', version, '-', packageInfo->arch, '.', extension);
}
string LibPkg::Package::computeRegularPackageName() const
string LibPkg::PackageBase::computeRegularPackageName() const
{
if (name == "mingw-w64-headers" || name == "mingw-w64-crt") {
return string();
@ -352,6 +352,17 @@ string LibPkg::Package::computeRegularPackageName() const
return string();
}
void PackageBase::clear()
{
origin = PackageOrigin::Default;
timestamp = buildDate = DateTime();
name.clear();
version.clear();
arch.clear();
archs.clear();
description.clear();
}
bool Package::addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bool force)
{
if (&otherPackage == this) {
@ -369,7 +380,7 @@ bool Package::addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bo
// add package info from other package if this package has none at all
if (!packageInfo && otherPackage.packageInfo) {
packageInfo = make_unique<PackageInfo>(*(otherPackage.packageInfo));
packageInfo = std::make_optional<PackageInfo>(*(otherPackage.packageInfo));
}
// add source info from other package; at least add missing make and check dependencies
if (!sourceInfo) {
@ -552,36 +563,112 @@ namespace ReflectiveRapidJSON {
namespace JsonReflector {
template <>
LIBPKG_EXPORT void push<LibPkg::PackageSpec>(
const LibPkg::PackageSpec &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)
template <typename PackageSpecType, Traits::EnableIf<Traits::IsSpecializationOf<PackageSpecType, LibPkg::GenericPackageSpec>> * = nullptr>
static void internalPush(
const PackageSpecType &reflectable, ::RAPIDJSON_NAMESPACE::Value::Object &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)
{
// just serialize the package (and ignore the ID)
push(reflectable.pkg, value, allocator);
if (reflectable.pkg) {
push(*reflectable.pkg, value, allocator);
}
}
template <>
LIBPKG_EXPORT void pull<LibPkg::PackageSpec>(LibPkg::PackageSpec &reflectable,
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors)
template <typename PackageSpecType, Traits::EnableIf<Traits::IsSpecializationOf<PackageSpecType, LibPkg::GenericPackageSpec>> * = nullptr>
static void internalPull(
PackageSpecType &reflectable, const ::RAPIDJSON_NAMESPACE::GenericValue<::RAPIDJSON_NAMESPACE::UTF8<char>>::ConstObject &value, JsonDeserializationErrors *errors)
{
// allow the package being specified with ID or directly
if (!value.IsObject()) {
if (errors) {
errors->reportTypeMismatch<LibPkg::PackageSpec>(value.GetType());
}
return;
}
// find member
if (const auto pkg = value.FindMember("pkg"); pkg != value.MemberEnd()) {
pull(reflectable.pkg, pkg->value, errors);
reflectable.pkg = std::make_shared<typename decltype(reflectable.pkg)::element_type>();
pull(*reflectable.pkg, pkg->value, errors);
if (const auto id = value.FindMember("id"); id != value.MemberEnd()) {
pull(reflectable.id, id->value, errors);
}
} else {
pull(reflectable.pkg, value, errors);
reflectable.pkg = std::make_shared<typename decltype(reflectable.pkg)::element_type>();
pull(*reflectable.pkg, value, errors);
}
}
template <>
LIBPKG_EXPORT void push<LibPkg::PackageSpec>(const LibPkg::PackageSpec &reflectable, ::RAPIDJSON_NAMESPACE::Value::Object &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)
{
internalPush(reflectable, value, allocator);
}
template <>
LIBPKG_EXPORT void push<LibPkg::GenericPackageSpec<LibPkg::PackageBase>>(const LibPkg::GenericPackageSpec<LibPkg::PackageBase> &reflectable, ::RAPIDJSON_NAMESPACE::Value::Object &value,
RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)
{
internalPush(reflectable, value, allocator);
}
template <>
LIBPKG_EXPORT void pull<LibPkg::PackageSpec>(LibPkg::PackageSpec &reflectable,const ::RAPIDJSON_NAMESPACE::GenericValue<::RAPIDJSON_NAMESPACE::UTF8<char>>::ConstObject &value,
JsonDeserializationErrors *errors)
{
internalPull(reflectable, value, errors);
}
template <>
LIBPKG_EXPORT void pull<LibPkg::GenericPackageSpec<LibPkg::PackageBase>>(LibPkg::GenericPackageSpec<LibPkg::PackageBase> &reflectable,
const ::RAPIDJSON_NAMESPACE::GenericValue<::RAPIDJSON_NAMESPACE::UTF8<char>>::ConstObject &value, JsonDeserializationErrors *errors)
{
internalPull(reflectable, value, errors);
}
} // namespace JsonReflector
namespace BinaryReflector {
template <typename PackageSpecType, Traits::EnableIf<Traits::IsSpecializationOf<PackageSpecType, LibPkg::GenericPackageSpec>> * = nullptr>
BinaryVersion internalReadCustomType(BinaryDeserializer &deserializer, PackageSpecType &reflectable, BinaryVersion version)
{
// read version
using V = Versioning<::ReflectiveRapidJSON::BinarySerializable<PackageSpecType>>;
if constexpr (V::enabled) {
V::assertVersion(version = deserializer.readVariableLengthUIntBE(), "LibPkg::GenericPackageSpec");
}
// read members
deserializer.read(reflectable.id, version);
deserializer.read(reflectable.pkg, version);
return version;
}
template <typename PackageSpecType, Traits::EnableIf<Traits::IsSpecializationOf<PackageSpecType, LibPkg::GenericPackageSpec>> * = nullptr>
void internalWriteCustomType(BinarySerializer &serializer, const PackageSpecType &reflectable, BinaryVersion version)
{
// write version
using V = Versioning<::ReflectiveRapidJSON::BinarySerializable<PackageSpecType>>;
if constexpr (V::enabled) {
serializer.writeVariableLengthUIntBE(V::applyDefault(version));
}
// write members
serializer.write(reflectable.id, version);
serializer.write(reflectable.pkg, version);
}
template <>
LIBPKG_EXPORT BinaryVersion readCustomType<LibPkg::PackageSpec>(BinaryDeserializer &deserializer, LibPkg::PackageSpec &reflectable, BinaryVersion version)
{
return internalReadCustomType(deserializer, reflectable, version);
}
template <>
LIBPKG_EXPORT BinaryVersion readCustomType<LibPkg::GenericPackageSpec<LibPkg::PackageBase>>(BinaryDeserializer &deserializer, LibPkg::GenericPackageSpec<LibPkg::PackageBase> &reflectable, BinaryVersion version)
{
return internalReadCustomType(deserializer, reflectable, version);
}
template <>
LIBPKG_EXPORT void writeCustomType<LibPkg::PackageSpec>(BinarySerializer &serializer, const LibPkg::PackageSpec &reflectable, BinaryVersion version)
{
internalWriteCustomType(serializer, reflectable, version);
}
template <>
LIBPKG_EXPORT void writeCustomType<LibPkg::GenericPackageSpec<LibPkg::PackageBase>>(BinarySerializer &serializer, const LibPkg::GenericPackageSpec<LibPkg::PackageBase> &reflectable, BinaryVersion version)
{
internalWriteCustomType(serializer, reflectable, version);
}
} // namespace BinaryReflector
} // namespace ReflectiveRapidJSON

114
libpkg/data/package.h
View File

@ -17,6 +17,7 @@
#include <cstring>
#include <limits>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <unordered_set>
@ -201,6 +202,7 @@ LIBPKG_EXPORT std::ostream &operator<<(std::ostream &o, const std::vector<Depend
struct SourceFile : public ReflectiveRapidJSON::JsonSerializable<SourceFile>, public ReflectiveRapidJSON::BinarySerializable<SourceFile> {
std::string path;
std::string contents;
bool operator==(const SourceFile &) const = default;
};
struct LIBPKG_EXPORT SourceInfo : public ReflectiveRapidJSON::JsonSerializable<SourceInfo>,
@ -219,6 +221,7 @@ struct LIBPKG_EXPORT SourceInfo : public ReflectiveRapidJSON::JsonSerializable<S
std::string url;
std::vector<SourceFile> sources;
std::string directory;
bool operator==(const SourceInfo &other) const = default;
};
struct LIBPKG_EXPORT PackageInfo : public ReflectiveRapidJSON::JsonSerializable<PackageInfo>,
@ -232,6 +235,7 @@ struct LIBPKG_EXPORT PackageInfo : public ReflectiveRapidJSON::JsonSerializable<
std::string pgpSignature;
std::string arch; // arch of concrete binary package
std::uint32_t size = 0;
bool operator==(const PackageInfo &other) const = default;
};
struct LIBPKG_EXPORT InstallInfo : public ReflectiveRapidJSON::JsonSerializable<InstallInfo>,
@ -241,6 +245,7 @@ struct LIBPKG_EXPORT InstallInfo : public ReflectiveRapidJSON::JsonSerializable<
std::vector<std::string> backupFiles;
InstallStatus installStatus = InstallStatus::Unknown;
PackageValidation validationMethods = PackageValidation::None;
bool operator==(const InstallInfo &other) const = default;
};
/*!
@ -333,22 +338,25 @@ struct Package;
* \brief The PackageSpec struct holds a reference to a package.
* \remarks If id is non-zero, the package is part of a database using that ID.
*/
struct LIBPKG_EXPORT PackageSpec : public ReflectiveRapidJSON::JsonSerializable<PackageSpec>,
public ReflectiveRapidJSON::BinarySerializable<PackageSpec> {
explicit PackageSpec(StorageID id = 0, const std::shared_ptr<Package> &pkg = nullptr);
bool operator==(const PackageSpec &other) const;
template <typename PackageType = Package>
struct LIBPKG_EXPORT GenericPackageSpec : public ReflectiveRapidJSON::JsonSerializable<GenericPackageSpec<PackageType>>,
public ReflectiveRapidJSON::BinarySerializable<GenericPackageSpec<PackageType>> {
explicit GenericPackageSpec(StorageID id = 0, const std::shared_ptr<PackageType> &pkg = nullptr);
bool operator==(const GenericPackageSpec &other) const;
StorageID id;
std::shared_ptr<Package> pkg;
std::shared_ptr<PackageType> pkg;
};
using PackageSpec = GenericPackageSpec<>;
inline PackageSpec::PackageSpec(StorageID id, const std::shared_ptr<Package> &pkg)
template <typename PackageType>
inline GenericPackageSpec<PackageType>::GenericPackageSpec(StorageID id, const std::shared_ptr<PackageType> &pkg)
: id(id)
, pkg(pkg)
{
}
inline bool PackageSpec::operator==(const PackageSpec &other) const
template <typename PackageType> inline bool GenericPackageSpec<PackageType>::operator==(const GenericPackageSpec &other) const
{
return id ? id == other.id : pkg == other.pkg;
}
@ -357,8 +365,8 @@ inline bool PackageSpec::operator==(const PackageSpec &other) const
namespace std {
template <> struct hash<LibPkg::PackageSpec> {
std::size_t operator()(const LibPkg::PackageSpec &spec) const
template <typename PackageType> struct hash<LibPkg::GenericPackageSpec<PackageType>> {
std::size_t operator()(const LibPkg::GenericPackageSpec<PackageType> &spec) const
{
using std::hash;
return spec.id ? hash<decltype(spec.id)>()(spec.id) : hash<decltype(spec.pkg)>()(spec.pkg);
@ -369,19 +377,38 @@ template <> struct hash<LibPkg::PackageSpec> {
namespace LibPkg {
struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Package>, public ReflectiveRapidJSON::BinarySerializable<Package, 1> {
struct LIBPKG_EXPORT PackageBase : public ReflectiveRapidJSON::JsonSerializable<PackageBase>,
public ReflectiveRapidJSON::BinarySerializable<PackageBase, 1> {
PackageBase() = default;
PackageBase(const PackageBase &other) = default;
PackageBase(PackageBase &&other) = default;
PackageBase &operator=(PackageBase &&other) = default;
bool isSame(const PackageBase &other) const;
PackageVersionComparison compareVersion(const PackageBase &other) const;
std::string computeRegularPackageName() const;
void clear();
PackageOrigin origin = PackageOrigin::Default;
CppUtilities::DateTime timestamp, buildDate;
std::string name;
std::string version;
std::string arch;
std::vector<std::string> archs; // set if a split package overrides the base archs; if empty, archs from sourceInfo apply
std::string description;
};
struct LIBPKG_EXPORT Package : public PackageBase,
public ReflectiveRapidJSON::JsonSerializable<Package>,
public ReflectiveRapidJSON::BinarySerializable<Package, 1> {
Package() = default;
Package(const Package &other);
Package(const Package &other) = default;
Package(Package &&other) = default;
//Package &operator=(const Package &other);
Package &operator=(Package &&other) = default;
bool providesDependency(const Dependency &dependency) const;
static void exportProvides(
const std::shared_ptr<Package> &package, DependencySet &destinationProvides, std::unordered_set<std::string> &destinationLibProvides);
bool isSame(const Package &other) const;
PackageVersionComparison compareVersion(const Package &other) const;
std::string computeFileName(const char *extension = "pkg.tar.zst") const;
std::string computeRegularPackageName() const;
PackageNameData decomposeName() const;
void addInfoFromPkgInfoFile(const std::string &info);
void addDepsAndProvidesFromContainedDirectory(std::string_view directoryPath);
@ -391,24 +418,26 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
std::vector<std::string> processDllsReferencedByImportLibs(std::set<std::string> &&dllsReferencedByImportLibs);
bool addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bool force = false);
bool isArchAny() const;
using ReflectiveRapidJSON::JsonSerializable<Package>::fromJson;
using ReflectiveRapidJSON::JsonSerializable<Package>::toJson;
using ReflectiveRapidJSON::JsonSerializable<Package>::toJsonDocument;
using ReflectiveRapidJSON::BinarySerializable<Package, 1>::toBinary;
using ReflectiveRapidJSON::BinarySerializable<Package, 1>::restoreFromBinary;
using ReflectiveRapidJSON::BinarySerializable<Package, 1>::fromBinary;
static bool isPkgInfoFileOrBinary(const char *filePath, const char *fileName, mode_t mode);
static bool isLicense(const char *filePath, const char *fileName, mode_t mode);
static std::vector<PackageSpec> fromInfo(const std::string &info, bool isPackageInfo = false);
static std::vector<GenericPackageSpec<Package>> fromInfo(const std::string &info, bool isPackageInfo = false);
static std::shared_ptr<Package> fromDescription(const std::vector<std::string> &descriptionParts);
static void fromDatabaseFile(const std::string &archivePath, const std::function<bool(const std::shared_ptr<Package> &)> &visitor);
static std::shared_ptr<Package> fromPkgFile(const std::string &path);
static std::tuple<std::string_view, std::string_view, std::string_view> fileNameComponents(std::string_view fileName);
static std::shared_ptr<Package> fromPkgFileName(std::string_view fileName);
static std::vector<PackageSpec> fromAurRpcJson(const char *jsonData, std::size_t jsonSize, PackageOrigin origin = PackageOrigin::AurRpcInfo);
static std::vector<GenericPackageSpec<Package>> fromAurRpcJson(
const char *jsonData, std::size_t jsonSize, PackageOrigin origin = PackageOrigin::AurRpcInfo);
PackageOrigin origin = PackageOrigin::Default;
CppUtilities::DateTime timestamp;
std::string name;
std::string version;
std::vector<std::string> archs; // set if a split package overrides the base archs; if empty, archs from sourceInfo apply
std::string description;
using PackageBase::version;
std::string upstreamUrl;
std::vector<std::string> licenses;
std::vector<std::string> groups;
@ -419,32 +448,12 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable<Pack
std::vector<Dependency> replaces;
std::set<std::string> libprovides;
std::set<std::string> libdepends;
std::shared_ptr<SourceInfo> sourceInfo;
std::unique_ptr<PackageInfo> packageInfo;
std::unique_ptr<InstallInfo> installInfo;
std::optional<SourceInfo> sourceInfo;
std::optional<PackageInfo> packageInfo;
std::optional<InstallInfo> installInfo;
};
inline Package::Package(const Package &other)
: origin(other.origin)
, timestamp(other.timestamp)
, name(other.name)
, version(other.version)
, description(other.description)
, upstreamUrl(other.upstreamUrl)
, licenses(other.licenses)
, groups(other.groups)
, dependencies(other.dependencies)
, optionalDependencies(other.optionalDependencies)
, conflicts(other.conflicts)
, provides(other.provides)
, replaces(other.replaces)
, sourceInfo(other.sourceInfo)
, packageInfo()
, installInfo()
{
}
inline bool Package::isSame(const Package &other) const
inline bool PackageBase::isSame(const PackageBase &other) const
{
return name == other.name && version == other.version;
}
@ -502,17 +511,6 @@ namespace ReflectiveRapidJSON {
REFLECTIVE_RAPIDJSON_TREAT_AS_MULTI_MAP_OR_HASH(LibPkg::DependencySet);
namespace JsonReflector {
// declare custom (de)serialization for LibPkg::PackageSpec
template <>
LIBPKG_EXPORT void push<LibPkg::PackageSpec>(
const LibPkg::PackageSpec &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator);
template <>
LIBPKG_EXPORT void pull<LibPkg::PackageSpec>(LibPkg::PackageSpec &reflectable,
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors);
} // namespace JsonReflector
} // namespace ReflectiveRapidJSON
#endif // LIBPKG_DATA_PACKAGE_H

2
libpkg/data/storageprivate.h
View File

@ -8,7 +8,7 @@
namespace LibPkg {
using PackageStorage = LMDBSafe::TypedDBI<Package, LMDBSafe::index_on<Package, std::string, &Package::name>>;
using PackageStorage = LMDBSafe::TypedDBI<Package, LMDBSafe::index_on_base_member<Package, std::string, PackageBase, &PackageBase::name>>;
using DependencyStorage = LMDBSafe::TypedDBI<DatabaseDependency, LMDBSafe::index_on<Dependency, std::string, &DatabaseDependency::name>>;
using LibraryDependencyStorage
= LMDBSafe::TypedDBI<DatabaseLibraryDependency, LMDBSafe::index_on<DatabaseLibraryDependency, std::string, &DatabaseLibraryDependency::name>>;

41
libpkg/parser/package.cpp
View File

@ -193,13 +193,13 @@ PackageVersion PackageVersion::fromString(const char *versionString, size_t vers
#define valueString std::string_view(value, valueSize)
#define ensure_pkg_info \
if (!package.packageInfo) \
package.packageInfo = make_unique<PackageInfo>()
package.packageInfo = make_optional<PackageInfo>()
#define ensure_install_info \
if (!package.installInfo) \
package.installInfo = make_unique<InstallInfo>()
package.installInfo = make_optional<InstallInfo>()
void addPackageInfo(
Package &package, PackageVersion &version, const char *field, size_t fieldSize, const char *value, size_t valueSize, bool isPackageInfo)
static void addPackageInfo(Package &package, PackageVersion &version, const char *field, size_t fieldSize, const char *value, size_t valueSize,
bool isPackageInfo, std::size_t packageCount)
{
if_field("pkgbase")
{
@ -235,7 +235,7 @@ void addPackageInfo(
// add as binary arch when parsing PKGINFO
ensure_pkg_info;
package.packageInfo->arch = valueString;
} else if (package.sourceInfo.use_count() <= 1) {
} else if (!packageCount) {
// add to sourceInfo when still parsing base info
package.sourceInfo->archs.emplace_back(value, valueSize);
} else {
@ -451,11 +451,12 @@ static void addVersionInfo(Package &package, PackageVersion &version, bool isPac
static void parsePkgInfo(const std::string &info, const std::function<Package *(Package &)> &nextPackage, bool isPackageInfo)
{
// define variables to store intermediate results while still parsing package base
PackageVersion version;
Package basePackage;
auto version = PackageVersion();
auto packageCount = std::size_t();
auto basePackage = Package();
basePackage.origin = isPackageInfo ? PackageOrigin::PackageInfo : PackageOrigin::SourceInfo;
basePackage.sourceInfo = make_shared<SourceInfo>();
std::string &packageBase = basePackage.sourceInfo->name;
basePackage.sourceInfo = std::make_optional<SourceInfo>();
auto &packageBase = basePackage.sourceInfo->name;
// states
enum {
@ -560,17 +561,18 @@ static void parsePkgInfo(const std::string &info, const std::function<Package *(
}
// find next package
currentPackage = nextPackage(basePackage);
++packageCount;
}
// -> add field to ...
try {
if (currentPackage) {
// ... concrete package info if there's already a concrete package
addPackageInfo(*currentPackage, version, currentFieldName, currentFieldNameSize, currentFieldValue, currentFieldValueSize,
isPackageInfo);
isPackageInfo, packageCount);
} else {
// ... base info if still parsing general info
addPackageInfo(
basePackage, version, currentFieldName, currentFieldNameSize, currentFieldValue, currentFieldValueSize, isPackageInfo);
addPackageInfo(basePackage, version, currentFieldName, currentFieldNameSize, currentFieldValue, currentFieldValueSize,
isPackageInfo, packageCount);
}
} catch (const ConversionException &) {
// FIXME: error handling
@ -624,8 +626,8 @@ std::shared_ptr<Package> Package::fromDescription(const std::vector<std::string>
{
auto package = std::make_shared<Package>();
package->origin = PackageOrigin::Database;
package->sourceInfo = make_shared<SourceInfo>();
package->packageInfo = make_unique<PackageInfo>();
package->sourceInfo = std::make_optional<SourceInfo>();
package->packageInfo = std::make_optional<PackageInfo>();
for (const auto &desc : descriptionParts) {
// states
enum {
@ -890,7 +892,7 @@ std::shared_ptr<Package> Package::fromPkgFile(const string &path)
throw runtime_error("Package " % path + " does not contain a valid .PKGINFO");
}
if (!package->packageInfo) {
package->packageInfo = make_unique<PackageInfo>();
package->packageInfo = std::make_optional<PackageInfo>();
}
package->packageInfo->fileName = fileName(path);
package->addDepsAndProvidesFromOtherPackage(tmpPackageForLibraryDeps, true);
@ -933,7 +935,7 @@ std::shared_ptr<Package> Package::fromPkgFileName(std::string_view fileName)
pkg->name = name;
pkg->version = version;
pkg->provides.emplace_back(pkg->name, pkg->version);
pkg->packageInfo = make_unique<PackageInfo>();
pkg->packageInfo = std::make_optional<PackageInfo>();
pkg->packageInfo->fileName = fileName;
pkg->packageInfo->arch = arch;
return pkg;
@ -948,8 +950,9 @@ std::vector<PackageSpec> Package::fromAurRpcJson(const char *jsonData, std::size
packages.reserve(rpcMultiInfo.results.size());
for (auto &result : rpcMultiInfo.results) {
auto package = std::make_shared<Package>();
auto sourceInfo = std::make_shared<SourceInfo>();
auto &spec = packages.emplace_back();
auto *package = &*(spec.pkg = std::make_shared<Package>());
auto &sourceInfo = package->sourceInfo = std::make_optional<SourceInfo>();
package->origin = origin;
package->name = std::move(result.Name);
package->version = std::move(result.Version);
@ -979,8 +982,6 @@ std::vector<PackageSpec> Package::fromAurRpcJson(const char *jsonData, std::size
sourceInfo->firstSubmitted = DateTime::fromTimeStampGmt(result.FirstSubmitted);
sourceInfo->lastModified = DateTime::fromTimeStampGmt(result.LastModified);
sourceInfo->url = std::move(result.URLPath);
package->sourceInfo = std::move(sourceInfo);
packages.emplace_back(0, std::move(package));
}
return packages;
}

6
libpkg/tests/data.cpp
View File

@ -274,7 +274,7 @@ void DataTests::testComputingFileName()
pkg.name = "test";
pkg.version = "1.2-3";
CPPUNIT_ASSERT_EQUAL_MESSAGE("packageInfo required for computing filename", string(), pkg.computeFileName());
pkg.packageInfo = make_unique<PackageInfo>();
pkg.packageInfo = std::make_optional<PackageInfo>();
pkg.packageInfo->arch = "x86_64";
CPPUNIT_ASSERT_EQUAL_MESSAGE("file name computed from name, version and arch", "test-1.2-3-x86_64.pkg.tar.zst"s, pkg.computeFileName());
pkg.packageInfo->fileName = "explicitly-specified-filename";
@ -349,8 +349,8 @@ void DataTests::testComputingBuildOrder()
CPPUNIT_ASSERT_EQUAL(0_st, res.ignored.size());
// ignore cycle if not interested in that particular package
m_pkg2->packageInfo = std::make_unique<PackageInfo>();
tar->packageInfo = std::make_unique<PackageInfo>();
m_pkg2->packageInfo = std::make_optional<PackageInfo>();
tar->packageInfo = std::make_optional<PackageInfo>();
tar->dependencies.clear();
tar->dependencies.emplace_back("bar");
db.forceUpdatePackage(tar);

26
libpkg/tests/parser.cpp
View File

@ -4,6 +4,15 @@
#include "../parser/config.h"
#include "../parser/package.h"
namespace CppUtilities {
inline std::ostream &operator<<(std::ostream &out, const LibPkg::SourceInfo &sourceInfo)
{
const auto buff = sourceInfo.toJson();
out.write(buff.GetString(), ReflectiveRapidJSON::JsonReflector::rapidJsonSize(buff.GetSize()));
return out;
}
} // namespace CppUtilities
#include <c++utilities/conversion/stringbuilder.h>
#include <c++utilities/conversion/stringconversion.h>
#include <c++utilities/io/misc.h>
@ -194,8 +203,8 @@ void ParserTests::testParsingPlainSrcInfo()
CPPUNIT_ASSERT_MESSAGE("no regular dependencies"s, pkg1.dependencies.empty());
CPPUNIT_ASSERT_EQUAL_MESSAGE("optional dependencies"s,
vector<Dependency>{ Dependency("c++utilities-doc"s, string(), DependencyMode::Any, "API documentation"s) }, pkg1.optionalDependencies);
CPPUNIT_ASSERT_MESSAGE("source info present", pkg1.sourceInfo != nullptr);
CPPUNIT_ASSERT_MESSAGE("no package info present", pkg1.packageInfo == nullptr);
CPPUNIT_ASSERT_MESSAGE("source info present", pkg1.sourceInfo.has_value());
CPPUNIT_ASSERT_MESSAGE("no package info present", !pkg1.packageInfo.has_value());
CPPUNIT_ASSERT_EQUAL_MESSAGE(
"make dependencies"s, vector<Dependency>{ Dependency("cmake"s, "3.0"s, DependencyMode::GreatherEqual) }, pkg1.sourceInfo->makeDependencies);
CPPUNIT_ASSERT_EQUAL_MESSAGE("check dependencies"s, vector<Dependency>{ Dependency("cppunit"s) }, pkg1.sourceInfo->checkDependencies);
@ -225,10 +234,10 @@ void ParserTests::testParsingSplitPackageSrcInfo()
Dependency("mingw-w64-icu"s),
};
CPPUNIT_ASSERT_EQUAL_MESSAGE("dependencies (2)"s, dependencies2, pkg2.dependencies);
CPPUNIT_ASSERT_MESSAGE("source info present (1)", pkg1.sourceInfo != nullptr);
CPPUNIT_ASSERT_MESSAGE("source info present (2)", pkg2.sourceInfo != nullptr);
CPPUNIT_ASSERT_MESSAGE("no package info present (1)", pkg1.packageInfo == nullptr);
CPPUNIT_ASSERT_MESSAGE("no package info present (2)", pkg2.packageInfo == nullptr);
CPPUNIT_ASSERT_MESSAGE("source info present (1)", pkg1.sourceInfo.has_value());
CPPUNIT_ASSERT_MESSAGE("source info present (2)", pkg2.sourceInfo.has_value());
CPPUNIT_ASSERT_MESSAGE("no package info present (1)", !pkg1.packageInfo.has_value());
CPPUNIT_ASSERT_MESSAGE("no package info present (2)", !pkg2.packageInfo.has_value());
CPPUNIT_ASSERT_EQUAL_MESSAGE("pkgbase (1)"s, "mingw-w64-harfbuzz"s, pkg1.sourceInfo->name);
CPPUNIT_ASSERT_EQUAL_MESSAGE("pkgbase (2)"s, "mingw-w64-harfbuzz"s, pkg2.sourceInfo->name);
const vector<string> archs = { "any"s };
@ -243,8 +252,9 @@ void ParserTests::testParsingSplitPackageSrcInfoWithDifferentArchs()
CPPUNIT_ASSERT_EQUAL_MESSAGE("3 (split) packages present"s, 3ul, packages.size());
const auto &jre = packages[0].pkg, &jdk = packages[1].pkg, &doc = packages[2].pkg;
CPPUNIT_ASSERT_MESSAGE("source info present", jdk->sourceInfo);
CPPUNIT_ASSERT_EQUAL_MESSAGE("jre has same source info as base", jdk->sourceInfo, jre->sourceInfo);
CPPUNIT_ASSERT_MESSAGE("source info present (jdk)", jdk->sourceInfo.has_value());
CPPUNIT_ASSERT_MESSAGE("source info present (jre)", jre->sourceInfo.has_value());
CPPUNIT_ASSERT_EQUAL_MESSAGE("jre has same source info as base", jdk->sourceInfo->archs, jre->sourceInfo->archs);
CPPUNIT_ASSERT_EQUAL_MESSAGE("jdk-doc has same source info as base", jdk->sourceInfo, doc->sourceInfo);
CPPUNIT_ASSERT_EQUAL_MESSAGE("jre name", "jre"s, jre->name);
CPPUNIT_ASSERT_EQUAL_MESSAGE("jdk name", "jdk"s, jdk->name);

46
librepomgr/buildactions/buildaction.h
View File

@ -55,7 +55,7 @@ struct LIBREPOMGR_EXPORT PackageBuildData : public ReflectiveRapidJSON::JsonSeri
std::vector<std::shared_ptr<LibPkg::Package>> existingPackages;
std::string sourceDirectory;
std::string originalSourceDirectory;
std::shared_ptr<LibPkg::SourceInfo> sourceInfo;
std::optional<LibPkg::SourceInfo> sourceInfo;
std::vector<LibPkg::PackageSpec> packages;
std::vector<std::string> warnings;
std::string error;
@ -154,7 +154,14 @@ struct LIBREPOMGR_EXPORT BuildActionMessages : public ReflectiveRapidJSON::JsonS
class BuildProcessSession;
struct ServiceSetup;
struct LIBREPOMGR_EXPORT BuildAction : public std::enable_shared_from_this<BuildAction>,
struct LIBREPOMGR_EXPORT BuildActionBase : public ReflectiveRapidJSON::JsonSerializable<BuildActionBase>,
public ReflectiveRapidJSON::BinarySerializable<BuildActionBase, 1> {
using IdType = BuildActionIdType;
static constexpr IdType invalidId = std::numeric_limits<BuildActionBase::IdType>::max();
};
struct LIBREPOMGR_EXPORT BuildAction : public BuildActionBase,
public std::enable_shared_from_this<BuildAction>,
public ReflectiveRapidJSON::JsonSerializable<BuildAction>,
public ReflectiveRapidJSON::BinarySerializable<BuildAction, 1> {
friend InternalBuildAction;
@ -168,9 +175,6 @@ struct LIBREPOMGR_EXPORT BuildAction : public std::enable_shared_from_this<Build
friend void WebAPI::Routes::postCloneBuildActions(const WebAPI::Params &params, WebAPI::ResponseHandler &&handler);
public:
using IdType = BuildActionIdType;
static constexpr IdType invalidId = std::numeric_limits<BuildAction::IdType>::max();
explicit BuildAction(IdType id = invalidId, ServiceSetup *setup = nullptr) noexcept;
BuildAction &operator=(BuildAction &&other);
~BuildAction();
@ -198,6 +202,12 @@ public:
void streamFile(const WebAPI::Params &params, const std::string &filePath, boost::beast::string_view fileMimeType,
boost::beast::string_view contentDisposition = boost::beast::string_view());
ServiceSetup *setup();
using ReflectiveRapidJSON::JsonSerializable<BuildAction>::fromJson;
using ReflectiveRapidJSON::JsonSerializable<BuildAction>::toJson;
using ReflectiveRapidJSON::JsonSerializable<BuildAction>::toJsonDocument;
using ReflectiveRapidJSON::BinarySerializable<BuildAction, 1>::toBinary;
using ReflectiveRapidJSON::BinarySerializable<BuildAction, 1>::restoreFromBinary;
using ReflectiveRapidJSON::BinarySerializable<BuildAction, 1>::fromBinary;
protected:
private:
@ -207,28 +217,26 @@ private:
public:
IdType id;
BuildActionType type = BuildActionType::Invalid;
std::string taskName;
std::string templateName;
std::string directory;
std::vector<std::string> packageNames;
std::vector<std::string> sourceDbs, destinationDbs;
std::unordered_map<std::string, std::string> settings;
BuildActionFlagType flags = noBuildActionFlags;
BuildActionType type = BuildActionType::Invalid;
// only the following member variables are supposed to change after the build action has been added
// to the overall list of build actions
BuildActionStatus status = BuildActionStatus::Created;
BuildActionResult result = BuildActionResult::None;
std::variant<std::string, std::vector<std::string>, LibPkg::LicenseResult, LibPkg::PackageUpdates, BuildPreparation, BuildProgress,
PackageMovementResult, std::unordered_map<std::string, std::vector<RepositoryProblem>>, BuildActionMessages>
resultData;
std::vector<std::string> logfiles;
std::vector<std::string> artefacts;
CppUtilities::DateTime created = CppUtilities::DateTime::gmtNow();
CppUtilities::DateTime started;
CppUtilities::DateTime finished;
std::vector<IdType> startAfter;
std::string directory;
std::vector<std::string> sourceDbs, destinationDbs;
std::vector<std::string> packageNames;
BuildActionFlagType flags = noBuildActionFlags;
std::unordered_map<std::string, std::string> settings;
std::vector<std::string> logfiles;
std::vector<std::string> artefacts;
std::variant<std::string, std::vector<std::string>, LibPkg::LicenseResult, LibPkg::PackageUpdates, BuildPreparation, BuildProgress,
PackageMovementResult, std::unordered_map<std::string, std::vector<RepositoryProblem>>, BuildActionMessages>
resultData;
private:
LogContext m_log;

6
librepomgr/buildactions/preparebuild.cpp
View File

@ -350,7 +350,7 @@ void PrepareBuild::addResultFromSrcInfo(WebClient::AurSnapshotQuerySession &mult
auto snapshotResult = WebClient::AurSnapshotResult{ .packageName = packageName, .packages = LibPkg::Package::fromInfo(srcInfo, false) };
if (snapshotResult.packages.empty() || snapshotResult.packages.front().pkg->name.empty()) {
snapshotResult.error = "Unable to parse .SRCINFO: no package name present";
} else if (!(snapshotResult.sourceInfo = snapshotResult.packages.front().pkg->sourceInfo)) {
} else if (!snapshotResult.packages.front().pkg->sourceInfo.has_value()) {
snapshotResult.error = "Unable to parse .SRCINFO: no source info present";
}
multiSession.addResponse(std::move(snapshotResult));
@ -822,7 +822,9 @@ void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::Conta
continue;
}
auto &buildData = m_buildDataByPackage[response.packageName];
buildData.sourceInfo = move(response.sourceInfo);
if (!response.packages.empty() && response.packages.front().pkg) {
buildData.sourceInfo = response.packages.front().pkg->sourceInfo;
}
buildData.packages = move(response.packages);
buildData.error = move(response.error);
buildData.hasSource = buildData.sourceInfo && !buildData.packages.empty() && buildData.error.empty();

4
librepomgr/buildactions/reloadlibrarydependencies.cpp
View File

@ -228,10 +228,10 @@ bool ReloadLibraryDependencies::addRelevantPackage(LibPkg::StorageID packageID,
relevantPkg.info.name = package->name;
// -> assign certain fields which are used by addDepsAndProvidesFromOtherPackage() to check whether the packages are matching
relevantPkg.info.version = package->version;
relevantPkg.info.packageInfo = std::make_unique<LibPkg::PackageInfo>();
relevantPkg.info.packageInfo = std::make_optional<LibPkg::PackageInfo>();
relevantPkg.info.packageInfo->buildDate = package->packageInfo->buildDate;
// -> gather source info such as make and check dependencies as well
relevantPkg.info.sourceInfo = std::make_shared<LibPkg::SourceInfo>();
relevantPkg.info.sourceInfo = std::make_optional<LibPkg::SourceInfo>();
++m_remainingPackages;
return false;
}

3
librepomgr/buildactions/updatecheck.cpp
View File

@ -27,7 +27,8 @@ void UpdateCheck::run()
if (m_fromAur && !m_packageLookupDone
&& WebClient::queryAurPackagesForDatabase(m_buildAction->log(), m_setup, m_setup.building.ioContext,
&std::get<std::shared_lock<std::shared_mutex>>(configReadLock), **m_destinationDbs.begin(), [this](std::vector<LibPkg::PackageSpec> &&) {
&std::get<std::shared_lock<std::shared_mutex>>(configReadLock), **m_destinationDbs.begin(),
[this](std::vector<LibPkg::PackageSpec> &&) {
m_packageLookupDone = true;
run();
})) {

4
librepomgr/serversetup.h
View File

@ -44,7 +44,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
std::string pacmanConfigFilePath = "/etc/pacman.conf";
std::filesystem::path initialWorkingDirectory;
std::string workingDirectory = "workingdir";
std::string dbPath = "libpkg.db";
std::string dbPath = "libpkg-1.db";
std::uint32_t maxDbs = 512;
std::size_t packageCacheLimit = 1000;
@ -130,7 +130,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable {
// never changed after startup
unsigned short threadCount = 4;
boost::asio::io_context ioContext;
std::string dbPath = "librepomgr.db";
std::string dbPath = "librepomgr-1.db";
void initStorage(const char *path);
bool hasStorage() const;

51
librepomgr/webapi/routes.cpp
View File

@ -227,15 +227,18 @@ void getPackages(const Params &params, ResponseHandler &&handler)
const auto dbIterator = dbs.find(db.name);
return dbIterator == dbs.end() || dbIterator->second.find(db.arch) == dbIterator->second.end();
});
const auto pushPackage = details
? LibPkg::Config::PackageVisitorConst([&array, &document, &limit](Database &, LibPkg::StorageID, const std::shared_ptr<Package> &pkg) {
ReflectiveRapidJSON::JsonReflector::push(pkg, array, document.GetAllocator());
return array.Size() >= limit;
})
: ([&array, &document, &limit](Database &db, LibPkg::StorageID id, const std::shared_ptr<Package> &pkg) {
ReflectiveRapidJSON::JsonReflector::push(LibPkg::PackageSearchResult(db, pkg, id), array, document.GetAllocator());
const auto pushPackage = LibPkg::Config::PackageVisitorBase([&array, &document, &limit](Database &db, LibPkg::StorageID id, const std::shared_ptr<PackageBase> &pkg) {
ReflectiveRapidJSON::JsonReflector::push(LibPkg::PackageBaseSearchResult(db, *pkg, id), array, document.GetAllocator());
return array.Size() >= limit;
});
const auto pushBasePackage = [&array, &document, &limit](Database &db, LibPkg::StorageID id, const PackageBase &pkg) {
ReflectiveRapidJSON::JsonReflector::push(LibPkg::PackageBaseSearchResult(db, pkg, id), array, document.GetAllocator());
return array.Size() >= limit;
};
const auto pushPackageDetails = !details ? LibPkg::Config::PackageVisitorConst() : [&array, &document, &limit](Database &, LibPkg::StorageID, const std::shared_ptr<Package> &pkg) {
ReflectiveRapidJSON::JsonReflector::push(pkg, array, document.GetAllocator());
return array.Size() >= limit;
};
auto aurPackages = std::vector<PackageSearchResult>();
auto neededAurPackages = std::vector<std::string>();
@ -261,21 +264,28 @@ void getPackages(const Params &params, ResponseHandler &&handler)
--limit;
}
if (!isDbAur && (!dbs.empty() || !onlyFromAur)) {
params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushPackage);
if (details) {
params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushPackageDetails);
} else {
params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushPackage);
}
}
break;
}
case Mode::NameContains:
case Mode::NameContains: {
auto basePackage = PackageBase();
params.setup.config.packagesByName(
visitDb, [&](LibPkg::Database &db, std::string_view packageName, const std::function<PackageSpec(void)> &getPackage) {
visitDb, [&](LibPkg::Database &db, std::string_view packageName, const std::function<StorageID(PackageBase&)> &getPackage) {
if (packageName.find(name) != std::string_view::npos) {
const auto [packageID, package] = getPackage();
if (!package) {
const auto packageID = getPackage(basePackage);
if (!packageID) {
cerr << Phrases::ErrorMessage << "Broken index in db \"" << db.name << "\": package \"" << packageName
<< "\" does not exist" << std::endl;
return false;
}
return pushPackage(db, packageID, package);
const auto stopSearch = pushBasePackage(db, packageID, basePackage);
basePackage.clear();
return stopSearch;
}
return false;
});
@ -283,19 +293,23 @@ void getPackages(const Params &params, ResponseHandler &&handler)
neededAurPackages.emplace_back(std::move(name));
}
break;
case Mode::Regex:
}
case Mode::Regex: {
try {
const auto regex = std::regex(name.data(), name.size());
auto basePackage = PackageBase();
params.setup.config.packagesByName(
visitDb, [&](LibPkg::Database &db, std::string_view packageName, const std::function<PackageSpec(void)> &getPackage) {
visitDb, [&](LibPkg::Database &db, std::string_view packageName, const std::function<StorageID(PackageBase&)> &getPackage) {
if (std::regex_match(packageName.begin(), packageName.end(), regex)) {
const auto [packageID, package] = getPackage();
if (!package) {
const auto packageID = getPackage(basePackage);
if (!packageID) {
cerr << Phrases::ErrorMessage << "Broken index in db \"" << db.name << "\": package \"" << packageName
<< "\" does not exist" << std::endl;
return false;
}
return pushPackage(db, packageID, package);
const auto stopSearch = pushBasePackage(db, packageID, basePackage);
basePackage.clear();
return stopSearch;
}
return false;
});
@ -303,6 +317,7 @@ void getPackages(const Params &params, ResponseHandler &&handler)
throw BadRequest(argsToString("regex is invalid: ", e.what()));
}
break;
}
case Mode::Provides:
case Mode::Depends:
params.setup.config.providingPackages(Dependency::fromString(name), mode == Mode::Depends, visitDb, pushPackage);

2
librepomgr/webclient/aur.cpp
View File

@ -282,7 +282,7 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vector<A
}
if (result.packages.empty() || result.packages.front().pkg->name.empty()) {
result.error = "Unable to parse .SRCINFO: no package name present";
} else if (!(result.sourceInfo = result.packages.front().pkg->sourceInfo)) {
} else if (!result.packages.front().pkg->sourceInfo.has_value()) {
result.error = "Unable to parse .SRCINFO: no source info present";
}
multiSession->addResponse(move(result));

1
librepomgr/webclient/aur.h
View File

@ -19,7 +19,6 @@ namespace WebClient {
struct AurSnapshotResult {
std::string packageName;
std::string errorOutput;
std::shared_ptr<LibPkg::SourceInfo> sourceInfo;
std::vector<LibPkg::PackageSpec> packages;
std::string error;
};