From 68d67f543fa1a90f2a6825a6745360fec61f8c55 Mon Sep 17 00:00:00 2001 From: Martchus Date: Sun, 15 May 2022 03:24:10 +0200 Subject: [PATCH] Refactor to optimize deserialization of base-data * Break backwards compatibility * Allow to deserialize only base-data of packages and build actions to potentially speed up showing tables * Speed up package search in many cases by only deserializing base-data (unless details are actually wanted) --- 3rdparty/lmdb-safe | 2 +- libpkg/algo/search.cpp | 34 +++++ libpkg/data/config.h | 5 + libpkg/data/database.cpp | 81 +++++++++--- libpkg/data/database.h | 36 ++++- libpkg/data/mrulist.h | 50 +++++++ libpkg/data/package.cpp | 125 +++++++++++++++--- libpkg/data/package.h | 114 ++++++++-------- libpkg/data/storageprivate.h | 2 +- libpkg/parser/package.cpp | 41 +++--- libpkg/tests/data.cpp | 6 +- libpkg/tests/parser.cpp | 26 ++-- librepomgr/buildactions/buildaction.h | 46 ++++--- librepomgr/buildactions/preparebuild.cpp | 6 +- .../reloadlibrarydependencies.cpp | 4 +- librepomgr/buildactions/updatecheck.cpp | 3 +- librepomgr/serversetup.h | 4 +- librepomgr/webapi/routes.cpp | 51 ++++--- librepomgr/webclient/aur.cpp | 2 +- librepomgr/webclient/aur.h | 1 - 20 files changed, 467 insertions(+), 172 deletions(-) create mode 100644 libpkg/data/mrulist.h diff --git a/3rdparty/lmdb-safe b/3rdparty/lmdb-safe index 2f5eb32..9a9bed0 160000 --- a/3rdparty/lmdb-safe +++ b/3rdparty/lmdb-safe @@ -1 +1 @@ -Subproject commit 2f5eb32a33130c4deb139118ebbd2dfa634d69bd +Subproject commit 9a9bed0f462244192e7f8524077c0daef64047c6 diff --git a/libpkg/algo/search.cpp b/libpkg/algo/search.cpp index 04866a3..54172a4 100644 --- a/libpkg/algo/search.cpp +++ b/libpkg/algo/search.cpp @@ -200,6 +200,29 @@ void Config::packages(std::string_view dbName, std::string_view dbArch, const st } } +void Config::packages(std::string_view dbName, std::string_view dbArch, const std::string &packageName, const DatabaseVisitor &databaseVisitor, + const PackageVisitorBase &visitor) +{ + // don't allow to iterate though all packages + if (packageName.empty()) { + return; + } + auto basePackage = std::make_shared(); + for (auto &db : databases) { + if ((!dbName.empty() && dbName != db.name) || (!dbArch.empty() && dbArch != db.arch) || (databaseVisitor && databaseVisitor(db))) { + continue; + } + if (!basePackage) { + basePackage = std::make_shared(); + } else { + basePackage->clear(); + } + if (const auto id = db.findBasePackageWithID(packageName, *basePackage)) { + visitor(db, id, std::move(basePackage)); + } + } +} + void Config::packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByName &visitor) { for (auto &db : databases) { @@ -211,6 +234,17 @@ void Config::packagesByName(const DatabaseVisitor &databaseVisitor, const Packag } } +void Config::packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByNameBase &visitor) +{ + for (auto &db : databases) { + if (databaseVisitor && databaseVisitor(db)) { + continue; + } + db.allPackagesByName( + [&](std::string_view packageName, const std::function &getPackage) { return visitor(db, packageName, getPackage); }); + } +} + void Config::providingPackages(const Dependency &dependency, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor) { for (auto &db : databases) { diff --git a/libpkg/data/config.h b/libpkg/data/config.h index 0da92a4..199eb0e 100644 --- a/libpkg/data/config.h +++ b/libpkg/data/config.h @@ -107,10 +107,12 @@ constexpr bool operator&(BuildOrderOptions lhs, BuildOrderOptions rhs) struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::BinarySerializable { using DatabaseVisitor = std::function; + using PackageVisitorBase = std::function &&)>; // package is invalidated/reused unless moved from!!! using PackageVisitorMove = std::function &&)>; // package is invalidated/reused unless moved from!!! using PackageVisitorConst = std::function &)>; using PackageVisitorByName = std::function &)>; + using PackageVisitorByNameBase = std::function &)>; explicit Config(); ~Config(); @@ -164,7 +166,10 @@ struct LIBPKG_EXPORT Config : public Lockable, public ReflectiveRapidJSON::Binar // package iteration void packages(std::string_view dbName, std::string_view dbArch, const std::string &packageName, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor); + void packages(std::string_view dbName, std::string_view dbArch, const std::string &packageName, const DatabaseVisitor &databaseVisitor, + const PackageVisitorBase &visitor); void packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByName &visitor); + void packagesByName(const DatabaseVisitor &databaseVisitor, const PackageVisitorByNameBase &visitor); void providingPackages(const Dependency &dependency, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor); void providingPackages(const std::string &libraryName, bool reverse, const DatabaseVisitor &databaseVisitor, const PackageVisitorConst &visitor); diff --git a/libpkg/data/database.cpp b/libpkg/data/database.cpp index fc3f8db..20d6c3b 100644 --- a/libpkg/data/database.cpp +++ b/libpkg/data/database.cpp @@ -299,12 +299,33 @@ void Database::allPackages(const PackageVisitorMove &visitor) } } +void Database::allPackages(const PackageVisitorBase &visitor) +{ + auto txn = m_storage->packages.getROTransaction(); + for (auto i = txn.begin(); i != txn.end(); ++i) { + if (visitor(i.getID(), std::move(i.getPointer()))) { + return; + } + } +} + void LibPkg::Database::allPackagesByName(const PackageVisitorByName &visitor) { auto txn = m_storage->packages.getROTransaction(); for (auto i = txn.begin_idx<0, std::shared_ptr>(); i != txn.end(); ++i) { const auto packageName = i.getKey().get(); - if (visitor(packageName, [this, &txn, &i]() { return m_storage->packageCache.retrieve(*m_storage, &txn, i.value()); })) { + if (visitor(packageName, [this, &txn, &i] { return m_storage->packageCache.retrieve(*m_storage, &txn, i.value()); })) { + return; + } + } +} + +void LibPkg::Database::allPackagesByName(const PackageVisitorByNameBase &visitor) +{ + auto txn = m_storage->packages.getROTransaction(); + for (auto i = txn.begin_idx<0, std::shared_ptr>(); i != txn.end(); ++i) { + const auto packageName = i.getKey().get(); + if (visitor(packageName, [&txn, &i] (PackageBase &pkg) { return txn.get(i.value(), pkg); })) { return; } } @@ -393,6 +414,12 @@ PackageSpec Database::findPackageWithID(const std::string &packageName) return m_storage->packageCache.retrieve(*m_storage, packageName); } +StorageID Database::findBasePackageWithID(const std::string &packageName, PackageBase &basePackage) +{ + auto txn = m_storage->packages.getROTransaction(); + return txn.get<0, PackageBase>(packageName, basePackage); +} + void Database::removePackage(const std::string &packageName) { const auto lock = std::unique_lock(m_storage->updateMutex); @@ -504,7 +531,7 @@ std::unordered_map Database::detectUnresolv // add packages to list of unresolved packages for (const auto &affectedPackageID : requiredDep.relevantPackages) { const auto affectedPackage = findPackage(affectedPackageID); - unresolvedPackages[PackageSpec(affectedPackageID, affectedPackage)].deps.emplace_back(requiredDep); + unresolvedPackages[GenericPackageSpec(affectedPackageID, affectedPackage)].deps.emplace_back(requiredDep); } } @@ -545,7 +572,7 @@ std::unordered_map Database::detectUnresolv // add packages to list of unresolved packages for (const auto &affectedPackageID : requiredLib.relevantPackages) { const auto affectedPackage = findPackage(affectedPackageID); - unresolvedPackages[PackageSpec(affectedPackageID, affectedPackage)].libs.emplace_back(requiredLib.name); + unresolvedPackages[GenericPackageSpec(affectedPackageID, affectedPackage)].libs.emplace_back(requiredLib.name); } } @@ -801,7 +828,7 @@ PackageUpdater::~PackageUpdater() { } -LibPkg::PackageSpec LibPkg::PackageUpdater::findPackageWithID(const std::string &packageName) +PackageSpec LibPkg::PackageUpdater::findPackageWithID(const std::string &packageName) { return m_database.m_storage->packageCache.retrieve(*m_database.m_storage, &m_d->packagesTxn, packageName); } @@ -901,14 +928,14 @@ void push( push(pkg->timestamp, "timestamp", obj, allocator); push(pkg->version, "version", obj, allocator); push(pkg->description, "description", obj, allocator); - if (const auto &pkgInfo = pkg->packageInfo) { - push(pkgInfo->arch, "arch", obj, allocator); - push(pkgInfo->buildDate, "buildDate", obj, allocator); + if (!pkg->arch.empty()) { + push(pkg->arch, "arch", obj, allocator); + } + if (!pkg->buildDate.isNull()) { + push(pkg->buildDate, "buildDate", obj, allocator); } if (!pkg->archs.empty()) { push(pkg->archs, "archs", obj, allocator); - } else if (const auto &srcInfo = pkg->sourceInfo) { - push(srcInfo->archs, "archs", obj, allocator); } if (const auto *const dbInfo = std::get_if(&reflectable.db)) { if (!dbInfo->name.empty()) { @@ -946,18 +973,42 @@ void pull(LibPkg::PackageSearchResult &reflectable, ReflectiveRapidJSON::JsonReflector::pull(pkg->timestamp, "timestamp", obj, errors); ReflectiveRapidJSON::JsonReflector::pull(pkg->version, "version", obj, errors); ReflectiveRapidJSON::JsonReflector::pull(pkg->description, "description", obj, errors); - auto &pkgInfo = pkg->packageInfo; - if (!pkgInfo) { - pkgInfo = make_unique(); - } - ReflectiveRapidJSON::JsonReflector::pull(pkgInfo->arch, "arch", obj, errors); - ReflectiveRapidJSON::JsonReflector::pull(pkgInfo->buildDate, "buildDate", obj, errors); + ReflectiveRapidJSON::JsonReflector::pull(pkg->arch, "arch", obj, errors); + ReflectiveRapidJSON::JsonReflector::pull(pkg->buildDate, "buildDate", obj, errors); ReflectiveRapidJSON::JsonReflector::pull(pkg->archs, "archs", obj, errors); auto &dbInfo = reflectable.db.emplace(); ReflectiveRapidJSON::JsonReflector::pull(dbInfo.name, "db", obj, errors); ReflectiveRapidJSON::JsonReflector::pull(dbInfo.arch, "dbArch", obj, errors); } +template <> +void push( + const LibPkg::PackageBaseSearchResult &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator) +{ + // serialize PackageBaseSearchResult object in accordance with PackageSearchResult + value.SetObject(); + auto obj = value.GetObject(); + if (auto &pkg = reflectable.pkg) { + push(*pkg, obj, allocator); + } + if (const auto &db = reflectable.db) { + push(db->name, "db", obj, allocator); + if (!db->arch.empty()) { + push(db->arch, "dbArch", obj, allocator); + } + } +} + +template <> +void pull(LibPkg::PackageBaseSearchResult &reflectable, + const RAPIDJSON_NAMESPACE::GenericValue> &value, JsonDeserializationErrors *errors) +{ + CPP_UTILITIES_UNUSED(reflectable) + CPP_UTILITIES_UNUSED(value) + CPP_UTILITIES_UNUSED(errors) + throw std::logic_error("Attempt to deserialize LibPkg::PackageBaseSearchResult"); +} + template <> void push( const LibPkg::AtomicDateTime &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator) diff --git a/libpkg/data/database.h b/libpkg/data/database.h index 4959a7f..c9d1a80 100644 --- a/libpkg/data/database.h +++ b/libpkg/data/database.h @@ -42,6 +42,20 @@ struct LIBPKG_EXPORT PackageSearchResult { StorageID id; }; +struct LIBPKG_EXPORT PackageBaseSearchResult { + PackageBaseSearchResult() = default; + PackageBaseSearchResult(const Database &database, const PackageBase &package, StorageID id); + + /// \brief The related database. + /// \remarks + /// - The find functions always uses Database* and it is guaranteed to be never nullptr. + /// - The deserialization functions always use DatabaseInfo and the values might be empty if the source was empty. + /// - The serialization functions can cope with both alternatives. + const Database *db = nullptr; + const PackageBase *pkg = nullptr; + StorageID id = 0; +}; + /*! * \brief The DatabaseUsage enum specifies the usage of a database within pacman. */ @@ -131,9 +145,11 @@ struct AtomicDateTime : public std::atomic { }; struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable, public ReflectiveRapidJSON::BinarySerializable { + using PackageVisitorBase = std::function &&)>; // package is invalidated/reused unless moved from!!! using PackageVisitorMove = std::function &&)>; // package is invalidated/reused unless moved from!!! using PackageVisitorConst = std::function &)>; using PackageVisitorByName = std::function &)>; + using PackageVisitorByNameBase = std::function &)>; friend struct PackageUpdater; @@ -153,7 +169,9 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable> findPackages(const std::function &pred); void allPackages(const PackageVisitorMove &visitor); + void allPackages(const PackageVisitorBase &visitor); void allPackagesByName(const PackageVisitorByName &visitor); + void allPackagesByName(const PackageVisitorByNameBase &visitor); std::size_t packageCount() const; void providingPackages(const Dependency &dependency, bool reverse, const PackageVisitorConst &visitor); void providingPackages(const std::string &libraryName, bool reverse, const PackageVisitorConst &visitor); @@ -162,6 +180,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable findPackage(StorageID packageID); std::shared_ptr findPackage(const std::string &packageName); PackageSpec findPackageWithID(const std::string &packageName); + StorageID findBasePackageWithID(const std::string &packageName, PackageBase &basePackage); void removePackage(const std::string &packageName); StorageID updatePackage(const std::shared_ptr &package); StorageID forceUpdatePackage(const std::shared_ptr &package); @@ -233,6 +252,13 @@ inline bool PackageSearchResult::operator==(const PackageSearchResult &other) co return ((!*db1 && !*db2) || (*db1 && *db2 && (**db1).name == (**db2).name)) && pkg == other.pkg; } +inline PackageBaseSearchResult::PackageBaseSearchResult(const Database &database, const PackageBase &package, StorageID id) + : db(&database) + , pkg(&package) + , id(id) +{ +} + } // namespace LibPkg namespace std { @@ -247,7 +273,7 @@ template <> struct hash { } else if (const auto *const db = std::get(res.db)) { dbName = &db->name; } - return ((hash()(dbName ? *dbName : string()) ^ (hash>()(res.pkg) << 1)) >> 1); + return ((hash()(dbName ? *dbName : string()) ^ (hash>()(res.pkg) << 1)) >> 1); } }; @@ -265,6 +291,14 @@ template <> LIBPKG_EXPORT void pull(LibPkg::PackageSearchResult &reflectable, const RAPIDJSON_NAMESPACE::GenericValue> &value, JsonDeserializationErrors *errors); +// declare custom (de)serialization for PackageBaseSearchResult +template <> +LIBPKG_EXPORT void push( + const LibPkg::PackageBaseSearchResult &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator); +template <> +LIBPKG_EXPORT void pull(LibPkg::PackageBaseSearchResult &reflectable, + const RAPIDJSON_NAMESPACE::GenericValue> &value, JsonDeserializationErrors *errors); + // declare custom (de)serialization for AtomicDateTime template <> LIBPKG_EXPORT void push( diff --git a/libpkg/data/mrulist.h b/libpkg/data/mrulist.h new file mode 100644 index 0000000..6db5b86 --- /dev/null +++ b/libpkg/data/mrulist.h @@ -0,0 +1,50 @@ +#ifndef LIBPKG_DATA_MRU_LIST_H +#define LIBPKG_DATA_MRU_LIST_H + +#include +#include +#include +#include + +namespace LibPkg { + +template class MostRecentUseList { + using ItemList = boost::multi_index::multi_index_container, boost::multi_index::hashed_unique>>>; + +public: + using item_type = Item; + using iterator = typename ItemList::iterator; + + explicit MostRecentUseList(std::size_t limit = 1000) + : m_limit(limit) + { + } + + void insert(const item_type &item) + { + auto [i, newItem] = std::pair(m_itemList.push_front(item)); + if (!newItem) { + m_itemList.relocate(m_itemList.begin(), i); + } else if (m_itemList.size() > m_limit) { + m_itemList.pop_back(); + } + } + + iterator begin() + { + return m_itemList.begin(); + } + iterator end() + { + return m_itemList.end(); + } + +private: + ItemList m_itemList; + std::size_t m_limit; +}; + +} // namespace LibPkg + +#endif // LIBPKG_DATA_MRU_LIST_H diff --git a/libpkg/data/package.cpp b/libpkg/data/package.cpp index ecf9a03..a100add 100644 --- a/libpkg/data/package.cpp +++ b/libpkg/data/package.cpp @@ -163,7 +163,7 @@ ostream &operator<<(ostream &o, const PackageVersionPartComparison &res) return o; } -PackageVersionComparison Package::compareVersion(const Package &other) const +PackageVersionComparison PackageBase::compareVersion(const PackageBase &other) const { return PackageVersion::fromString(version).compare(PackageVersion::fromString(other.version)); } @@ -331,7 +331,7 @@ string LibPkg::Package::computeFileName(const char *extension) const return argsToString(name, '-', version, '-', packageInfo->arch, '.', extension); } -string LibPkg::Package::computeRegularPackageName() const +string LibPkg::PackageBase::computeRegularPackageName() const { if (name == "mingw-w64-headers" || name == "mingw-w64-crt") { return string(); @@ -352,6 +352,17 @@ string LibPkg::Package::computeRegularPackageName() const return string(); } +void PackageBase::clear() +{ + origin = PackageOrigin::Default; + timestamp = buildDate = DateTime(); + name.clear(); + version.clear(); + arch.clear(); + archs.clear(); + description.clear(); +} + bool Package::addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bool force) { if (&otherPackage == this) { @@ -369,7 +380,7 @@ bool Package::addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bo // add package info from other package if this package has none at all if (!packageInfo && otherPackage.packageInfo) { - packageInfo = make_unique(*(otherPackage.packageInfo)); + packageInfo = std::make_optional(*(otherPackage.packageInfo)); } // add source info from other package; at least add missing make and check dependencies if (!sourceInfo) { @@ -552,36 +563,112 @@ namespace ReflectiveRapidJSON { namespace JsonReflector { -template <> -LIBPKG_EXPORT void push( - const LibPkg::PackageSpec &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator) +template > * = nullptr> +static void internalPush( + const PackageSpecType &reflectable, ::RAPIDJSON_NAMESPACE::Value::Object &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator) { // just serialize the package (and ignore the ID) - push(reflectable.pkg, value, allocator); + if (reflectable.pkg) { + push(*reflectable.pkg, value, allocator); + } } -template <> -LIBPKG_EXPORT void pull(LibPkg::PackageSpec &reflectable, - const RAPIDJSON_NAMESPACE::GenericValue> &value, JsonDeserializationErrors *errors) +template > * = nullptr> +static void internalPull( + PackageSpecType &reflectable, const ::RAPIDJSON_NAMESPACE::GenericValue<::RAPIDJSON_NAMESPACE::UTF8>::ConstObject &value, JsonDeserializationErrors *errors) { - // allow the package being specified with ID or directly - if (!value.IsObject()) { - if (errors) { - errors->reportTypeMismatch(value.GetType()); - } - return; - } // find member if (const auto pkg = value.FindMember("pkg"); pkg != value.MemberEnd()) { - pull(reflectable.pkg, pkg->value, errors); + reflectable.pkg = std::make_shared(); + pull(*reflectable.pkg, pkg->value, errors); if (const auto id = value.FindMember("id"); id != value.MemberEnd()) { pull(reflectable.id, id->value, errors); } } else { - pull(reflectable.pkg, value, errors); + reflectable.pkg = std::make_shared(); + pull(*reflectable.pkg, value, errors); } } +template <> +LIBPKG_EXPORT void push(const LibPkg::PackageSpec &reflectable, ::RAPIDJSON_NAMESPACE::Value::Object &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator) +{ + internalPush(reflectable, value, allocator); +} +template <> +LIBPKG_EXPORT void push>(const LibPkg::GenericPackageSpec &reflectable, ::RAPIDJSON_NAMESPACE::Value::Object &value, + RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator) +{ + internalPush(reflectable, value, allocator); +} +template <> +LIBPKG_EXPORT void pull(LibPkg::PackageSpec &reflectable,const ::RAPIDJSON_NAMESPACE::GenericValue<::RAPIDJSON_NAMESPACE::UTF8>::ConstObject &value, + JsonDeserializationErrors *errors) +{ + internalPull(reflectable, value, errors); +} +template <> +LIBPKG_EXPORT void pull>(LibPkg::GenericPackageSpec &reflectable, + const ::RAPIDJSON_NAMESPACE::GenericValue<::RAPIDJSON_NAMESPACE::UTF8>::ConstObject &value, JsonDeserializationErrors *errors) +{ + internalPull(reflectable, value, errors); +} + } // namespace JsonReflector +namespace BinaryReflector { + +template > * = nullptr> +BinaryVersion internalReadCustomType(BinaryDeserializer &deserializer, PackageSpecType &reflectable, BinaryVersion version) +{ + // read version + using V = Versioning<::ReflectiveRapidJSON::BinarySerializable>; + if constexpr (V::enabled) { + V::assertVersion(version = deserializer.readVariableLengthUIntBE(), "LibPkg::GenericPackageSpec"); + } + // read members + deserializer.read(reflectable.id, version); + deserializer.read(reflectable.pkg, version); + return version; +} + +template > * = nullptr> +void internalWriteCustomType(BinarySerializer &serializer, const PackageSpecType &reflectable, BinaryVersion version) +{ + // write version + using V = Versioning<::ReflectiveRapidJSON::BinarySerializable>; + if constexpr (V::enabled) { + serializer.writeVariableLengthUIntBE(V::applyDefault(version)); + } + // write members + serializer.write(reflectable.id, version); + serializer.write(reflectable.pkg, version); +} + +template <> +LIBPKG_EXPORT BinaryVersion readCustomType(BinaryDeserializer &deserializer, LibPkg::PackageSpec &reflectable, BinaryVersion version) +{ + return internalReadCustomType(deserializer, reflectable, version); +} + +template <> +LIBPKG_EXPORT BinaryVersion readCustomType>(BinaryDeserializer &deserializer, LibPkg::GenericPackageSpec &reflectable, BinaryVersion version) +{ + return internalReadCustomType(deserializer, reflectable, version); +} + +template <> +LIBPKG_EXPORT void writeCustomType(BinarySerializer &serializer, const LibPkg::PackageSpec &reflectable, BinaryVersion version) +{ + internalWriteCustomType(serializer, reflectable, version); +} + +template <> +LIBPKG_EXPORT void writeCustomType>(BinarySerializer &serializer, const LibPkg::GenericPackageSpec &reflectable, BinaryVersion version) +{ + internalWriteCustomType(serializer, reflectable, version); +} + +} // namespace BinaryReflector + } // namespace ReflectiveRapidJSON diff --git a/libpkg/data/package.h b/libpkg/data/package.h index d710b48..ec1ca6c 100644 --- a/libpkg/data/package.h +++ b/libpkg/data/package.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -201,6 +202,7 @@ LIBPKG_EXPORT std::ostream &operator<<(std::ostream &o, const std::vector, public ReflectiveRapidJSON::BinarySerializable { std::string path; std::string contents; + bool operator==(const SourceFile &) const = default; }; struct LIBPKG_EXPORT SourceInfo : public ReflectiveRapidJSON::JsonSerializable, @@ -219,6 +221,7 @@ struct LIBPKG_EXPORT SourceInfo : public ReflectiveRapidJSON::JsonSerializable sources; std::string directory; + bool operator==(const SourceInfo &other) const = default; }; struct LIBPKG_EXPORT PackageInfo : public ReflectiveRapidJSON::JsonSerializable, @@ -232,6 +235,7 @@ struct LIBPKG_EXPORT PackageInfo : public ReflectiveRapidJSON::JsonSerializable< std::string pgpSignature; std::string arch; // arch of concrete binary package std::uint32_t size = 0; + bool operator==(const PackageInfo &other) const = default; }; struct LIBPKG_EXPORT InstallInfo : public ReflectiveRapidJSON::JsonSerializable, @@ -241,6 +245,7 @@ struct LIBPKG_EXPORT InstallInfo : public ReflectiveRapidJSON::JsonSerializable< std::vector backupFiles; InstallStatus installStatus = InstallStatus::Unknown; PackageValidation validationMethods = PackageValidation::None; + bool operator==(const InstallInfo &other) const = default; }; /*! @@ -333,22 +338,25 @@ struct Package; * \brief The PackageSpec struct holds a reference to a package. * \remarks If id is non-zero, the package is part of a database using that ID. */ -struct LIBPKG_EXPORT PackageSpec : public ReflectiveRapidJSON::JsonSerializable, - public ReflectiveRapidJSON::BinarySerializable { - explicit PackageSpec(StorageID id = 0, const std::shared_ptr &pkg = nullptr); - bool operator==(const PackageSpec &other) const; +template +struct LIBPKG_EXPORT GenericPackageSpec : public ReflectiveRapidJSON::JsonSerializable>, + public ReflectiveRapidJSON::BinarySerializable> { + explicit GenericPackageSpec(StorageID id = 0, const std::shared_ptr &pkg = nullptr); + bool operator==(const GenericPackageSpec &other) const; StorageID id; - std::shared_ptr pkg; + std::shared_ptr pkg; }; +using PackageSpec = GenericPackageSpec<>; -inline PackageSpec::PackageSpec(StorageID id, const std::shared_ptr &pkg) +template +inline GenericPackageSpec::GenericPackageSpec(StorageID id, const std::shared_ptr &pkg) : id(id) , pkg(pkg) { } -inline bool PackageSpec::operator==(const PackageSpec &other) const +template inline bool GenericPackageSpec::operator==(const GenericPackageSpec &other) const { return id ? id == other.id : pkg == other.pkg; } @@ -357,8 +365,8 @@ inline bool PackageSpec::operator==(const PackageSpec &other) const namespace std { -template <> struct hash { - std::size_t operator()(const LibPkg::PackageSpec &spec) const +template struct hash> { + std::size_t operator()(const LibPkg::GenericPackageSpec &spec) const { using std::hash; return spec.id ? hash()(spec.id) : hash()(spec.pkg); @@ -369,19 +377,38 @@ template <> struct hash { namespace LibPkg { -struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable, public ReflectiveRapidJSON::BinarySerializable { +struct LIBPKG_EXPORT PackageBase : public ReflectiveRapidJSON::JsonSerializable, + public ReflectiveRapidJSON::BinarySerializable { + PackageBase() = default; + PackageBase(const PackageBase &other) = default; + PackageBase(PackageBase &&other) = default; + PackageBase &operator=(PackageBase &&other) = default; + + bool isSame(const PackageBase &other) const; + PackageVersionComparison compareVersion(const PackageBase &other) const; + std::string computeRegularPackageName() const; + void clear(); + + PackageOrigin origin = PackageOrigin::Default; + CppUtilities::DateTime timestamp, buildDate; + std::string name; + std::string version; + std::string arch; + std::vector archs; // set if a split package overrides the base archs; if empty, archs from sourceInfo apply + std::string description; +}; + +struct LIBPKG_EXPORT Package : public PackageBase, + public ReflectiveRapidJSON::JsonSerializable, + public ReflectiveRapidJSON::BinarySerializable { Package() = default; - Package(const Package &other); + Package(const Package &other) = default; Package(Package &&other) = default; - //Package &operator=(const Package &other); Package &operator=(Package &&other) = default; bool providesDependency(const Dependency &dependency) const; static void exportProvides( const std::shared_ptr &package, DependencySet &destinationProvides, std::unordered_set &destinationLibProvides); - bool isSame(const Package &other) const; - PackageVersionComparison compareVersion(const Package &other) const; std::string computeFileName(const char *extension = "pkg.tar.zst") const; - std::string computeRegularPackageName() const; PackageNameData decomposeName() const; void addInfoFromPkgInfoFile(const std::string &info); void addDepsAndProvidesFromContainedDirectory(std::string_view directoryPath); @@ -391,24 +418,26 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable processDllsReferencedByImportLibs(std::set &&dllsReferencedByImportLibs); bool addDepsAndProvidesFromOtherPackage(const Package &otherPackage, bool force = false); bool isArchAny() const; + using ReflectiveRapidJSON::JsonSerializable::fromJson; + using ReflectiveRapidJSON::JsonSerializable::toJson; + using ReflectiveRapidJSON::JsonSerializable::toJsonDocument; + using ReflectiveRapidJSON::BinarySerializable::toBinary; + using ReflectiveRapidJSON::BinarySerializable::restoreFromBinary; + using ReflectiveRapidJSON::BinarySerializable::fromBinary; static bool isPkgInfoFileOrBinary(const char *filePath, const char *fileName, mode_t mode); static bool isLicense(const char *filePath, const char *fileName, mode_t mode); - static std::vector fromInfo(const std::string &info, bool isPackageInfo = false); + static std::vector> fromInfo(const std::string &info, bool isPackageInfo = false); static std::shared_ptr fromDescription(const std::vector &descriptionParts); static void fromDatabaseFile(const std::string &archivePath, const std::function &)> &visitor); static std::shared_ptr fromPkgFile(const std::string &path); static std::tuple fileNameComponents(std::string_view fileName); static std::shared_ptr fromPkgFileName(std::string_view fileName); - static std::vector fromAurRpcJson(const char *jsonData, std::size_t jsonSize, PackageOrigin origin = PackageOrigin::AurRpcInfo); + static std::vector> fromAurRpcJson( + const char *jsonData, std::size_t jsonSize, PackageOrigin origin = PackageOrigin::AurRpcInfo); - PackageOrigin origin = PackageOrigin::Default; - CppUtilities::DateTime timestamp; - std::string name; - std::string version; - std::vector archs; // set if a split package overrides the base archs; if empty, archs from sourceInfo apply - std::string description; + using PackageBase::version; std::string upstreamUrl; std::vector licenses; std::vector groups; @@ -419,32 +448,12 @@ struct LIBPKG_EXPORT Package : public ReflectiveRapidJSON::JsonSerializable replaces; std::set libprovides; std::set libdepends; - std::shared_ptr sourceInfo; - std::unique_ptr packageInfo; - std::unique_ptr installInfo; + std::optional sourceInfo; + std::optional packageInfo; + std::optional installInfo; }; -inline Package::Package(const Package &other) - : origin(other.origin) - , timestamp(other.timestamp) - , name(other.name) - , version(other.version) - , description(other.description) - , upstreamUrl(other.upstreamUrl) - , licenses(other.licenses) - , groups(other.groups) - , dependencies(other.dependencies) - , optionalDependencies(other.optionalDependencies) - , conflicts(other.conflicts) - , provides(other.provides) - , replaces(other.replaces) - , sourceInfo(other.sourceInfo) - , packageInfo() - , installInfo() -{ -} - -inline bool Package::isSame(const Package &other) const +inline bool PackageBase::isSame(const PackageBase &other) const { return name == other.name && version == other.version; } @@ -502,17 +511,6 @@ namespace ReflectiveRapidJSON { REFLECTIVE_RAPIDJSON_TREAT_AS_MULTI_MAP_OR_HASH(LibPkg::DependencySet); -namespace JsonReflector { - -// declare custom (de)serialization for LibPkg::PackageSpec -template <> -LIBPKG_EXPORT void push( - const LibPkg::PackageSpec &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator); -template <> -LIBPKG_EXPORT void pull(LibPkg::PackageSpec &reflectable, - const RAPIDJSON_NAMESPACE::GenericValue> &value, JsonDeserializationErrors *errors); - -} // namespace JsonReflector } // namespace ReflectiveRapidJSON #endif // LIBPKG_DATA_PACKAGE_H diff --git a/libpkg/data/storageprivate.h b/libpkg/data/storageprivate.h index 0b7a0b8..29f5401 100644 --- a/libpkg/data/storageprivate.h +++ b/libpkg/data/storageprivate.h @@ -8,7 +8,7 @@ namespace LibPkg { -using PackageStorage = LMDBSafe::TypedDBI>; +using PackageStorage = LMDBSafe::TypedDBI>; using DependencyStorage = LMDBSafe::TypedDBI>; using LibraryDependencyStorage = LMDBSafe::TypedDBI>; diff --git a/libpkg/parser/package.cpp b/libpkg/parser/package.cpp index b3c8461..5a73132 100644 --- a/libpkg/parser/package.cpp +++ b/libpkg/parser/package.cpp @@ -193,13 +193,13 @@ PackageVersion PackageVersion::fromString(const char *versionString, size_t vers #define valueString std::string_view(value, valueSize) #define ensure_pkg_info \ if (!package.packageInfo) \ - package.packageInfo = make_unique() + package.packageInfo = make_optional() #define ensure_install_info \ if (!package.installInfo) \ - package.installInfo = make_unique() + package.installInfo = make_optional() -void addPackageInfo( - Package &package, PackageVersion &version, const char *field, size_t fieldSize, const char *value, size_t valueSize, bool isPackageInfo) +static void addPackageInfo(Package &package, PackageVersion &version, const char *field, size_t fieldSize, const char *value, size_t valueSize, + bool isPackageInfo, std::size_t packageCount) { if_field("pkgbase") { @@ -235,7 +235,7 @@ void addPackageInfo( // add as binary arch when parsing PKGINFO ensure_pkg_info; package.packageInfo->arch = valueString; - } else if (package.sourceInfo.use_count() <= 1) { + } else if (!packageCount) { // add to sourceInfo when still parsing base info package.sourceInfo->archs.emplace_back(value, valueSize); } else { @@ -451,11 +451,12 @@ static void addVersionInfo(Package &package, PackageVersion &version, bool isPac static void parsePkgInfo(const std::string &info, const std::function &nextPackage, bool isPackageInfo) { // define variables to store intermediate results while still parsing package base - PackageVersion version; - Package basePackage; + auto version = PackageVersion(); + auto packageCount = std::size_t(); + auto basePackage = Package(); basePackage.origin = isPackageInfo ? PackageOrigin::PackageInfo : PackageOrigin::SourceInfo; - basePackage.sourceInfo = make_shared(); - std::string &packageBase = basePackage.sourceInfo->name; + basePackage.sourceInfo = std::make_optional(); + auto &packageBase = basePackage.sourceInfo->name; // states enum { @@ -560,17 +561,18 @@ static void parsePkgInfo(const std::string &info, const std::function add field to ... try { if (currentPackage) { // ... concrete package info if there's already a concrete package addPackageInfo(*currentPackage, version, currentFieldName, currentFieldNameSize, currentFieldValue, currentFieldValueSize, - isPackageInfo); + isPackageInfo, packageCount); } else { // ... base info if still parsing general info - addPackageInfo( - basePackage, version, currentFieldName, currentFieldNameSize, currentFieldValue, currentFieldValueSize, isPackageInfo); + addPackageInfo(basePackage, version, currentFieldName, currentFieldNameSize, currentFieldValue, currentFieldValueSize, + isPackageInfo, packageCount); } } catch (const ConversionException &) { // FIXME: error handling @@ -624,8 +626,8 @@ std::shared_ptr Package::fromDescription(const std::vector { auto package = std::make_shared(); package->origin = PackageOrigin::Database; - package->sourceInfo = make_shared(); - package->packageInfo = make_unique(); + package->sourceInfo = std::make_optional(); + package->packageInfo = std::make_optional(); for (const auto &desc : descriptionParts) { // states enum { @@ -890,7 +892,7 @@ std::shared_ptr Package::fromPkgFile(const string &path) throw runtime_error("Package " % path + " does not contain a valid .PKGINFO"); } if (!package->packageInfo) { - package->packageInfo = make_unique(); + package->packageInfo = std::make_optional(); } package->packageInfo->fileName = fileName(path); package->addDepsAndProvidesFromOtherPackage(tmpPackageForLibraryDeps, true); @@ -933,7 +935,7 @@ std::shared_ptr Package::fromPkgFileName(std::string_view fileName) pkg->name = name; pkg->version = version; pkg->provides.emplace_back(pkg->name, pkg->version); - pkg->packageInfo = make_unique(); + pkg->packageInfo = std::make_optional(); pkg->packageInfo->fileName = fileName; pkg->packageInfo->arch = arch; return pkg; @@ -948,8 +950,9 @@ std::vector Package::fromAurRpcJson(const char *jsonData, std::size packages.reserve(rpcMultiInfo.results.size()); for (auto &result : rpcMultiInfo.results) { - auto package = std::make_shared(); - auto sourceInfo = std::make_shared(); + auto &spec = packages.emplace_back(); + auto *package = &*(spec.pkg = std::make_shared()); + auto &sourceInfo = package->sourceInfo = std::make_optional(); package->origin = origin; package->name = std::move(result.Name); package->version = std::move(result.Version); @@ -979,8 +982,6 @@ std::vector Package::fromAurRpcJson(const char *jsonData, std::size sourceInfo->firstSubmitted = DateTime::fromTimeStampGmt(result.FirstSubmitted); sourceInfo->lastModified = DateTime::fromTimeStampGmt(result.LastModified); sourceInfo->url = std::move(result.URLPath); - package->sourceInfo = std::move(sourceInfo); - packages.emplace_back(0, std::move(package)); } return packages; } diff --git a/libpkg/tests/data.cpp b/libpkg/tests/data.cpp index 6d0df0c..19930ae 100644 --- a/libpkg/tests/data.cpp +++ b/libpkg/tests/data.cpp @@ -274,7 +274,7 @@ void DataTests::testComputingFileName() pkg.name = "test"; pkg.version = "1.2-3"; CPPUNIT_ASSERT_EQUAL_MESSAGE("packageInfo required for computing filename", string(), pkg.computeFileName()); - pkg.packageInfo = make_unique(); + pkg.packageInfo = std::make_optional(); pkg.packageInfo->arch = "x86_64"; CPPUNIT_ASSERT_EQUAL_MESSAGE("file name computed from name, version and arch", "test-1.2-3-x86_64.pkg.tar.zst"s, pkg.computeFileName()); pkg.packageInfo->fileName = "explicitly-specified-filename"; @@ -349,8 +349,8 @@ void DataTests::testComputingBuildOrder() CPPUNIT_ASSERT_EQUAL(0_st, res.ignored.size()); // ignore cycle if not interested in that particular package - m_pkg2->packageInfo = std::make_unique(); - tar->packageInfo = std::make_unique(); + m_pkg2->packageInfo = std::make_optional(); + tar->packageInfo = std::make_optional(); tar->dependencies.clear(); tar->dependencies.emplace_back("bar"); db.forceUpdatePackage(tar); diff --git a/libpkg/tests/parser.cpp b/libpkg/tests/parser.cpp index 2c1b9b0..5ae54ef 100644 --- a/libpkg/tests/parser.cpp +++ b/libpkg/tests/parser.cpp @@ -4,6 +4,15 @@ #include "../parser/config.h" #include "../parser/package.h" +namespace CppUtilities { +inline std::ostream &operator<<(std::ostream &out, const LibPkg::SourceInfo &sourceInfo) +{ + const auto buff = sourceInfo.toJson(); + out.write(buff.GetString(), ReflectiveRapidJSON::JsonReflector::rapidJsonSize(buff.GetSize())); + return out; +} +} // namespace CppUtilities + #include #include #include @@ -194,8 +203,8 @@ void ParserTests::testParsingPlainSrcInfo() CPPUNIT_ASSERT_MESSAGE("no regular dependencies"s, pkg1.dependencies.empty()); CPPUNIT_ASSERT_EQUAL_MESSAGE("optional dependencies"s, vector{ Dependency("c++utilities-doc"s, string(), DependencyMode::Any, "API documentation"s) }, pkg1.optionalDependencies); - CPPUNIT_ASSERT_MESSAGE("source info present", pkg1.sourceInfo != nullptr); - CPPUNIT_ASSERT_MESSAGE("no package info present", pkg1.packageInfo == nullptr); + CPPUNIT_ASSERT_MESSAGE("source info present", pkg1.sourceInfo.has_value()); + CPPUNIT_ASSERT_MESSAGE("no package info present", !pkg1.packageInfo.has_value()); CPPUNIT_ASSERT_EQUAL_MESSAGE( "make dependencies"s, vector{ Dependency("cmake"s, "3.0"s, DependencyMode::GreatherEqual) }, pkg1.sourceInfo->makeDependencies); CPPUNIT_ASSERT_EQUAL_MESSAGE("check dependencies"s, vector{ Dependency("cppunit"s) }, pkg1.sourceInfo->checkDependencies); @@ -225,10 +234,10 @@ void ParserTests::testParsingSplitPackageSrcInfo() Dependency("mingw-w64-icu"s), }; CPPUNIT_ASSERT_EQUAL_MESSAGE("dependencies (2)"s, dependencies2, pkg2.dependencies); - CPPUNIT_ASSERT_MESSAGE("source info present (1)", pkg1.sourceInfo != nullptr); - CPPUNIT_ASSERT_MESSAGE("source info present (2)", pkg2.sourceInfo != nullptr); - CPPUNIT_ASSERT_MESSAGE("no package info present (1)", pkg1.packageInfo == nullptr); - CPPUNIT_ASSERT_MESSAGE("no package info present (2)", pkg2.packageInfo == nullptr); + CPPUNIT_ASSERT_MESSAGE("source info present (1)", pkg1.sourceInfo.has_value()); + CPPUNIT_ASSERT_MESSAGE("source info present (2)", pkg2.sourceInfo.has_value()); + CPPUNIT_ASSERT_MESSAGE("no package info present (1)", !pkg1.packageInfo.has_value()); + CPPUNIT_ASSERT_MESSAGE("no package info present (2)", !pkg2.packageInfo.has_value()); CPPUNIT_ASSERT_EQUAL_MESSAGE("pkgbase (1)"s, "mingw-w64-harfbuzz"s, pkg1.sourceInfo->name); CPPUNIT_ASSERT_EQUAL_MESSAGE("pkgbase (2)"s, "mingw-w64-harfbuzz"s, pkg2.sourceInfo->name); const vector archs = { "any"s }; @@ -243,8 +252,9 @@ void ParserTests::testParsingSplitPackageSrcInfoWithDifferentArchs() CPPUNIT_ASSERT_EQUAL_MESSAGE("3 (split) packages present"s, 3ul, packages.size()); const auto &jre = packages[0].pkg, &jdk = packages[1].pkg, &doc = packages[2].pkg; - CPPUNIT_ASSERT_MESSAGE("source info present", jdk->sourceInfo); - CPPUNIT_ASSERT_EQUAL_MESSAGE("jre has same source info as base", jdk->sourceInfo, jre->sourceInfo); + CPPUNIT_ASSERT_MESSAGE("source info present (jdk)", jdk->sourceInfo.has_value()); + CPPUNIT_ASSERT_MESSAGE("source info present (jre)", jre->sourceInfo.has_value()); + CPPUNIT_ASSERT_EQUAL_MESSAGE("jre has same source info as base", jdk->sourceInfo->archs, jre->sourceInfo->archs); CPPUNIT_ASSERT_EQUAL_MESSAGE("jdk-doc has same source info as base", jdk->sourceInfo, doc->sourceInfo); CPPUNIT_ASSERT_EQUAL_MESSAGE("jre name", "jre"s, jre->name); CPPUNIT_ASSERT_EQUAL_MESSAGE("jdk name", "jdk"s, jdk->name); diff --git a/librepomgr/buildactions/buildaction.h b/librepomgr/buildactions/buildaction.h index 3d2e5bd..5b5d3f0 100644 --- a/librepomgr/buildactions/buildaction.h +++ b/librepomgr/buildactions/buildaction.h @@ -55,7 +55,7 @@ struct LIBREPOMGR_EXPORT PackageBuildData : public ReflectiveRapidJSON::JsonSeri std::vector> existingPackages; std::string sourceDirectory; std::string originalSourceDirectory; - std::shared_ptr sourceInfo; + std::optional sourceInfo; std::vector packages; std::vector warnings; std::string error; @@ -154,7 +154,14 @@ struct LIBREPOMGR_EXPORT BuildActionMessages : public ReflectiveRapidJSON::JsonS class BuildProcessSession; struct ServiceSetup; -struct LIBREPOMGR_EXPORT BuildAction : public std::enable_shared_from_this, +struct LIBREPOMGR_EXPORT BuildActionBase : public ReflectiveRapidJSON::JsonSerializable, + public ReflectiveRapidJSON::BinarySerializable { + using IdType = BuildActionIdType; + static constexpr IdType invalidId = std::numeric_limits::max(); +}; + +struct LIBREPOMGR_EXPORT BuildAction : public BuildActionBase, + public std::enable_shared_from_this, public ReflectiveRapidJSON::JsonSerializable, public ReflectiveRapidJSON::BinarySerializable { friend InternalBuildAction; @@ -168,9 +175,6 @@ struct LIBREPOMGR_EXPORT BuildAction : public std::enable_shared_from_this::max(); - explicit BuildAction(IdType id = invalidId, ServiceSetup *setup = nullptr) noexcept; BuildAction &operator=(BuildAction &&other); ~BuildAction(); @@ -198,6 +202,12 @@ public: void streamFile(const WebAPI::Params ¶ms, const std::string &filePath, boost::beast::string_view fileMimeType, boost::beast::string_view contentDisposition = boost::beast::string_view()); ServiceSetup *setup(); + using ReflectiveRapidJSON::JsonSerializable::fromJson; + using ReflectiveRapidJSON::JsonSerializable::toJson; + using ReflectiveRapidJSON::JsonSerializable::toJsonDocument; + using ReflectiveRapidJSON::BinarySerializable::toBinary; + using ReflectiveRapidJSON::BinarySerializable::restoreFromBinary; + using ReflectiveRapidJSON::BinarySerializable::fromBinary; protected: private: @@ -207,28 +217,26 @@ private: public: IdType id; + BuildActionType type = BuildActionType::Invalid; std::string taskName; std::string templateName; - std::string directory; - std::vector packageNames; - std::vector sourceDbs, destinationDbs; - std::unordered_map settings; - BuildActionFlagType flags = noBuildActionFlags; - BuildActionType type = BuildActionType::Invalid; - - // only the following member variables are supposed to change after the build action has been added - // to the overall list of build actions BuildActionStatus status = BuildActionStatus::Created; BuildActionResult result = BuildActionResult::None; - std::variant, LibPkg::LicenseResult, LibPkg::PackageUpdates, BuildPreparation, BuildProgress, - PackageMovementResult, std::unordered_map>, BuildActionMessages> - resultData; - std::vector logfiles; - std::vector artefacts; CppUtilities::DateTime created = CppUtilities::DateTime::gmtNow(); CppUtilities::DateTime started; CppUtilities::DateTime finished; std::vector startAfter; + std::string directory; + std::vector sourceDbs, destinationDbs; + std::vector packageNames; + BuildActionFlagType flags = noBuildActionFlags; + std::unordered_map settings; + + std::vector logfiles; + std::vector artefacts; + std::variant, LibPkg::LicenseResult, LibPkg::PackageUpdates, BuildPreparation, BuildProgress, + PackageMovementResult, std::unordered_map>, BuildActionMessages> + resultData; private: LogContext m_log; diff --git a/librepomgr/buildactions/preparebuild.cpp b/librepomgr/buildactions/preparebuild.cpp index 1441781..110ae5d 100644 --- a/librepomgr/buildactions/preparebuild.cpp +++ b/librepomgr/buildactions/preparebuild.cpp @@ -350,7 +350,7 @@ void PrepareBuild::addResultFromSrcInfo(WebClient::AurSnapshotQuerySession &mult auto snapshotResult = WebClient::AurSnapshotResult{ .packageName = packageName, .packages = LibPkg::Package::fromInfo(srcInfo, false) }; if (snapshotResult.packages.empty() || snapshotResult.packages.front().pkg->name.empty()) { snapshotResult.error = "Unable to parse .SRCINFO: no package name present"; - } else if (!(snapshotResult.sourceInfo = snapshotResult.packages.front().pkg->sourceInfo)) { + } else if (!snapshotResult.packages.front().pkg->sourceInfo.has_value()) { snapshotResult.error = "Unable to parse .SRCINFO: no source info present"; } multiSession.addResponse(std::move(snapshotResult)); @@ -822,7 +822,9 @@ void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::Conta continue; } auto &buildData = m_buildDataByPackage[response.packageName]; - buildData.sourceInfo = move(response.sourceInfo); + if (!response.packages.empty() && response.packages.front().pkg) { + buildData.sourceInfo = response.packages.front().pkg->sourceInfo; + } buildData.packages = move(response.packages); buildData.error = move(response.error); buildData.hasSource = buildData.sourceInfo && !buildData.packages.empty() && buildData.error.empty(); diff --git a/librepomgr/buildactions/reloadlibrarydependencies.cpp b/librepomgr/buildactions/reloadlibrarydependencies.cpp index bb9f925..925a662 100644 --- a/librepomgr/buildactions/reloadlibrarydependencies.cpp +++ b/librepomgr/buildactions/reloadlibrarydependencies.cpp @@ -228,10 +228,10 @@ bool ReloadLibraryDependencies::addRelevantPackage(LibPkg::StorageID packageID, relevantPkg.info.name = package->name; // -> assign certain fields which are used by addDepsAndProvidesFromOtherPackage() to check whether the packages are matching relevantPkg.info.version = package->version; - relevantPkg.info.packageInfo = std::make_unique(); + relevantPkg.info.packageInfo = std::make_optional(); relevantPkg.info.packageInfo->buildDate = package->packageInfo->buildDate; // -> gather source info such as make and check dependencies as well - relevantPkg.info.sourceInfo = std::make_shared(); + relevantPkg.info.sourceInfo = std::make_optional(); ++m_remainingPackages; return false; } diff --git a/librepomgr/buildactions/updatecheck.cpp b/librepomgr/buildactions/updatecheck.cpp index bfd4f14..de0699d 100644 --- a/librepomgr/buildactions/updatecheck.cpp +++ b/librepomgr/buildactions/updatecheck.cpp @@ -27,7 +27,8 @@ void UpdateCheck::run() if (m_fromAur && !m_packageLookupDone && WebClient::queryAurPackagesForDatabase(m_buildAction->log(), m_setup, m_setup.building.ioContext, - &std::get>(configReadLock), **m_destinationDbs.begin(), [this](std::vector &&) { + &std::get>(configReadLock), **m_destinationDbs.begin(), + [this](std::vector &&) { m_packageLookupDone = true; run(); })) { diff --git a/librepomgr/serversetup.h b/librepomgr/serversetup.h index 0972ff2..635da3d 100644 --- a/librepomgr/serversetup.h +++ b/librepomgr/serversetup.h @@ -44,7 +44,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { std::string pacmanConfigFilePath = "/etc/pacman.conf"; std::filesystem::path initialWorkingDirectory; std::string workingDirectory = "workingdir"; - std::string dbPath = "libpkg.db"; + std::string dbPath = "libpkg-1.db"; std::uint32_t maxDbs = 512; std::size_t packageCacheLimit = 1000; @@ -130,7 +130,7 @@ struct LIBREPOMGR_EXPORT ServiceSetup : public LibPkg::Lockable { // never changed after startup unsigned short threadCount = 4; boost::asio::io_context ioContext; - std::string dbPath = "librepomgr.db"; + std::string dbPath = "librepomgr-1.db"; void initStorage(const char *path); bool hasStorage() const; diff --git a/librepomgr/webapi/routes.cpp b/librepomgr/webapi/routes.cpp index 240b495..6b3ad5d 100644 --- a/librepomgr/webapi/routes.cpp +++ b/librepomgr/webapi/routes.cpp @@ -227,15 +227,18 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) const auto dbIterator = dbs.find(db.name); return dbIterator == dbs.end() || dbIterator->second.find(db.arch) == dbIterator->second.end(); }); - const auto pushPackage = details - ? LibPkg::Config::PackageVisitorConst([&array, &document, &limit](Database &, LibPkg::StorageID, const std::shared_ptr &pkg) { - ReflectiveRapidJSON::JsonReflector::push(pkg, array, document.GetAllocator()); - return array.Size() >= limit; - }) - : ([&array, &document, &limit](Database &db, LibPkg::StorageID id, const std::shared_ptr &pkg) { - ReflectiveRapidJSON::JsonReflector::push(LibPkg::PackageSearchResult(db, pkg, id), array, document.GetAllocator()); + const auto pushPackage = LibPkg::Config::PackageVisitorBase([&array, &document, &limit](Database &db, LibPkg::StorageID id, const std::shared_ptr &pkg) { + ReflectiveRapidJSON::JsonReflector::push(LibPkg::PackageBaseSearchResult(db, *pkg, id), array, document.GetAllocator()); return array.Size() >= limit; }); + const auto pushBasePackage = [&array, &document, &limit](Database &db, LibPkg::StorageID id, const PackageBase &pkg) { + ReflectiveRapidJSON::JsonReflector::push(LibPkg::PackageBaseSearchResult(db, pkg, id), array, document.GetAllocator()); + return array.Size() >= limit; + }; + const auto pushPackageDetails = !details ? LibPkg::Config::PackageVisitorConst() : [&array, &document, &limit](Database &, LibPkg::StorageID, const std::shared_ptr &pkg) { + ReflectiveRapidJSON::JsonReflector::push(pkg, array, document.GetAllocator()); + return array.Size() >= limit; + }; auto aurPackages = std::vector(); auto neededAurPackages = std::vector(); @@ -261,21 +264,28 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) --limit; } if (!isDbAur && (!dbs.empty() || !onlyFromAur)) { - params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushPackage); + if (details) { + params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushPackageDetails); + } else { + params.setup.config.packages(dbName, dbArch, packageNameStr, visitDb, pushPackage); + } } break; } - case Mode::NameContains: + case Mode::NameContains: { + auto basePackage = PackageBase(); params.setup.config.packagesByName( - visitDb, [&](LibPkg::Database &db, std::string_view packageName, const std::function &getPackage) { + visitDb, [&](LibPkg::Database &db, std::string_view packageName, const std::function &getPackage) { if (packageName.find(name) != std::string_view::npos) { - const auto [packageID, package] = getPackage(); - if (!package) { + const auto packageID = getPackage(basePackage); + if (!packageID) { cerr << Phrases::ErrorMessage << "Broken index in db \"" << db.name << "\": package \"" << packageName << "\" does not exist" << std::endl; return false; } - return pushPackage(db, packageID, package); + const auto stopSearch = pushBasePackage(db, packageID, basePackage); + basePackage.clear(); + return stopSearch; } return false; }); @@ -283,19 +293,23 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) neededAurPackages.emplace_back(std::move(name)); } break; - case Mode::Regex: + } + case Mode::Regex: { try { const auto regex = std::regex(name.data(), name.size()); + auto basePackage = PackageBase(); params.setup.config.packagesByName( - visitDb, [&](LibPkg::Database &db, std::string_view packageName, const std::function &getPackage) { + visitDb, [&](LibPkg::Database &db, std::string_view packageName, const std::function &getPackage) { if (std::regex_match(packageName.begin(), packageName.end(), regex)) { - const auto [packageID, package] = getPackage(); - if (!package) { + const auto packageID = getPackage(basePackage); + if (!packageID) { cerr << Phrases::ErrorMessage << "Broken index in db \"" << db.name << "\": package \"" << packageName << "\" does not exist" << std::endl; return false; } - return pushPackage(db, packageID, package); + const auto stopSearch = pushBasePackage(db, packageID, basePackage); + basePackage.clear(); + return stopSearch; } return false; }); @@ -303,6 +317,7 @@ void getPackages(const Params ¶ms, ResponseHandler &&handler) throw BadRequest(argsToString("regex is invalid: ", e.what())); } break; + } case Mode::Provides: case Mode::Depends: params.setup.config.providingPackages(Dependency::fromString(name), mode == Mode::Depends, visitDb, pushPackage); diff --git a/librepomgr/webclient/aur.cpp b/librepomgr/webclient/aur.cpp index b6e18b2..1c1282e 100644 --- a/librepomgr/webclient/aur.cpp +++ b/librepomgr/webclient/aur.cpp @@ -282,7 +282,7 @@ void queryAurSnapshots(LogContext &log, ServiceSetup &setup, const std::vectorname.empty()) { result.error = "Unable to parse .SRCINFO: no package name present"; - } else if (!(result.sourceInfo = result.packages.front().pkg->sourceInfo)) { + } else if (!result.packages.front().pkg->sourceInfo.has_value()) { result.error = "Unable to parse .SRCINFO: no source info present"; } multiSession->addResponse(move(result)); diff --git a/librepomgr/webclient/aur.h b/librepomgr/webclient/aur.h index b3ef41a..aeebe21 100644 --- a/librepomgr/webclient/aur.h +++ b/librepomgr/webclient/aur.h @@ -19,7 +19,6 @@ namespace WebClient { struct AurSnapshotResult { std::string packageName; std::string errorOutput; - std::shared_ptr sourceInfo; std::vector packages; std::string error; };