Avoid locking whole config when updating DBs
* Only lock the config for writing the reloading the config file * Make sure all write operations to the database acquire an "update mutex" to ensure only one write operation happens at a time * Do *not* acquire any additional locks when reading from a database as it should be safe to do so even when a write operation happens because * LMDB read and write transactions can happen at the same time * The package cache has its own mutex anyways * Write ops to the package cache try to lock the "update mutex" to prevent writing "old" data to the cache during updates * Make "lastUpdate" atomic to avoid locking the config when accessing it
This commit is contained in:
parent
afc61bcad6
commit
7ade757c8d
|
@ -25,12 +25,14 @@ struct PackageUpdaterPrivate {
|
||||||
using AffectedDeps = std::unordered_multimap<std::string, AffectedPackagesWithDependencyDetail>;
|
using AffectedDeps = std::unordered_multimap<std::string, AffectedPackagesWithDependencyDetail>;
|
||||||
using AffectedLibs = std::unordered_map<std::string, AffectedPackages>;
|
using AffectedLibs = std::unordered_map<std::string, AffectedPackages>;
|
||||||
|
|
||||||
explicit PackageUpdaterPrivate(DatabaseStorage &storage);
|
explicit PackageUpdaterPrivate(DatabaseStorage &storage, bool clear);
|
||||||
void update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package);
|
void update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package);
|
||||||
void update(const StorageID packageID, bool removed, const std::shared_ptr<Package> &package);
|
void update(const StorageID packageID, bool removed, const std::shared_ptr<Package> &package);
|
||||||
void submit(const std::string &dependencyName, AffectedDeps::mapped_type &affected, DependencyStorage::RWTransaction &txn);
|
void submit(const std::string &dependencyName, AffectedDeps::mapped_type &affected, DependencyStorage::RWTransaction &txn);
|
||||||
void submit(const std::string &libraryName, AffectedLibs::mapped_type &affected, LibraryDependencyStorage::RWTransaction &txn);
|
void submit(const std::string &libraryName, AffectedLibs::mapped_type &affected, LibraryDependencyStorage::RWTransaction &txn);
|
||||||
|
|
||||||
|
bool clear = false;
|
||||||
|
std::unique_lock<std::mutex> lock;
|
||||||
PackageStorage::RWTransaction packagesTxn;
|
PackageStorage::RWTransaction packagesTxn;
|
||||||
AffectedDeps affectedProvidedDeps;
|
AffectedDeps affectedProvidedDeps;
|
||||||
AffectedDeps affectedRequiredDeps;
|
AffectedDeps affectedRequiredDeps;
|
||||||
|
@ -95,10 +97,12 @@ void Database::resetConfiguration()
|
||||||
|
|
||||||
void Database::clearPackages()
|
void Database::clearPackages()
|
||||||
{
|
{
|
||||||
lastUpdate = DateTime();
|
if (!m_storage) {
|
||||||
if (m_storage) {
|
return;
|
||||||
m_storage->packageCache.clear(*m_storage);
|
|
||||||
}
|
}
|
||||||
|
const auto lock = std::unique_lock(m_storage->updateMutex);
|
||||||
|
m_storage->packageCache.clear(*m_storage);
|
||||||
|
lastUpdate = DateTime();
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<std::shared_ptr<Package>> Database::findPackages(const std::function<bool(const Database &, const Package &)> &pred)
|
std::vector<std::shared_ptr<Package>> Database::findPackages(const std::function<bool(const Database &, const Package &)> &pred)
|
||||||
|
@ -340,6 +344,7 @@ PackageSpec Database::findPackageWithID(const std::string &packageName)
|
||||||
|
|
||||||
void Database::removePackage(const std::string &packageName)
|
void Database::removePackage(const std::string &packageName)
|
||||||
{
|
{
|
||||||
|
const auto lock = std::unique_lock(m_storage->updateMutex);
|
||||||
const auto [packageID, package] = m_storage->packageCache.retrieve(*m_storage, packageName);
|
const auto [packageID, package] = m_storage->packageCache.retrieve(*m_storage, packageName);
|
||||||
if (package) {
|
if (package) {
|
||||||
removePackageDependencies(packageID, package);
|
removePackageDependencies(packageID, package);
|
||||||
|
@ -349,6 +354,7 @@ void Database::removePackage(const std::string &packageName)
|
||||||
|
|
||||||
StorageID Database::updatePackage(const std::shared_ptr<Package> &package)
|
StorageID Database::updatePackage(const std::shared_ptr<Package> &package)
|
||||||
{
|
{
|
||||||
|
const auto lock = std::unique_lock(m_storage->updateMutex);
|
||||||
const auto res = m_storage->packageCache.store(*m_storage, package, false);
|
const auto res = m_storage->packageCache.store(*m_storage, package, false);
|
||||||
if (!res.updated) {
|
if (!res.updated) {
|
||||||
return res.id;
|
return res.id;
|
||||||
|
@ -362,6 +368,7 @@ StorageID Database::updatePackage(const std::shared_ptr<Package> &package)
|
||||||
|
|
||||||
StorageID Database::forceUpdatePackage(const std::shared_ptr<Package> &package)
|
StorageID Database::forceUpdatePackage(const std::shared_ptr<Package> &package)
|
||||||
{
|
{
|
||||||
|
const auto lock = std::unique_lock(m_storage->updateMutex);
|
||||||
const auto res = m_storage->packageCache.store(*m_storage, package, true);
|
const auto res = m_storage->packageCache.store(*m_storage, package, true);
|
||||||
if (res.oldEntry) {
|
if (res.oldEntry) {
|
||||||
removePackageDependencies(res.id, res.oldEntry);
|
removePackageDependencies(res.id, res.oldEntry);
|
||||||
|
@ -377,8 +384,7 @@ void Database::replacePackages(const std::vector<std::shared_ptr<Package>> &newP
|
||||||
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
|
package->addDepsAndProvidesFromOtherPackage(*existingPackage);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
clearPackages();
|
auto updater = PackageUpdater(*this, true);
|
||||||
auto updater = PackageUpdater(*this);
|
|
||||||
for (const auto &package : newPackages) {
|
for (const auto &package : newPackages) {
|
||||||
updater.update(package);
|
updater.update(package);
|
||||||
}
|
}
|
||||||
|
@ -623,15 +629,21 @@ std::string Database::filesPathFromRegularPath() const
|
||||||
return ext == std::string::npos ? path : argsToString(std::string_view(path.data(), ext), ".files");
|
return ext == std::string::npos ? path : argsToString(std::string_view(path.data(), ext), ".files");
|
||||||
}
|
}
|
||||||
|
|
||||||
PackageUpdaterPrivate::PackageUpdaterPrivate(DatabaseStorage &storage)
|
PackageUpdaterPrivate::PackageUpdaterPrivate(DatabaseStorage &storage, bool clear)
|
||||||
: packagesTxn(storage.packages.getRWTransaction())
|
: clear(clear)
|
||||||
|
, lock(storage.updateMutex)
|
||||||
|
, packagesTxn(storage.packages.getRWTransaction())
|
||||||
{
|
{
|
||||||
|
if (clear) {
|
||||||
|
storage.packageCache.clearCacheOnly(storage);
|
||||||
|
packagesTxn.clear();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package)
|
void PackageUpdaterPrivate::update(const PackageCache::StoreResult &res, const std::shared_ptr<Package> &package)
|
||||||
{
|
{
|
||||||
update(res.id, false, package);
|
update(res.id, false, package);
|
||||||
if (res.oldEntry) {
|
if (!clear && res.oldEntry) {
|
||||||
update(res.id, true, res.oldEntry);
|
update(res.id, true, res.oldEntry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -731,9 +743,9 @@ void PackageUpdaterPrivate::addLibrary(StorageID packageID, const std::string &l
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
PackageUpdater::PackageUpdater(Database &database)
|
PackageUpdater::PackageUpdater(Database &database, bool clear)
|
||||||
: m_database(database)
|
: m_database(database)
|
||||||
, m_d(std::make_unique<PackageUpdaterPrivate>(*m_database.m_storage))
|
, m_d(std::make_unique<PackageUpdaterPrivate>(*m_database.m_storage, clear))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -748,42 +760,57 @@ LibPkg::PackageSpec LibPkg::PackageUpdater::findPackageWithID(const std::string
|
||||||
|
|
||||||
StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
|
StorageID PackageUpdater::update(const std::shared_ptr<Package> &package)
|
||||||
{
|
{
|
||||||
const auto res = m_database.m_storage->packageCache.store(*m_database.m_storage, m_d->packagesTxn, package);
|
const auto &storage = m_database.m_storage;
|
||||||
|
const auto res = storage->packageCache.store(*m_database.m_storage, m_d->packagesTxn, package);
|
||||||
m_d->update(res, package);
|
m_d->update(res, package);
|
||||||
return res.id;
|
return res.id;
|
||||||
}
|
}
|
||||||
|
|
||||||
void PackageUpdater::commit()
|
void PackageUpdater::commit()
|
||||||
{
|
{
|
||||||
|
const auto &storage = m_database.m_storage;
|
||||||
m_d->packagesTxn.commit();
|
m_d->packagesTxn.commit();
|
||||||
{
|
{
|
||||||
auto txn = m_database.m_storage->providedDeps.getRWTransaction();
|
auto txn = storage->providedDeps.getRWTransaction();
|
||||||
|
if (m_d->clear) {
|
||||||
|
txn.clear();
|
||||||
|
}
|
||||||
for (auto &[dependencyName, affected] : m_d->affectedProvidedDeps) {
|
for (auto &[dependencyName, affected] : m_d->affectedProvidedDeps) {
|
||||||
m_d->submit(dependencyName, affected, txn);
|
m_d->submit(dependencyName, affected, txn);
|
||||||
}
|
}
|
||||||
txn.commit();
|
txn.commit();
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
auto txn = m_database.m_storage->requiredDeps.getRWTransaction();
|
auto txn = storage->requiredDeps.getRWTransaction();
|
||||||
|
if (m_d->clear) {
|
||||||
|
txn.clear();
|
||||||
|
}
|
||||||
for (auto &[dependencyName, affected] : m_d->affectedRequiredDeps) {
|
for (auto &[dependencyName, affected] : m_d->affectedRequiredDeps) {
|
||||||
m_d->submit(dependencyName, affected, txn);
|
m_d->submit(dependencyName, affected, txn);
|
||||||
}
|
}
|
||||||
txn.commit();
|
txn.commit();
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
auto txn = m_database.m_storage->providedLibs.getRWTransaction();
|
auto txn = storage->providedLibs.getRWTransaction();
|
||||||
|
if (m_d->clear) {
|
||||||
|
txn.clear();
|
||||||
|
}
|
||||||
for (auto &[libraryName, affected] : m_d->affectedProvidedLibs) {
|
for (auto &[libraryName, affected] : m_d->affectedProvidedLibs) {
|
||||||
m_d->submit(libraryName, affected, txn);
|
m_d->submit(libraryName, affected, txn);
|
||||||
}
|
}
|
||||||
txn.commit();
|
txn.commit();
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
auto txn = m_database.m_storage->requiredLibs.getRWTransaction();
|
auto txn = storage->requiredLibs.getRWTransaction();
|
||||||
|
if (m_d->clear) {
|
||||||
|
txn.clear();
|
||||||
|
}
|
||||||
for (auto &[libraryName, affected] : m_d->affectedRequiredLibs) {
|
for (auto &[libraryName, affected] : m_d->affectedRequiredLibs) {
|
||||||
m_d->submit(libraryName, affected, txn);
|
m_d->submit(libraryName, affected, txn);
|
||||||
}
|
}
|
||||||
txn.commit();
|
txn.commit();
|
||||||
}
|
}
|
||||||
|
m_d->lock.unlock();
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace LibPkg
|
} // namespace LibPkg
|
||||||
|
@ -793,7 +820,7 @@ namespace ReflectiveRapidJSON {
|
||||||
namespace JsonReflector {
|
namespace JsonReflector {
|
||||||
|
|
||||||
template <>
|
template <>
|
||||||
LIBPKG_EXPORT void push<LibPkg::PackageSearchResult>(
|
void push<LibPkg::PackageSearchResult>(
|
||||||
const LibPkg::PackageSearchResult &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)
|
const LibPkg::PackageSearchResult &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)
|
||||||
{
|
{
|
||||||
// customize serialization of PackageSearchResult to render as if it was pkg itself with an additional db property
|
// customize serialization of PackageSearchResult to render as if it was pkg itself with an additional db property
|
||||||
|
@ -830,7 +857,7 @@ LIBPKG_EXPORT void push<LibPkg::PackageSearchResult>(
|
||||||
}
|
}
|
||||||
|
|
||||||
template <>
|
template <>
|
||||||
LIBPKG_EXPORT void pull<LibPkg::PackageSearchResult>(LibPkg::PackageSearchResult &reflectable,
|
void pull<LibPkg::PackageSearchResult>(LibPkg::PackageSearchResult &reflectable,
|
||||||
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors)
|
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors)
|
||||||
{
|
{
|
||||||
if (!value.IsObject()) {
|
if (!value.IsObject()) {
|
||||||
|
@ -862,12 +889,28 @@ LIBPKG_EXPORT void pull<LibPkg::PackageSearchResult>(LibPkg::PackageSearchResult
|
||||||
ReflectiveRapidJSON::JsonReflector::pull(dbInfo.arch, "dbArch", obj, errors);
|
ReflectiveRapidJSON::JsonReflector::pull(dbInfo.arch, "dbArch", obj, errors);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <>
|
||||||
|
void push<LibPkg::AtomicDateTime>(
|
||||||
|
const LibPkg::AtomicDateTime &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator)
|
||||||
|
{
|
||||||
|
push<CppUtilities::DateTime>(reflectable.load(), value, allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <>
|
||||||
|
void pull<LibPkg::AtomicDateTime>(LibPkg::AtomicDateTime &reflectable,
|
||||||
|
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors)
|
||||||
|
{
|
||||||
|
auto d = CppUtilities::DateTime();
|
||||||
|
pull<CppUtilities::DateTime>(d, value, errors);
|
||||||
|
reflectable.store(d);
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace JsonReflector
|
} // namespace JsonReflector
|
||||||
|
|
||||||
namespace BinaryReflector {
|
namespace BinaryReflector {
|
||||||
|
|
||||||
template <>
|
template <>
|
||||||
LIBPKG_EXPORT void writeCustomType<LibPkg::PackageSearchResult>(
|
void writeCustomType<LibPkg::PackageSearchResult>(
|
||||||
BinarySerializer &serializer, const LibPkg::PackageSearchResult &packageSearchResult, BinaryVersion version)
|
BinarySerializer &serializer, const LibPkg::PackageSearchResult &packageSearchResult, BinaryVersion version)
|
||||||
{
|
{
|
||||||
if (const auto *const dbInfo = std::get_if<LibPkg::DatabaseInfo>(&packageSearchResult.db)) {
|
if (const auto *const dbInfo = std::get_if<LibPkg::DatabaseInfo>(&packageSearchResult.db)) {
|
||||||
|
@ -881,7 +924,7 @@ LIBPKG_EXPORT void writeCustomType<LibPkg::PackageSearchResult>(
|
||||||
}
|
}
|
||||||
|
|
||||||
template <>
|
template <>
|
||||||
LIBPKG_EXPORT BinaryVersion readCustomType<LibPkg::PackageSearchResult>(
|
BinaryVersion readCustomType<LibPkg::PackageSearchResult>(
|
||||||
BinaryDeserializer &deserializer, LibPkg::PackageSearchResult &packageSearchResult, BinaryVersion version)
|
BinaryDeserializer &deserializer, LibPkg::PackageSearchResult &packageSearchResult, BinaryVersion version)
|
||||||
{
|
{
|
||||||
deserializer.read(packageSearchResult.db.emplace<LibPkg::DatabaseInfo>().name, version);
|
deserializer.read(packageSearchResult.db.emplace<LibPkg::DatabaseInfo>().name, version);
|
||||||
|
@ -889,6 +932,20 @@ LIBPKG_EXPORT BinaryVersion readCustomType<LibPkg::PackageSearchResult>(
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <> void writeCustomType<LibPkg::AtomicDateTime>(BinarySerializer &serializer, const LibPkg::AtomicDateTime &dateTime, BinaryVersion version)
|
||||||
|
{
|
||||||
|
writeCustomType<CppUtilities::DateTime>(serializer, dateTime.load(), version);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <>
|
||||||
|
BinaryVersion readCustomType<LibPkg::AtomicDateTime>(BinaryDeserializer &deserializer, LibPkg::AtomicDateTime &dateTime, BinaryVersion version)
|
||||||
|
{
|
||||||
|
auto d = CppUtilities::DateTime();
|
||||||
|
auto v = readCustomType<CppUtilities::DateTime>(deserializer, d, version);
|
||||||
|
dateTime.store(d);
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace BinaryReflector
|
} // namespace BinaryReflector
|
||||||
|
|
||||||
} // namespace ReflectiveRapidJSON
|
} // namespace ReflectiveRapidJSON
|
||||||
|
|
|
@ -10,6 +10,7 @@
|
||||||
#include <c++utilities/chrono/datetime.h>
|
#include <c++utilities/chrono/datetime.h>
|
||||||
#include <c++utilities/misc/flagenumclass.h>
|
#include <c++utilities/misc/flagenumclass.h>
|
||||||
|
|
||||||
|
#include <atomic>
|
||||||
#include <filesystem>
|
#include <filesystem>
|
||||||
#include <optional>
|
#include <optional>
|
||||||
#include <unordered_set>
|
#include <unordered_set>
|
||||||
|
@ -100,7 +101,7 @@ struct LIBPKG_EXPORT UnresolvedDependencies : public ReflectiveRapidJSON::JsonSe
|
||||||
struct PackageUpdaterPrivate;
|
struct PackageUpdaterPrivate;
|
||||||
|
|
||||||
struct LIBPKG_EXPORT PackageUpdater {
|
struct LIBPKG_EXPORT PackageUpdater {
|
||||||
explicit PackageUpdater(Database &database);
|
explicit PackageUpdater(Database &database, bool clear = false);
|
||||||
~PackageUpdater();
|
~PackageUpdater();
|
||||||
|
|
||||||
PackageSpec findPackageWithID(const std::string &packageName);
|
PackageSpec findPackageWithID(const std::string &packageName);
|
||||||
|
@ -112,6 +113,22 @@ private:
|
||||||
std::unique_ptr<PackageUpdaterPrivate> m_d;
|
std::unique_ptr<PackageUpdaterPrivate> m_d;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct AtomicDateTime : public std::atomic<CppUtilities::DateTime> {
|
||||||
|
AtomicDateTime(CppUtilities::DateTime value = CppUtilities::DateTime())
|
||||||
|
: std::atomic<CppUtilities::DateTime>(value)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
AtomicDateTime(AtomicDateTime &&other)
|
||||||
|
: std::atomic<CppUtilities::DateTime>(other.load())
|
||||||
|
{
|
||||||
|
}
|
||||||
|
AtomicDateTime &operator=(AtomicDateTime &&other)
|
||||||
|
{
|
||||||
|
store(other.load());
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Database>, public ReflectiveRapidJSON::BinarySerializable<Database> {
|
struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Database>, public ReflectiveRapidJSON::BinarySerializable<Database> {
|
||||||
using PackageVisitorMove = std::function<bool(StorageID, std::shared_ptr<Package> &&)>; // package is invalidated/reused unless moved from!!!
|
using PackageVisitorMove = std::function<bool(StorageID, std::shared_ptr<Package> &&)>; // package is invalidated/reused unless moved from!!!
|
||||||
using PackageVisitorConst = std::function<bool(StorageID, const std::shared_ptr<Package> &)>;
|
using PackageVisitorConst = std::function<bool(StorageID, const std::shared_ptr<Package> &)>;
|
||||||
|
@ -134,8 +151,6 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
|
||||||
void loadPackages(FileMap &&databaseFiles, CppUtilities::DateTime lastModified);
|
void loadPackages(FileMap &&databaseFiles, CppUtilities::DateTime lastModified);
|
||||||
static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);
|
static bool isFileRelevant(const char *filePath, const char *fileName, mode_t);
|
||||||
std::vector<std::shared_ptr<Package>> findPackages(const std::function<bool(const Database &, const Package &)> &pred);
|
std::vector<std::shared_ptr<Package>> findPackages(const std::function<bool(const Database &, const Package &)> &pred);
|
||||||
void removePackageDependencies(StorageID packageID, const std::shared_ptr<Package> &package);
|
|
||||||
void addPackageDependencies(StorageID packageID, const std::shared_ptr<Package> &package);
|
|
||||||
void allPackages(const PackageVisitorMove &visitor);
|
void allPackages(const PackageVisitorMove &visitor);
|
||||||
void allPackagesByName(const PackageVisitorByName &visitor);
|
void allPackagesByName(const PackageVisitorByName &visitor);
|
||||||
std::size_t packageCount() const;
|
std::size_t packageCount() const;
|
||||||
|
@ -158,6 +173,11 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
|
||||||
PackageLocation locatePackage(const std::string &packageName) const;
|
PackageLocation locatePackage(const std::string &packageName) const;
|
||||||
std::string filesPathFromRegularPath() const;
|
std::string filesPathFromRegularPath() const;
|
||||||
|
|
||||||
|
private:
|
||||||
|
void removePackageDependencies(StorageID packageID, const std::shared_ptr<Package> &package);
|
||||||
|
void addPackageDependencies(StorageID packageID, const std::shared_ptr<Package> &package);
|
||||||
|
|
||||||
|
public:
|
||||||
std::string name;
|
std::string name;
|
||||||
std::string path;
|
std::string path;
|
||||||
std::string filesPath;
|
std::string filesPath;
|
||||||
|
@ -168,7 +188,7 @@ struct LIBPKG_EXPORT Database : public ReflectiveRapidJSON::JsonSerializable<Dat
|
||||||
std::vector<std::string> dependencies;
|
std::vector<std::string> dependencies;
|
||||||
std::string localPkgDir;
|
std::string localPkgDir;
|
||||||
std::string localDbDir;
|
std::string localDbDir;
|
||||||
CppUtilities::DateTime lastUpdate;
|
AtomicDateTime lastUpdate;
|
||||||
bool syncFromMirror = false;
|
bool syncFromMirror = false;
|
||||||
bool toBeDiscarded = false;
|
bool toBeDiscarded = false;
|
||||||
|
|
||||||
|
@ -245,6 +265,14 @@ template <>
|
||||||
LIBPKG_EXPORT void pull<LibPkg::PackageSearchResult>(LibPkg::PackageSearchResult &reflectable,
|
LIBPKG_EXPORT void pull<LibPkg::PackageSearchResult>(LibPkg::PackageSearchResult &reflectable,
|
||||||
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors);
|
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors);
|
||||||
|
|
||||||
|
// declare custom (de)serialization for AtomicDateTime
|
||||||
|
template <>
|
||||||
|
LIBPKG_EXPORT void push<LibPkg::AtomicDateTime>(
|
||||||
|
const LibPkg::AtomicDateTime &reflectable, RAPIDJSON_NAMESPACE::Value &value, RAPIDJSON_NAMESPACE::Document::AllocatorType &allocator);
|
||||||
|
template <>
|
||||||
|
LIBPKG_EXPORT void pull<LibPkg::AtomicDateTime>(LibPkg::AtomicDateTime &reflectable,
|
||||||
|
const RAPIDJSON_NAMESPACE::GenericValue<RAPIDJSON_NAMESPACE::UTF8<char>> &value, JsonDeserializationErrors *errors);
|
||||||
|
|
||||||
} // namespace JsonReflector
|
} // namespace JsonReflector
|
||||||
|
|
||||||
namespace BinaryReflector {
|
namespace BinaryReflector {
|
||||||
|
@ -256,6 +284,13 @@ template <>
|
||||||
LIBPKG_EXPORT BinaryVersion readCustomType<LibPkg::PackageSearchResult>(
|
LIBPKG_EXPORT BinaryVersion readCustomType<LibPkg::PackageSearchResult>(
|
||||||
BinaryDeserializer &deserializer, LibPkg::PackageSearchResult &packageSearchResult, BinaryVersion version);
|
BinaryDeserializer &deserializer, LibPkg::PackageSearchResult &packageSearchResult, BinaryVersion version);
|
||||||
|
|
||||||
|
template <>
|
||||||
|
LIBPKG_EXPORT void writeCustomType<LibPkg::AtomicDateTime>(
|
||||||
|
BinarySerializer &serializer, const LibPkg::AtomicDateTime &packageSearchResult, BinaryVersion version);
|
||||||
|
template <>
|
||||||
|
LIBPKG_EXPORT BinaryVersion readCustomType<LibPkg::AtomicDateTime>(
|
||||||
|
BinaryDeserializer &deserializer, LibPkg::AtomicDateTime &packageSearchResult, BinaryVersion version);
|
||||||
|
|
||||||
} // namespace BinaryReflector
|
} // namespace BinaryReflector
|
||||||
|
|
||||||
} // namespace ReflectiveRapidJSON
|
} // namespace ReflectiveRapidJSON
|
||||||
|
|
|
@ -57,7 +57,7 @@ template <typename StorageEntryType> void StorageCacheEntries<StorageEntryType>:
|
||||||
template <typename StorageEntriesType, typename StorageType, typename SpecType>
|
template <typename StorageEntriesType, typename StorageType, typename SpecType>
|
||||||
auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, ROTxn *txn, StorageID storageID) -> SpecType
|
auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &storage, ROTxn *txn, StorageID storageID) -> SpecType
|
||||||
{
|
{
|
||||||
// check for package in cache
|
// check for package in cache, should be ok even if the db is being updated
|
||||||
const auto ref = typename StorageEntryByID<typename Entries::StorageEntry>::result_type{ storageID, &storage };
|
const auto ref = typename StorageEntryByID<typename Entries::StorageEntry>::result_type{ storageID, &storage };
|
||||||
auto lock = std::unique_lock(m_mutex);
|
auto lock = std::unique_lock(m_mutex);
|
||||||
if (auto *const existingCacheEntry = m_entries.find(ref)) {
|
if (auto *const existingCacheEntry = m_entries.find(ref)) {
|
||||||
|
@ -67,13 +67,16 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
|
||||||
lock.unlock();
|
lock.unlock();
|
||||||
auto entry = std::make_shared<Entry>();
|
auto entry = std::make_shared<Entry>();
|
||||||
if (auto id = txn ? txn->get(storageID, *entry) : storage.packages.getROTransaction().get(storageID, *entry)) {
|
if (auto id = txn ? txn->get(storageID, *entry) : storage.packages.getROTransaction().get(storageID, *entry)) {
|
||||||
using CacheEntry = typename Entries::StorageEntry;
|
// try to acquire update lock to avoid update existing cache entries while db is being updated
|
||||||
using CacheRef = typename Entries::Ref;
|
if (const auto updateLock = std::unique_lock(storage.updateMutex, std::try_to_lock)) {
|
||||||
auto newCacheEntry = CacheEntry(CacheRef(storage, entry), id);
|
using CacheEntry = typename Entries::StorageEntry;
|
||||||
newCacheEntry.entry = entry;
|
using CacheRef = typename Entries::Ref;
|
||||||
lock = std::unique_lock(m_mutex);
|
auto newCacheEntry = CacheEntry(CacheRef(storage, entry), id);
|
||||||
m_entries.insert(std::move(newCacheEntry));
|
newCacheEntry.entry = entry;
|
||||||
lock.unlock();
|
lock = std::unique_lock(m_mutex);
|
||||||
|
m_entries.insert(std::move(newCacheEntry));
|
||||||
|
lock.unlock();
|
||||||
|
}
|
||||||
return SpecType(id, entry);
|
return SpecType(id, entry);
|
||||||
}
|
}
|
||||||
return SpecType(0, std::shared_ptr<Entry>());
|
return SpecType(0, std::shared_ptr<Entry>());
|
||||||
|
@ -92,7 +95,7 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
|
||||||
if (entryName.empty()) {
|
if (entryName.empty()) {
|
||||||
return SpecType(0, std::shared_ptr<Entry>());
|
return SpecType(0, std::shared_ptr<Entry>());
|
||||||
}
|
}
|
||||||
// check for package in cache
|
// check for package in cache, should be ok even if the db is being updated
|
||||||
using CacheRef = typename Entries::Ref;
|
using CacheRef = typename Entries::Ref;
|
||||||
const auto ref = CacheRef(storage, entryName);
|
const auto ref = CacheRef(storage, entryName);
|
||||||
auto lock = std::unique_lock(m_mutex);
|
auto lock = std::unique_lock(m_mutex);
|
||||||
|
@ -103,12 +106,15 @@ auto StorageCache<StorageEntriesType, StorageType, SpecType>::retrieve(Storage &
|
||||||
// check for package in storage, populate cache entry
|
// check for package in storage, populate cache entry
|
||||||
auto entry = std::make_shared<Entry>();
|
auto entry = std::make_shared<Entry>();
|
||||||
if (auto id = txn ? txn->template get<0>(entryName, *entry) : storage.packages.getROTransaction().template get<0>(entryName, *entry)) {
|
if (auto id = txn ? txn->template get<0>(entryName, *entry) : storage.packages.getROTransaction().template get<0>(entryName, *entry)) {
|
||||||
using CacheEntry = typename Entries::StorageEntry;
|
// try to acquire update lock to avoid update existing cache entries while db is being updated
|
||||||
auto newCacheEntry = CacheEntry(CacheRef(storage, entry), id);
|
if (const auto updateLock = std::unique_lock(storage.updateMutex, std::try_to_lock)) {
|
||||||
newCacheEntry.entry = entry;
|
using CacheEntry = typename Entries::StorageEntry;
|
||||||
lock = std::unique_lock(m_mutex);
|
auto newCacheEntry = CacheEntry(CacheRef(storage, entry), id);
|
||||||
m_entries.insert(std::move(newCacheEntry));
|
newCacheEntry.entry = entry;
|
||||||
lock.unlock();
|
lock = std::unique_lock(m_mutex);
|
||||||
|
m_entries.insert(std::move(newCacheEntry));
|
||||||
|
lock.unlock();
|
||||||
|
}
|
||||||
return SpecType(id, entry);
|
return SpecType(id, entry);
|
||||||
}
|
}
|
||||||
return SpecType(0, std::shared_ptr<Entry>());
|
return SpecType(0, std::shared_ptr<Entry>());
|
||||||
|
|
|
@ -4,6 +4,8 @@
|
||||||
#include "./package.h"
|
#include "./package.h"
|
||||||
#include "./storagegeneric.h"
|
#include "./storagegeneric.h"
|
||||||
|
|
||||||
|
#include <mutex>
|
||||||
|
|
||||||
namespace LibPkg {
|
namespace LibPkg {
|
||||||
|
|
||||||
using PackageStorage = LMDBSafe::TypedDBI<Package, LMDBSafe::index_on<Package, std::string, &Package::name>>;
|
using PackageStorage = LMDBSafe::TypedDBI<Package, LMDBSafe::index_on<Package, std::string, &Package::name>>;
|
||||||
|
@ -50,6 +52,7 @@ struct DatabaseStorage {
|
||||||
DependencyStorage requiredDeps;
|
DependencyStorage requiredDeps;
|
||||||
LibraryDependencyStorage providedLibs;
|
LibraryDependencyStorage providedLibs;
|
||||||
LibraryDependencyStorage requiredLibs;
|
LibraryDependencyStorage requiredLibs;
|
||||||
|
std::mutex updateMutex; // must be acquired to update packages, concurrent reads should still be possible
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::shared_ptr<LMDBSafe::MDBEnv> m_env;
|
std::shared_ptr<LMDBSafe::MDBEnv> m_env;
|
||||||
|
|
|
@ -85,7 +85,7 @@ void ReloadDatabase::run()
|
||||||
if (!force) {
|
if (!force) {
|
||||||
auto configReadLock2 = m_setup.config.lockToRead();
|
auto configReadLock2 = m_setup.config.lockToRead();
|
||||||
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
||||||
if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) {
|
if (const auto lastUpdate = destinationDb->lastUpdate.load(); lastModified <= lastUpdate) {
|
||||||
configReadLock2.unlock();
|
configReadLock2.unlock();
|
||||||
m_buildAction->appendOutput(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
|
m_buildAction->appendOutput(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
|
||||||
"\" from local file \"", dbPath, "\"; last modification time <= last update (", lastModified.toString(), '<', '=',
|
"\" from local file \"", dbPath, "\"; last modification time <= last update (", lastModified.toString(), '<', '=',
|
||||||
|
@ -98,7 +98,7 @@ void ReloadDatabase::run()
|
||||||
dbFileLock.lock().unlock();
|
dbFileLock.lock().unlock();
|
||||||
m_buildAction->appendOutput(
|
m_buildAction->appendOutput(
|
||||||
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
Phrases::InfoMessage, "Loading database \"", dbName, '@', dbArch, "\" from local file \"", dbPath, "\"\n");
|
||||||
const auto configLock = m_setup.config.lockToWrite();
|
const auto configLock = m_setup.config.lockToRead();
|
||||||
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
auto *const destinationDb = m_setup.config.findDatabase(dbName, dbArch);
|
||||||
if (!destinationDb) {
|
if (!destinationDb) {
|
||||||
m_buildAction->appendOutput(
|
m_buildAction->appendOutput(
|
||||||
|
@ -123,7 +123,7 @@ void ReloadDatabase::run()
|
||||||
|
|
||||||
// clear AUR cache
|
// clear AUR cache
|
||||||
if (m_toAur) {
|
if (m_toAur) {
|
||||||
auto lock = m_setup.config.lockToWrite();
|
auto lock = m_setup.config.lockToRead();
|
||||||
m_setup.config.aur.clearPackages();
|
m_setup.config.aur.clearPackages();
|
||||||
lock.unlock();
|
lock.unlock();
|
||||||
m_buildAction->log()(Phrases::InfoMessage, "Cleared AUR cache\n");
|
m_buildAction->log()(Phrases::InfoMessage, "Cleared AUR cache\n");
|
||||||
|
|
|
@ -362,7 +362,7 @@ void ReloadLibraryDependencies::loadPackageInfoFromContents()
|
||||||
m_buildAction->appendOutput(Phrases::SuccessMessage, "Adding parsed information to databases ...\n");
|
m_buildAction->appendOutput(Phrases::SuccessMessage, "Adding parsed information to databases ...\n");
|
||||||
std::size_t counter = 0;
|
std::size_t counter = 0;
|
||||||
for (DatabaseToConsider &relevantDb : m_relevantPackagesByDatabase) {
|
for (DatabaseToConsider &relevantDb : m_relevantPackagesByDatabase) {
|
||||||
auto configWritelock = m_setup.config.lockToWrite(); // acquire lock within loop to allow intermediate reads
|
auto configWritelock = m_setup.config.lockToRead();
|
||||||
auto *const db = m_setup.config.findDatabase(relevantDb.name, relevantDb.arch);
|
auto *const db = m_setup.config.findDatabase(relevantDb.name, relevantDb.arch);
|
||||||
if (!db) {
|
if (!db) {
|
||||||
continue; // the whole database has been removed while we were loading package contents
|
continue; // the whole database has been removed while we were loading package contents
|
||||||
|
|
|
@ -591,7 +591,7 @@ void CleanRepository::run()
|
||||||
const auto lastModified = LibPkg::lastModified(dbFile);
|
const auto lastModified = LibPkg::lastModified(dbFile);
|
||||||
if (lastModified != db->lastUpdate) {
|
if (lastModified != db->lastUpdate) {
|
||||||
m_messages.errors.emplace_back("The db file's last modification (" % lastModified.toString() % ") does not match the last db update ("
|
m_messages.errors.emplace_back("The db file's last modification (" % lastModified.toString() % ") does not match the last db update ("
|
||||||
% db->lastUpdate.toString()
|
% db->lastUpdate.load().toString()
|
||||||
+ ").");
|
+ ").");
|
||||||
fatalError = true;
|
fatalError = true;
|
||||||
}
|
}
|
||||||
|
|
|
@ -608,7 +608,7 @@ void ServiceSetup::printDatabases()
|
||||||
cerr << Phrases::SuccessMessage << "Found " << config.databases.size() << " databases:" << Phrases::End;
|
cerr << Phrases::SuccessMessage << "Found " << config.databases.size() << " databases:" << Phrases::End;
|
||||||
for (const auto &db : config.databases) {
|
for (const auto &db : config.databases) {
|
||||||
cerr << Phrases::SubMessage << db.name << "@" << db.arch << ": " << db.packageCount() << " packages, last updated on "
|
cerr << Phrases::SubMessage << db.name << "@" << db.arch << ": " << db.packageCount() << " packages, last updated on "
|
||||||
<< db.lastUpdate.toString(DateTimeOutputFormat::DateAndTime) << Phrases::End << " - path: " << db.path
|
<< db.lastUpdate.load().toString(DateTimeOutputFormat::DateAndTime) << Phrases::End << " - path: " << db.path
|
||||||
<< "\n - local db dir: " << db.localDbDir << "\n - local package dir: " << db.localPkgDir << '\n';
|
<< "\n - local db dir: " << db.localDbDir << "\n - local package dir: " << db.localPkgDir << '\n';
|
||||||
}
|
}
|
||||||
cerr << Phrases::SubMessage << "AUR (" << config.aur.packageCount() << " packages cached)" << Phrases::End;
|
cerr << Phrases::SubMessage << "AUR (" << config.aur.packageCount() << " packages cached)" << Phrases::End;
|
||||||
|
|
|
@ -357,7 +357,7 @@ void postLoadPackages(const Params ¶ms, ResponseHandler &&handler)
|
||||||
{
|
{
|
||||||
const auto withFiles = params.target.hasFlag("with-files");
|
const auto withFiles = params.target.hasFlag("with-files");
|
||||||
const auto force = params.target.hasFlag("force");
|
const auto force = params.target.hasFlag("force");
|
||||||
auto lock = params.setup.config.lockToWrite();
|
auto lock = params.setup.config.lockToRead();
|
||||||
params.setup.config.loadAllPackages(withFiles, force);
|
params.setup.config.loadAllPackages(withFiles, force);
|
||||||
lock.unlock();
|
lock.unlock();
|
||||||
handler(makeText(params.request(), "packages loaded"));
|
handler(makeText(params.request(), "packages loaded"));
|
||||||
|
|
|
@ -52,7 +52,7 @@ void searchAurPackages(LogContext &log, ServiceSetup &setup, const std::string &
|
||||||
try {
|
try {
|
||||||
// parse and cache the AUR packages
|
// parse and cache the AUR packages
|
||||||
auto packages = Package::fromAurRpcJson(body.data(), body.size(), PackageOrigin::AurRpcSearch);
|
auto packages = Package::fromAurRpcJson(body.data(), body.size(), PackageOrigin::AurRpcSearch);
|
||||||
auto lock = setup.config.lockToWrite();
|
auto lock = setup.config.lockToRead();
|
||||||
auto updater = LibPkg::PackageUpdater(setup.config.aur);
|
auto updater = LibPkg::PackageUpdater(setup.config.aur);
|
||||||
for (auto &[packageID, package] : packages) {
|
for (auto &[packageID, package] : packages) {
|
||||||
packageID = updater.update(package);
|
packageID = updater.update(package);
|
||||||
|
@ -100,7 +100,7 @@ std::shared_ptr<AurQuerySession> queryAurPackagesInternal(LogContext &log, Servi
|
||||||
try {
|
try {
|
||||||
// parse and cache the AUR packages
|
// parse and cache the AUR packages
|
||||||
auto packagesFromAur = Package::fromAurRpcJson(body.data(), body.size());
|
auto packagesFromAur = Package::fromAurRpcJson(body.data(), body.size());
|
||||||
auto lock = setup.config.lockToWrite();
|
auto lock = setup.config.lockToRead();
|
||||||
auto updater = PackageUpdater(setup.config.aur);
|
auto updater = PackageUpdater(setup.config.aur);
|
||||||
for (auto &[packageID, package] : packagesFromAur) {
|
for (auto &[packageID, package] : packagesFromAur) {
|
||||||
packageID = updater.update(package);
|
packageID = updater.update(package);
|
||||||
|
|
|
@ -100,7 +100,7 @@ void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQu
|
||||||
session3.skip = true;
|
session3.skip = true;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const auto lastUpdate = destinationDb->lastUpdate;
|
const auto lastUpdate = destinationDb->lastUpdate.load();
|
||||||
configReadLock.unlock();
|
configReadLock.unlock();
|
||||||
if (lastModified > lastUpdate) {
|
if (lastModified > lastUpdate) {
|
||||||
return;
|
return;
|
||||||
|
@ -138,7 +138,7 @@ void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQu
|
||||||
} else if (!force) {
|
} else if (!force) {
|
||||||
auto configReadLock = setup.config.lockToRead();
|
auto configReadLock = setup.config.lockToRead();
|
||||||
if (auto *const destinationDb = setup.config.findDatabase(dbName, dbArch)) {
|
if (auto *const destinationDb = setup.config.findDatabase(dbName, dbArch)) {
|
||||||
if (const auto lastUpdate = destinationDb->lastUpdate; lastModified <= lastUpdate) {
|
if (const auto lastUpdate = destinationDb->lastUpdate.load(); lastModified <= lastUpdate) {
|
||||||
configReadLock.unlock();
|
configReadLock.unlock();
|
||||||
log(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
|
log(Phrases::InfoMessage, "Skip loading database \"", dbName, '@', dbArch,
|
||||||
"\" from mirror response; last modification time <= last update (", lastModified.toString(),
|
"\" from mirror response; last modification time <= last update (", lastModified.toString(),
|
||||||
|
@ -157,9 +157,10 @@ void queryDatabases(LogContext &log, ServiceSetup &setup, std::vector<DatabaseQu
|
||||||
auto packages = Package::fromDatabaseFile(std::move(files));
|
auto packages = Package::fromDatabaseFile(std::move(files));
|
||||||
|
|
||||||
// insert packages
|
// insert packages
|
||||||
auto lock = setup.config.lockToWrite();
|
auto lock = setup.config.lockToRead();
|
||||||
auto db = setup.config.findDatabase(dbName, dbArch);
|
auto db = setup.config.findDatabase(dbName, dbArch);
|
||||||
if (!db) {
|
if (!db) {
|
||||||
|
lock.unlock();
|
||||||
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
log(Phrases::ErrorMessage, "Retrieved database file for \"", dbName, '@', dbArch, "\" but it no longer exists; discarding\n");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue