improved caching
This commit is contained in:
parent
503eef27ed
commit
549060fa90
|
@ -19,9 +19,10 @@ namespace RepoIndex {
|
|||
using namespace Utilities;
|
||||
|
||||
/*!
|
||||
* \brief The AlpmDataBase class wraps an ALPM data base struct and holds additional meta information.
|
||||
* \class AlpmDatabase
|
||||
* \brief The AlpmDatabase class wraps an ALPM data base struct and holds additional meta information.
|
||||
*
|
||||
* All packages returned by the AlpmDataBase class are AlpmPackage instances.
|
||||
* All packages returned by the AlpmDatabase class are AlpmPackage instances.
|
||||
*/
|
||||
|
||||
class LoadPackage
|
||||
|
@ -47,24 +48,24 @@ private:
|
|||
QMutex *m_mutex;
|
||||
};
|
||||
|
||||
PackageLoader::PackageLoader(AlpmDatabase *db)
|
||||
AlpmPackageLoader::AlpmPackageLoader(AlpmDatabase *repository)
|
||||
{
|
||||
for(auto *pkg : PackageList(alpm_db_get_pkgcache(db->ptr()))) {
|
||||
for(auto *pkg : PackageList(alpm_db_get_pkgcache(repository->ptr()))) {
|
||||
m_packages << pkg;
|
||||
}
|
||||
m_future = QtConcurrent::map(m_packages, LoadPackage(db, &m_mutex));
|
||||
m_future = QtConcurrent::map(m_packages, LoadPackage(repository, &m_mutex));
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Creates a new instance wrapping the specified database struct.
|
||||
*/
|
||||
RepoIndex::AlpmDatabase::AlpmDatabase(alpm_db_t *dataBase, const QString &dbPath, uint32 index, QObject *parent) :
|
||||
AlpmDatabase::AlpmDatabase(alpm_db_t *dataBase, const QString &dbPath, uint32 index, QObject *parent) :
|
||||
Repository(QString::fromLocal8Bit(alpm_db_get_name(dataBase)), index, parent),
|
||||
m_ptr(dataBase),
|
||||
m_dbFile(QStringLiteral("%1/sync/%2").arg(dbPath, m_name))
|
||||
{}
|
||||
|
||||
PackageLoader *AlpmDatabase::init()
|
||||
AlpmPackageLoader *AlpmDatabase::init()
|
||||
{
|
||||
if(alpm_db_get_usage(m_ptr, &m_usage)) {
|
||||
m_usage = static_cast<decltype(m_usage)>(ALPM_DB_USAGE_ALL);
|
||||
|
@ -78,19 +79,11 @@ PackageLoader *AlpmDatabase::init()
|
|||
m_description = QStringLiteral("Database »%1«").arg(m_name);
|
||||
}
|
||||
}
|
||||
// for(auto *nativePkg : PackageList(alpm_db_get_pkgcache(m_ptr))) {
|
||||
// auto pkg = make_unique<AlpmPackage>(nativePkg, this);
|
||||
// QString pkgName = pkg->name();
|
||||
// for(const auto &grpName : pkg->groups()) {
|
||||
// m_groups[grpName] << pkg.get();
|
||||
// }
|
||||
// m_packages.emplace(pkgName, move(pkg));
|
||||
// }
|
||||
for(const char *str : servers()) {
|
||||
m_serverUrls << qstr(str);
|
||||
}
|
||||
m_sigLevel = alpm_db_get_siglevel(m_ptr);
|
||||
return new PackageLoader(this);
|
||||
return new AlpmPackageLoader(this);
|
||||
}
|
||||
|
||||
//AlpmDatabase::AlpmDatabase(const QString &dataBaseFile, QObject *parent) :
|
||||
|
|
|
@ -15,30 +15,20 @@ namespace RepoIndex {
|
|||
class AlpmPackage;
|
||||
class AlpmDatabase;
|
||||
|
||||
class PackageLoader
|
||||
class AlpmPackageLoader : public PackageLoader
|
||||
{
|
||||
public:
|
||||
PackageLoader(AlpmDatabase *db);
|
||||
QFuture<void> &future();
|
||||
AlpmPackageLoader(AlpmDatabase *db);
|
||||
|
||||
private:
|
||||
QMutex m_mutex;
|
||||
QList<alpm_pkg_t *> m_packages;
|
||||
QFuture<void> m_future;
|
||||
};
|
||||
|
||||
inline QFuture<void> &PackageLoader::future()
|
||||
{
|
||||
return m_future;
|
||||
}
|
||||
|
||||
class AlpmDatabase : public Repository
|
||||
{
|
||||
friend class EmplacePackage;
|
||||
|
||||
public:
|
||||
explicit AlpmDatabase(alpm_db_t *dataBase, const QString &dbPath, uint32 index = invalidIndex, QObject *parent = nullptr);
|
||||
PackageLoader *init();
|
||||
AlpmPackageLoader *init();
|
||||
// explicit AlpmDatabase(const QString &dataBaseFile, QObject *parent = nullptr);
|
||||
|
||||
RepositoryType type() const;
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
#include <QStringBuilder>
|
||||
#include <QFile>
|
||||
#include <QDataStream>
|
||||
#include <QTimer>
|
||||
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
|
@ -506,6 +507,35 @@ void Manager::initAlpmDataBases(bool computeRequiredBy)
|
|||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Returns whether automatic cache maintenance is enabled.
|
||||
* \remarks If enabled, the cache will be cleaned and saved frequently.
|
||||
*/
|
||||
bool Manager::isAutoCacheMaintenanceEnabled() const
|
||||
{
|
||||
return m_cacheTimer && m_cacheTimer->isActive();
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Sets whether automatic cache maintenacne is enabled.
|
||||
* \sa isAutoCacheMaintenanceEnabled()
|
||||
*/
|
||||
void Manager::setAutoCacheMaintenanceEnabled(bool enabled)
|
||||
{
|
||||
if(isAutoCacheMaintenanceEnabled() != enabled) {
|
||||
if(enabled) {
|
||||
if(!m_cacheTimer) {
|
||||
m_cacheTimer = make_unique<QTimer>();
|
||||
m_cacheTimer->setInterval(5 * 60 * 1000);
|
||||
QObject::connect(m_cacheTimer.get(), &QTimer::timeout, bind(&Manager::maintainCache, this));
|
||||
}
|
||||
m_cacheTimer->start();
|
||||
} else {
|
||||
m_cacheTimer->stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Writes the cache for all repositories where caching makes sense.
|
||||
*/
|
||||
|
@ -543,6 +573,29 @@ void Manager::restoreCache()
|
|||
}
|
||||
}
|
||||
|
||||
void Manager::cleanCache()
|
||||
{
|
||||
// currently clear only AUR cache
|
||||
if(userRepository()) {
|
||||
userRepository()->cleanOutdatedPackages();
|
||||
}
|
||||
}
|
||||
|
||||
void Manager::wipeCache()
|
||||
{
|
||||
// currently wipe only AUR cache
|
||||
if(userRepository()) {
|
||||
userRepository()->wipePackages();
|
||||
}
|
||||
}
|
||||
|
||||
void Manager::maintainCache()
|
||||
{
|
||||
cleanCache();
|
||||
// TODO: write cache only when modified
|
||||
writeCache();
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Unregisters all registred sync databases.
|
||||
*/
|
||||
|
|
|
@ -14,6 +14,8 @@
|
|||
#include <map>
|
||||
#include <memory>
|
||||
|
||||
QT_FORWARD_DECLARE_CLASS(QTimer)
|
||||
|
||||
namespace RepoIndex {
|
||||
|
||||
class Config;
|
||||
|
@ -38,8 +40,6 @@ public:
|
|||
|
||||
// configuration, signature level, etc
|
||||
const Config &config() const;
|
||||
bool writeCacheBeforeGone() const;
|
||||
void setWriteCacheBeforeGone(bool writeCacheBeforeGone);
|
||||
int sigLevel() const;
|
||||
void setSigLevel(int sigLevel);
|
||||
int localFileSigLevel() const;
|
||||
|
@ -52,8 +52,17 @@ public:
|
|||
void registerDataBasesFromPacmanConfig();
|
||||
void registerDatabasesFromRepoIndexConfig();
|
||||
void initAlpmDataBases(bool computeRequiredBy);
|
||||
|
||||
// caching
|
||||
bool writeCacheBeforeGone() const;
|
||||
void setWriteCacheBeforeGone(bool writeCacheBeforeGone);
|
||||
bool isAutoCacheMaintenanceEnabled() const;
|
||||
void setAutoCacheMaintenanceEnabled(bool enabled);
|
||||
void writeCache();
|
||||
void restoreCache();
|
||||
void cleanCache();
|
||||
void wipeCache();
|
||||
void maintainCache();
|
||||
|
||||
void unregisterSyncDataBases();
|
||||
|
||||
|
@ -89,6 +98,7 @@ public:
|
|||
private:
|
||||
const Config &m_config;
|
||||
bool m_writeCacheBeforeGone;
|
||||
std::unique_ptr<QTimer> m_cacheTimer;
|
||||
alpm_handle_t *m_handle;
|
||||
int m_sigLevel;
|
||||
int m_localFileSigLevel;
|
||||
|
|
|
@ -189,6 +189,7 @@ public:
|
|||
// caching
|
||||
void writeToCacheStream(QDataStream &out);
|
||||
void restoreFromCacheStream(QDataStream &in);
|
||||
void reinitEntries();
|
||||
|
||||
// parsing src/pkg info
|
||||
void putInfo(const QList<QPair<QString, QString> > &baseInfo, const QList<QPair<QString, QString> > &pkgInfo, bool includesSourceRelatedMetaData = false);
|
||||
|
|
|
@ -12,6 +12,7 @@
|
|||
#include <cassert>
|
||||
|
||||
using namespace std;
|
||||
using namespace ChronoUtilities;
|
||||
|
||||
namespace RepoIndex {
|
||||
|
||||
|
@ -70,6 +71,19 @@ const QStringList RepoIndex::Repository::packageNames() const
|
|||
return names;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Initializes the repository.
|
||||
* \remarks
|
||||
* - Does not restore cache. For restoring cache see restoreFromCacheStream().
|
||||
* - Performs asynchronously and hence returns immidiately. Returns a PackageLoader
|
||||
* object which QFuture can be used to wait for initializing to finish.
|
||||
* - Might return nullptr if initialization is tivial.
|
||||
*/
|
||||
PackageLoader *Repository::init()
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Requests suggestions for the specified search phrase.
|
||||
* \returns Returns a reply object used for the request. The reply must be destroyed by the caller
|
||||
|
@ -92,6 +106,7 @@ Repository::Repository(const QString &name, uint32 index, QObject *parent) :
|
|||
QObject(parent),
|
||||
m_index(index),
|
||||
m_name(name),
|
||||
m_maxPackageAge(TimeSpan::infinity()),
|
||||
m_usage(static_cast<alpm_db_usage_t>(0)),
|
||||
m_sigLevel(static_cast<alpm_siglevel_t>(ALPM_SIGSTATUS_INVALID)),
|
||||
m_lock(QReadWriteLock::Recursive)
|
||||
|
@ -431,7 +446,7 @@ void Repository::writeToCacheStream(QDataStream &out)
|
|||
/*!
|
||||
* \brief Restores the repository information from cache.
|
||||
*/
|
||||
void Repository::restoreFromCacheStream(QDataStream &in)
|
||||
void Repository::restoreFromCacheStream(QDataStream &in, bool skipOutdated)
|
||||
{
|
||||
quint32 magic;
|
||||
in >> magic;
|
||||
|
@ -447,11 +462,16 @@ void Repository::restoreFromCacheStream(QDataStream &in)
|
|||
quint32 packageCount;
|
||||
in >> packageCount;
|
||||
bool good = true;
|
||||
const auto now = DateTime::now();
|
||||
for(quint32 i = 0; i < packageCount && good && in.status() == QDataStream::Ok; ++i) {
|
||||
if(auto package = emptyPackage()) {
|
||||
package->restoreFromCacheStream(in);
|
||||
if(!package->name().isEmpty()) {
|
||||
if(!skipOutdated || !((now - package->timeStamp()) > maxPackageAge())) {
|
||||
m_packages[package->name()] = move(package);
|
||||
} else {
|
||||
cerr << shchar << "Info: Cache entry for package \"" << package->name().toLocal8Bit().data() << "\" is outdated and won't be restored." << endl;
|
||||
}
|
||||
} else {
|
||||
good = false;
|
||||
}
|
||||
|
@ -510,6 +530,26 @@ void Repository::restoreSpecificCacheHeader(QDataStream &in)
|
|||
Q_UNUSED(in)
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Cleans the repository from outdated packages.
|
||||
* \remarks Does nothing if maxPackageAge() is infinity (which is the default).
|
||||
*/
|
||||
void Repository::cleanOutdatedPackages()
|
||||
{
|
||||
if(maxPackageAge().isInfinity()) {
|
||||
return;
|
||||
}
|
||||
auto now = DateTime::now();
|
||||
for(auto i = m_packages.begin(); i != m_packages.end(); ) {
|
||||
const Package &pkg = *i->second;
|
||||
if((now - pkg.timeStamp()) > maxPackageAge()) {
|
||||
i = m_packages.erase(i);
|
||||
} else {
|
||||
++i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Adds packages parsed from the specified \a srcInfo.
|
||||
* \returns Returns the added packages.
|
||||
|
@ -632,5 +672,4 @@ QList<Package *> Repository::addPackagesFromSrcInfo(const QByteArray &srcInfo)
|
|||
return packages;
|
||||
}
|
||||
|
||||
|
||||
} // namespace PackageManagement
|
||||
|
|
|
@ -112,6 +112,29 @@ inline const Repository *SuggestionsReply::repository() const
|
|||
return m_repo;
|
||||
}
|
||||
|
||||
class PackageLoader
|
||||
{
|
||||
public:
|
||||
QFuture<void> &future();
|
||||
virtual ~PackageLoader();
|
||||
|
||||
protected:
|
||||
PackageLoader();
|
||||
QMutex m_mutex;
|
||||
QFuture<void> m_future;
|
||||
};
|
||||
|
||||
inline PackageLoader::PackageLoader()
|
||||
{}
|
||||
|
||||
inline PackageLoader::~PackageLoader()
|
||||
{}
|
||||
|
||||
inline QFuture<void> &PackageLoader::future()
|
||||
{
|
||||
return m_future;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief The RepositoryType enum specifies the type of a repository object.
|
||||
*/
|
||||
|
@ -170,6 +193,7 @@ public:
|
|||
alpm_siglevel_t sigLevel() const;
|
||||
|
||||
// gathering data
|
||||
virtual PackageLoader *init();
|
||||
virtual PackageDetailAvailability requestsRequired(PackageDetail packageDetail = PackageDetail::Basics) const;
|
||||
virtual SuggestionsReply *requestSuggestions(const QString &phrase);
|
||||
virtual PackageReply *requestPackageInfo(const QStringList &packageNames, bool forceUpdate = false);
|
||||
|
@ -207,10 +231,14 @@ public:
|
|||
// caching
|
||||
bool isCachingUseful() const;
|
||||
void writeToCacheStream(QDataStream &out);
|
||||
void restoreFromCacheStream(QDataStream &in);
|
||||
void restoreFromCacheStream(QDataStream &in, bool skipOutdated = true);
|
||||
virtual void writeSpecificCacheHeader(QDataStream &out);
|
||||
virtual std::unique_ptr<Package> emptyPackage();
|
||||
virtual void restoreSpecificCacheHeader(QDataStream &in);
|
||||
ChronoUtilities::TimeSpan maxPackageAge() const;
|
||||
void setMaxPackageAge(ChronoUtilities::TimeSpan maxPackageAge);
|
||||
void cleanOutdatedPackages();
|
||||
void wipePackages();
|
||||
|
||||
// parsing src/pkg info
|
||||
QList<Package *> addPackagesFromSrcInfo(const QByteArray &srcInfo);
|
||||
|
@ -228,6 +256,7 @@ protected:
|
|||
QString m_name;
|
||||
QString m_description;
|
||||
std::map<QString, std::unique_ptr<Package> > m_packages;
|
||||
ChronoUtilities::TimeSpan m_maxPackageAge;
|
||||
alpm_db_usage_t m_usage;
|
||||
std::map<QString, QList<Package *> > m_groups;
|
||||
QStringList m_serverUrls;
|
||||
|
@ -424,6 +453,21 @@ inline bool Repository::isCachingUseful() const
|
|||
}
|
||||
}
|
||||
|
||||
inline ChronoUtilities::TimeSpan Repository::maxPackageAge() const
|
||||
{
|
||||
return m_maxPackageAge;
|
||||
}
|
||||
|
||||
inline void Repository::setMaxPackageAge(ChronoUtilities::TimeSpan maxPackageAge)
|
||||
{
|
||||
m_maxPackageAge = maxPackageAge;
|
||||
}
|
||||
|
||||
inline void Repository::wipePackages()
|
||||
{
|
||||
m_packages.clear();
|
||||
}
|
||||
|
||||
inline QReadWriteLock *Repository::lock() const
|
||||
{
|
||||
return const_cast<QReadWriteLock *>(&m_lock);
|
||||
|
|
3
main.cpp
3
main.cpp
|
@ -73,13 +73,12 @@ int main(int argc, char *argv[])
|
|||
if(configArgs.serverArg.isPresent()) {
|
||||
// setup the server
|
||||
Server server(manager, manager.config());
|
||||
manager.setAutoCacheMaintenanceEnabled(true);
|
||||
QObject::connect(&server, &Server::closed, &application, &QCoreApplication::quit);
|
||||
// run Qt loop
|
||||
return application.exec();
|
||||
} else if(configArgs.buildOrderArg.isPresent()) {
|
||||
BuildOrderResolverCli resolver(manager, configArgs.buildOrderArg.values(), configArgs.addSourceOnlyDeps.isPresent());
|
||||
return resolver.exec();
|
||||
//BuildOrderResolver::printResults(resolver.resolve(configArgs.buildOrderArg.values()));
|
||||
} else if(configArgs.mingwBundleArg.isPresent()) {
|
||||
MingwBundle bundle(manager, configArgs.mingwBundleArg.values(), configArgs.iconThemesArg.values(), configArgs.extraPackagesArg.values());
|
||||
bundle.createBundle(configArgs.targetDirArg.isPresent() ? configArgs.targetDirArg.values().front() : string("."),
|
||||
|
|
|
@ -21,6 +21,7 @@
|
|||
#include <QStringBuilder>
|
||||
|
||||
using namespace std;
|
||||
using namespace ChronoUtilities;
|
||||
|
||||
namespace RepoIndex {
|
||||
|
||||
|
@ -190,6 +191,7 @@ UserRepository::UserRepository(QNetworkAccessManager &networkAccessManager, QObj
|
|||
m_networkAccessManager(networkAccessManager)
|
||||
{
|
||||
m_description = QStringLiteral("Arch User Repository");
|
||||
m_maxPackageAge = TimeSpan::fromDays(1.0);
|
||||
}
|
||||
|
||||
RepositoryType UserRepository::type() const
|
||||
|
|
Loading…
Reference in New Issue