Martchus
/
repoindex
1
0
Fork 0
Code Issues Pull Requests Releases 1 Wiki Activity

improved caching

This commit is contained in:
Martchus 2016-01-18 20:34:29 +01:00
parent 503eef27ed
commit 549060fa90
9 changed files with 168 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
alpm/alpmdatabase.cpp
View File

@ -19,9 +19,10 @@ namespace RepoIndex {
using namespace Utilities;
/*!
* \brief The AlpmDataBase class wraps an ALPM data base struct and holds additional meta information.
* \class AlpmDatabase
* \brief The AlpmDatabase class wraps an ALPM data base struct and holds additional meta information.
*
* All packages returned by the AlpmDataBase class are AlpmPackage instances.
* All packages returned by the AlpmDatabase class are AlpmPackage instances.
*/
class LoadPackage
@ -47,24 +48,24 @@ private:
QMutex *m_mutex;
};
PackageLoader::PackageLoader(AlpmDatabase *db)
AlpmPackageLoader::AlpmPackageLoader(AlpmDatabase *repository)
{
for(auto *pkg : PackageList(alpm_db_get_pkgcache(db->ptr()))) {
for(auto *pkg : PackageList(alpm_db_get_pkgcache(repository->ptr()))) {
m_packages << pkg;
}
m_future = QtConcurrent::map(m_packages, LoadPackage(db, &m_mutex));
m_future = QtConcurrent::map(m_packages, LoadPackage(repository, &m_mutex));
}
/*!
* \brief Creates a new instance wrapping the specified database struct.
*/
RepoIndex::AlpmDatabase::AlpmDatabase(alpm_db_t *dataBase, const QString &dbPath, uint32 index, QObject *parent) :
AlpmDatabase::AlpmDatabase(alpm_db_t *dataBase, const QString &dbPath, uint32 index, QObject *parent) :
Repository(QString::fromLocal8Bit(alpm_db_get_name(dataBase)), index, parent),
m_ptr(dataBase),
m_dbFile(QStringLiteral("%1/sync/%2").arg(dbPath, m_name))
{}
PackageLoader *AlpmDatabase::init()
AlpmPackageLoader *AlpmDatabase::init()
{
if(alpm_db_get_usage(m_ptr, &m_usage)) {
m_usage = static_cast<decltype(m_usage)>(ALPM_DB_USAGE_ALL);
@ -78,19 +79,11 @@ PackageLoader *AlpmDatabase::init()
m_description = QStringLiteral("Database »%1«").arg(m_name);
}
}
// for(auto *nativePkg : PackageList(alpm_db_get_pkgcache(m_ptr))) {
// auto pkg = make_unique<AlpmPackage>(nativePkg, this);
// QString pkgName = pkg->name();
// for(const auto &grpName : pkg->groups()) {
// m_groups[grpName] << pkg.get();
// }
// m_packages.emplace(pkgName, move(pkg));
// }
for(const char *str : servers()) {
m_serverUrls << qstr(str);
}
m_sigLevel = alpm_db_get_siglevel(m_ptr);
return new PackageLoader(this);
return new AlpmPackageLoader(this);
}
//AlpmDatabase::AlpmDatabase(const QString &dataBaseFile, QObject *parent) :

16
alpm/alpmdatabase.h
View File

@ -15,30 +15,20 @@ namespace RepoIndex {
class AlpmPackage;
class AlpmDatabase;
class PackageLoader
class AlpmPackageLoader : public PackageLoader
{
public:
PackageLoader(AlpmDatabase *db);
QFuture<void> &future();
AlpmPackageLoader(AlpmDatabase *db);
private:
QMutex m_mutex;
QList<alpm_pkg_t *> m_packages;
QFuture<void> m_future;
};
inline QFuture<void> &PackageLoader::future()
{
return m_future;
}
class AlpmDatabase : public Repository
{
friend class EmplacePackage;
public:
explicit AlpmDatabase(alpm_db_t *dataBase, const QString &dbPath, uint32 index = invalidIndex, QObject *parent = nullptr);
PackageLoader *init();
AlpmPackageLoader *init();
// explicit AlpmDatabase(const QString &dataBaseFile, QObject *parent = nullptr);
RepositoryType type() const;

53
alpm/manager.cpp
View File

@ -16,6 +16,7 @@
#include <QStringBuilder>
#include <QFile>
#include <QDataStream>
#include <QTimer>
#include <fstream>
#include <iostream>
@ -506,6 +507,35 @@ void Manager::initAlpmDataBases(bool computeRequiredBy)
}
}
/*!
* \brief Returns whether automatic cache maintenance is enabled.
* \remarks If enabled, the cache will be cleaned and saved frequently.
*/
bool Manager::isAutoCacheMaintenanceEnabled() const
{
return m_cacheTimer && m_cacheTimer->isActive();
}
/*!
* \brief Sets whether automatic cache maintenacne is enabled.
* \sa isAutoCacheMaintenanceEnabled()
*/
void Manager::setAutoCacheMaintenanceEnabled(bool enabled)
{
if(isAutoCacheMaintenanceEnabled() != enabled) {
if(enabled) {
if(!m_cacheTimer) {
m_cacheTimer = make_unique<QTimer>();
m_cacheTimer->setInterval(5 * 60 * 1000);
QObject::connect(m_cacheTimer.get(), &QTimer::timeout, bind(&Manager::maintainCache, this));
}
m_cacheTimer->start();
} else {
m_cacheTimer->stop();
}
}
}
/*!
* \brief Writes the cache for all repositories where caching makes sense.
*/
@ -543,6 +573,29 @@ void Manager::restoreCache()
}
}
void Manager::cleanCache()
{
// currently clear only AUR cache
if(userRepository()) {
userRepository()->cleanOutdatedPackages();
}
}
void Manager::wipeCache()
{
// currently wipe only AUR cache
if(userRepository()) {
userRepository()->wipePackages();
}
}
void Manager::maintainCache()
{
cleanCache();
// TODO: write cache only when modified
writeCache();
}
/*!
* \brief Unregisters all registred sync databases.
*/

14
alpm/manager.h
View File

@ -14,6 +14,8 @@
#include <map>
#include <memory>
QT_FORWARD_DECLARE_CLASS(QTimer)
namespace RepoIndex {
class Config;
@ -38,8 +40,6 @@ public:
// configuration, signature level, etc
const Config &config() const;
bool writeCacheBeforeGone() const;
void setWriteCacheBeforeGone(bool writeCacheBeforeGone);
int sigLevel() const;
void setSigLevel(int sigLevel);
int localFileSigLevel() const;
@ -52,8 +52,17 @@ public:
void registerDataBasesFromPacmanConfig();
void registerDatabasesFromRepoIndexConfig();
void initAlpmDataBases(bool computeRequiredBy);
// caching
bool writeCacheBeforeGone() const;
void setWriteCacheBeforeGone(bool writeCacheBeforeGone);
bool isAutoCacheMaintenanceEnabled() const;
void setAutoCacheMaintenanceEnabled(bool enabled);
void writeCache();
void restoreCache();
void cleanCache();
void wipeCache();
void maintainCache();
void unregisterSyncDataBases();
@ -89,6 +98,7 @@ public:
private:
const Config &m_config;
bool m_writeCacheBeforeGone;
std::unique_ptr<QTimer> m_cacheTimer;
alpm_handle_t *m_handle;
int m_sigLevel;
int m_localFileSigLevel;

1
alpm/package.h
View File

@ -189,6 +189,7 @@ public:
// caching
void writeToCacheStream(QDataStream &out);
void restoreFromCacheStream(QDataStream &in);
void reinitEntries();
// parsing src/pkg info
void putInfo(const QList<QPair<QString, QString> > &baseInfo, const QList<QPair<QString, QString> > &pkgInfo, bool includesSourceRelatedMetaData = false);

45
alpm/repository.cpp
View File

@ -12,6 +12,7 @@
#include <cassert>
using namespace std;
using namespace ChronoUtilities;
namespace RepoIndex {
@ -70,6 +71,19 @@ const QStringList RepoIndex::Repository::packageNames() const
return names;
}
/*!
* \brief Initializes the repository.
* \remarks
* - Does not restore cache. For restoring cache see restoreFromCacheStream().
* - Performs asynchronously and hence returns immidiately. Returns a PackageLoader
* object which QFuture can be used to wait for initializing to finish.
* - Might return nullptr if initialization is tivial.
*/
PackageLoader *Repository::init()
{
return nullptr;
}
/*!
* \brief Requests suggestions for the specified search phrase.
* \returns Returns a reply object used for the request. The reply must be destroyed by the caller
@ -92,6 +106,7 @@ Repository::Repository(const QString &name, uint32 index, QObject *parent) :
QObject(parent),
m_index(index),
m_name(name),
m_maxPackageAge(TimeSpan::infinity()),
m_usage(static_cast<alpm_db_usage_t>(0)),
m_sigLevel(static_cast<alpm_siglevel_t>(ALPM_SIGSTATUS_INVALID)),
m_lock(QReadWriteLock::Recursive)
@ -431,7 +446,7 @@ void Repository::writeToCacheStream(QDataStream &out)
/*!
* \brief Restores the repository information from cache.
*/
void Repository::restoreFromCacheStream(QDataStream &in)
void Repository::restoreFromCacheStream(QDataStream &in, bool skipOutdated)
{
quint32 magic;
in >> magic;
@ -447,11 +462,16 @@ void Repository::restoreFromCacheStream(QDataStream &in)
quint32 packageCount;
in >> packageCount;
bool good = true;
const auto now = DateTime::now();
for(quint32 i = 0; i < packageCount && good && in.status() == QDataStream::Ok; ++i) {
if(auto package = emptyPackage()) {
package->restoreFromCacheStream(in);
if(!package->name().isEmpty()) {
m_packages[package->name()] = move(package);
if(!skipOutdated || !((now - package->timeStamp()) > maxPackageAge())) {
m_packages[package->name()] = move(package);
} else {
cerr << shchar << "Info: Cache entry for package \"" << package->name().toLocal8Bit().data() << "\" is outdated and won't be restored." << endl;
}
} else {
good = false;
}
@ -510,6 +530,26 @@ void Repository::restoreSpecificCacheHeader(QDataStream &in)
Q_UNUSED(in)
}
/*!
* \brief Cleans the repository from outdated packages.
* \remarks Does nothing if maxPackageAge() is infinity (which is the default).
*/
void Repository::cleanOutdatedPackages()
{
if(maxPackageAge().isInfinity()) {
return;
}
auto now = DateTime::now();
for(auto i = m_packages.begin(); i != m_packages.end(); ) {
const Package &pkg = *i->second;
if((now - pkg.timeStamp()) > maxPackageAge()) {
i = m_packages.erase(i);
} else {
++i;
}
}
}
/*!
* \brief Adds packages parsed from the specified \a srcInfo.
* \returns Returns the added packages.
@ -632,5 +672,4 @@ QList<Package *> Repository::addPackagesFromSrcInfo(const QByteArray &srcInfo)
return packages;
}
} // namespace PackageManagement

46
alpm/repository.h
View File

@ -112,6 +112,29 @@ inline const Repository *SuggestionsReply::repository() const
return m_repo;
}
class PackageLoader
{
public:
QFuture<void> &future();
virtual ~PackageLoader();
protected:
PackageLoader();
QMutex m_mutex;
QFuture<void> m_future;
};
inline PackageLoader::PackageLoader()
{}
inline PackageLoader::~PackageLoader()
{}
inline QFuture<void> &PackageLoader::future()
{
return m_future;
}
/*!
* \brief The RepositoryType enum specifies the type of a repository object.
*/
@ -170,6 +193,7 @@ public:
alpm_siglevel_t sigLevel() const;
// gathering data
virtual PackageLoader *init();
virtual PackageDetailAvailability requestsRequired(PackageDetail packageDetail = PackageDetail::Basics) const;
virtual SuggestionsReply *requestSuggestions(const QString &phrase);
virtual PackageReply *requestPackageInfo(const QStringList &packageNames, bool forceUpdate = false);
@ -207,10 +231,14 @@ public:
// caching
bool isCachingUseful() const;
void writeToCacheStream(QDataStream &out);
void restoreFromCacheStream(QDataStream &in);
void restoreFromCacheStream(QDataStream &in, bool skipOutdated = true);
virtual void writeSpecificCacheHeader(QDataStream &out);
virtual std::unique_ptr<Package> emptyPackage();
virtual void restoreSpecificCacheHeader(QDataStream &in);
ChronoUtilities::TimeSpan maxPackageAge() const;
void setMaxPackageAge(ChronoUtilities::TimeSpan maxPackageAge);
void cleanOutdatedPackages();
void wipePackages();
// parsing src/pkg info
QList<Package *> addPackagesFromSrcInfo(const QByteArray &srcInfo);
@ -228,6 +256,7 @@ protected:
QString m_name;
QString m_description;
std::map<QString, std::unique_ptr<Package> > m_packages;
ChronoUtilities::TimeSpan m_maxPackageAge;
alpm_db_usage_t m_usage;
std::map<QString, QList<Package *> > m_groups;
QStringList m_serverUrls;
@ -424,6 +453,21 @@ inline bool Repository::isCachingUseful() const
}
}
inline ChronoUtilities::TimeSpan Repository::maxPackageAge() const
{
return m_maxPackageAge;
}
inline void Repository::setMaxPackageAge(ChronoUtilities::TimeSpan maxPackageAge)
{
m_maxPackageAge = maxPackageAge;
}
inline void Repository::wipePackages()
{
m_packages.clear();
}
inline QReadWriteLock *Repository::lock() const
{
return const_cast<QReadWriteLock *>(&m_lock);

3
main.cpp
View File

@ -73,13 +73,12 @@ int main(int argc, char *argv[])
if(configArgs.serverArg.isPresent()) {
// setup the server
Server server(manager, manager.config());
manager.setAutoCacheMaintenanceEnabled(true);
QObject::connect(&server, &Server::closed, &application, &QCoreApplication::quit);
// run Qt loop
return application.exec();
} else if(configArgs.buildOrderArg.isPresent()) {
BuildOrderResolverCli resolver(manager, configArgs.buildOrderArg.values(), configArgs.addSourceOnlyDeps.isPresent());
return resolver.exec();
//BuildOrderResolver::printResults(resolver.resolve(configArgs.buildOrderArg.values()));
} else if(configArgs.mingwBundleArg.isPresent()) {
MingwBundle bundle(manager, configArgs.mingwBundleArg.values(), configArgs.iconThemesArg.values(), configArgs.extraPackagesArg.values());
bundle.createBundle(configArgs.targetDirArg.isPresent() ? configArgs.targetDirArg.values().front() : string("."),

2
network/userrepository.cpp
View File

@ -21,6 +21,7 @@
#include <QStringBuilder>
using namespace std;
using namespace ChronoUtilities;
namespace RepoIndex {
@ -190,6 +191,7 @@ UserRepository::UserRepository(QNetworkAccessManager &networkAccessManager, QObj
m_networkAccessManager(networkAccessManager)
{
m_description = QStringLiteral("Arch User Repository");
m_maxPackageAge = TimeSpan::fromDays(1.0);
}
RepositoryType UserRepository::type() const