#include "./repository.h" #include "./upgradelookup.h" #include "./utilities.h" #include "./config.h" #include #include #include #include #include #include using namespace std; using namespace ChronoUtilities; namespace RepoIndex { /*! * \brief Constructs a new reply for a single network reply. */ Reply::Reply(QNetworkReply *networkReply) : m_remainingReplies(1) { networkReply->setParent(this); connect(networkReply, &QNetworkReply::finished, this, &Reply::replyFinished); m_networkReplies.reserve(1); m_networkReplies << networkReply; } /*! * \brief Constructs a new reply for multiple network replies. */ Reply::Reply(const QList networkReplies) : m_networkReplies(networkReplies), m_remainingReplies(networkReplies.size()) { for(auto *networkReply : networkReplies) { networkReply->setParent(this); connect(networkReply, &QNetworkReply::finished, this, &Reply::replyFinished); } } /*! * \brief Called when a network reply has finished. */ void Reply::replyFinished() { #ifdef DEBUG_BUILD assert(m_remainingReplies); #endif processData(static_cast(sender())); if(!--m_remainingReplies) { emit resultsAvailable(); } } /*! * \fn Repository::type() * \brief Returns the type of the package source. */ /*! * \brief Returns a list of all package names. */ const QStringList Repository::packageNames() const { QStringList names; names.reserve(m_packages.size()); for(const auto &entry : m_packages) { names << entry.first; } return names; } void Repository::updateGroups() { m_groups.clear(); for(auto &entry : m_packages) { for(const QString &group : entry.second->groups()) { m_groups[group] << entry.second.get(); } } } /*! * \brief Initializes the repository. * \remarks * - The repository mustn't be busy if this method is called. * - Does not restore cache. For restoring cache see restoreFromCacheStream(). * - Performs asynchronously and hence returns immidiately. Returns a PackageLoader * object which QFuture can be used to wait until the initialization is finished. * - Alternatively the available() and initialized() signals can be used. * - Might return nullptr if initialization is tivial. In this case the available * and initialized() signals are not emitted. * - The returned future might be not running indicating the process * has already finished. In this case the available and initialized() signals are not emitted. * - Locks the repository for write access. Flags the repository as busy. */ PackageLoader *Repository::init() { addBusyFlag(); QWriteLocker locker(lock()); if(PackageLoader *loader = internalInit()) { if(loader->future().isRunning()) { auto watcher = new QFutureWatcher; connect(watcher, &QFutureWatcher::finished, this, &Repository::removeBusyFlag); connect(watcher, &QFutureWatcher::finished, this, &Repository::initialized); connect(watcher, &QFutureWatcher::finished, watcher, &QFutureWatcher::deleteLater); watcher->setFuture(loader->future()); } return loader; } else { return nullptr; } } void Repository::initAsSoonAsPossible() { if(isBusy()) { auto connection = make_shared(); *connection = connect(this, &Repository::available, [connection, this] { disconnect(*connection); init(); }); } else { init(); } } /*! * \brief This method can must overriden when subclassing to initialize the repository. * \remarks * - Mustn't emit any signals. * - The repository is already locked when this method is called. Hence mustn't lock the repository. * \sa init() */ PackageLoader *Repository::internalInit() { return nullptr; } /*! * \brief Requests suggestions for the specified search phrase. * \returns Returns a reply object used for the request. The reply must be destroyed by the caller * using destroyLater() after resultsAvailable() has been emitted. */ SuggestionsReply *Repository::requestSuggestions(const QString &) { return nullptr; } /*! * \class Repository * \brief The Repository class represents a repository (binary repositories as well as source-only repos). */ /*! * \brief Constructs a new repository (protected since this is a pure virtual class). */ Repository::Repository(const QString &name, uint32 index, QObject *parent) : QObject(parent), m_index(index), m_name(name), m_maxPackageAge(TimeSpan::infinity()), m_usage(RepositoryUsage::None), m_sigLevel(SignatureLevel::UseDefault), m_lock(QReadWriteLock::Recursive) {} /*! * \brief Destroys the repository. */ Repository::~Repository() {} /*! * \brief Returns whether explicit requests are required to get the specified information * about the package of this repository. * * AlpmDataBase instances load all available packages in the cache * at the beginning and hence do not require explicit requests for package names, version, * description, dependencies and most other information. However make dependencies are not available at all. * * UserRepository instances on the other hand have an empty package * cache at the beginning so packages must be requested explicitely * using the requestPackageInfo() method. */ PackageDetailAvailability Repository::requestsRequired(PackageDetail ) const { return PackageDetailAvailability::Never; } /*! * \brief Requests package information for the specified package. * \returns Returns a reply object used for the request. The reply must be destroyed by the caller * using destroyLater() after resultsAvailable() has been emitted. * \remarks * If \a forceUpdate is true, package information which has already been retrieved * and is still cached is requested again. Otherwise these packages will not be * requested again. If it turns out, that all packages are already cached, nullptr * is returned in this case. */ PackageReply *Repository::requestPackageInfo(const QStringList &, bool ) { return nullptr; } /*! * \brief Requests full package information for the specified package. * \returns Returns a reply object used for the request. The reply must be destroyed by the caller * using destroyLater() after resultsAvailable() has been emitted. * \remarks * If \a forceUpdate is true, package information which has already been retrieved * and is still cached is requested again. Otherwise these packages will not be * requested again. If it turns out, that all packages are already cached, nullptr * is returned in this case. */ PackageReply *Repository::requestFullPackageInfo(const QStringList &, bool ) { return nullptr; } /*! * \brief Returns the first package providing the specified \a dependency. * \remarks Returns nullptr if no packages provides the \a dependency. */ const Package *Repository::packageProviding(const Dependency &dependency) const { for(const auto &entry : m_packages) { if(entry.second->matches(dependency)) { // check whether package matches "directly" return entry.second.get(); } } return nullptr; } /*! * \brief Returns the first package providing the specified \a dependency. * \remarks Returns nullptr if no packages provides the \a dependency. */ Package *Repository::packageProviding(const Dependency &dependency) { for(auto &entry : m_packages) { if(entry.second->matches(dependency)) { // check whether package matches "directly" return entry.second.get(); } } return nullptr; } /*! * \brief Returns all packages providing the specified \a dependency. */ QList Repository::packagesProviding(const Dependency &dependency) const { QList res; for(const auto &entry : m_packages) { if(entry.second->matches(dependency)) { res << entry.second.get(); } } return res; } /*! * \brief Returns all packages matching the specified predicate. */ QList Repository::packageByFilter(std::function pred) { QList packages; for(const auto &entry : m_packages) { if(pred(entry.second.get())) { packages << entry.second.get(); } } return packages; } /*! * \cond */ class Blocker { public: Blocker(const QList &relevantRepos) { m_blockedRepos.reserve(relevantRepos.size()); for(Repository *repo : relevantRepos) { if(repo->lock()->tryLockForWrite()) { m_blockedRepos << repo; } } } ~Blocker() { for(Repository *repo : m_blockedRepos) { repo->lock()->unlock(); } } private: QList m_blockedRepos; }; class ComputeRequired { public: ComputeRequired(const QList &relevantRepos, bool forceUpdate) : m_relevantRepos(relevantRepos), m_forceUpdate(forceUpdate) {} void operator () (const pair > &packageEntry) { if(m_forceUpdate || !packageEntry.second->isRequiredByComputed()) { packageEntry.second->computeRequiredBy(m_relevantRepos); } } private: const QList m_relevantRepos; bool m_forceUpdate; }; /*! * \endcond */ /*! * \brief Computes required-by and optional-for for all packages. * * Sources the packages of all \a relevantRepositories for packages depending on the packages of this repository. * * \remarks * - Computation is done async. * - The repository mustn't be busy. Flags the repository as busy. * - \a relevantRepositories might contain the current instance. * - The available() and requiredByComputed() signals are emitted after computition has finished. */ QFuture Repository::computeRequiredBy(const QList &relevantRepositories, bool forceUpdate) { addBusyFlag(); // flag repository as busy auto *watcher = new QFutureWatcher; connect(watcher, &QFutureWatcher::finished, this, &Repository::removeBusyFlag); connect(watcher, &QFutureWatcher::finished, this, &Repository::requiredByComputed); connect(watcher, &QFutureWatcher::finished, watcher, &QFutureWatcher::deleteLater); watcher->setFuture(QtConcurrent::map(m_packages, ComputeRequired(relevantRepositories, forceUpdate))); return watcher->future(); } /*! * \brief Returns suggestions for the specified \a term. */ QJsonObject Repository::suggestions(const QString &term) const { QJsonArray suggestions; // size_t remainingSuggestions = 20; // for(auto i = packages().lower_bound(term), end = packages().cend(); i != end && remainingSuggestions; ++i, --remainingSuggestions) { // if(i->first.startsWith(term, Qt::CaseInsensitive)) { // suggestions << i->first; // } else { // break; // } // } for(const auto &pkgEntry : packages()) { if(pkgEntry.first.contains(term, Qt::CaseInsensitive)) { suggestions << pkgEntry.first; } } QJsonObject res; res.insert(QStringLiteral("repo"), name()); res.insert(QStringLiteral("res"), suggestions); return res; } QJsonArray Repository::upgradeSourcesJsonArray() const { QJsonArray sources; for(const auto *source : upgradeSources()) { sources << source->name(); } return sources; } void Repository::checkForUpgrades(UpgradeLookupResults &results, const QList &upgradeSources) const { if(upgradeSources.isEmpty()) { results.noSources = true; } else { for(const auto &pkgEntry : packages()) { bool orphaned = true; for(const auto *src : upgradeSources) { if(const auto *syncPkg = src->packageByName(pkgEntry.first)) { switch(pkgEntry.second->compareVersion(syncPkg)) { case PackageVersionComparsion::Equal: break; // ignore equal packages case PackageVersionComparsion::SoftwareUpgrade: results.newVersions << UpgradeResult(syncPkg, pkgEntry.second->version()); break; case PackageVersionComparsion::PackageUpgradeOnly: results.newReleases << UpgradeResult(syncPkg, pkgEntry.second->version()); break; case PackageVersionComparsion::NewerThenSyncVersion: results.downgrades << UpgradeResult(syncPkg, pkgEntry.second->version()); } orphaned = false; } } if(orphaned) { results.orphaned << pkgEntry.second.get(); } } } } /*! * \brief Returns all package names as JSON array. */ QJsonArray Repository::packageNamesJsonArray() const { QJsonArray names; for(const auto &entry : m_packages) { names << entry.first; } return names; } /*! * \brief Returns an object with the package names of the repository as keys (and empty objects as value). */ QJsonObject Repository::packagesObjectSkeleton() const { QJsonObject skel; for(const auto &entry : m_packages) { skel.insert(entry.first, QJsonValue(QJsonValue::Object)); } return skel; } /*! * \cond */ inline void putString(QJsonObject &obj, const QString &key, const QJsonValue &value) { if(!value.isNull()) { obj.insert(key, value); } } inline void putString(QJsonObject &obj, const QString &key, const QStringList &values) { if(!values.isEmpty()) { putString(obj, key, QJsonArray::fromStringList(values)); } } /*! * \endcond */ /*! * \brief Returns basic information about the repository. */ QJsonObject Repository::basicInfo(bool includeName) const { QJsonObject info; if(includeName) { putString(info, QStringLiteral("name"), name()); } if(index() != invalidIndex) { info.insert(QStringLiteral("index"), static_cast(index())); } putString(info, QStringLiteral("desc"), description()); putString(info, QStringLiteral("servers"), serverUrls()); putString(info, QStringLiteral("usage"), Utilities::usageStrings(usage())); putString(info, QStringLiteral("sigLevel"), Utilities::sigLevelStrings(sigLevel())); putString(info, QStringLiteral("upgradeSources"), upgradeSourcesJsonArray()); putString(info, QStringLiteral("packages"), packagesObjectSkeleton()); if(requestsRequired(PackageDetail::Basics) == PackageDetailAvailability::Immediately) { info.insert(QStringLiteral("packageCount"), static_cast(m_packages.size())); } putString(info, QStringLiteral("srcOnly"), isSourceOnly()); putString(info, QStringLiteral("pkgOnly"), isPackageOnly()); return info; } /*! * \brief Returns group information as JSON object. */ QJsonObject Repository::groupInfo() const { QJsonObject info; putString(info, QStringLiteral("repo"), name()); QJsonArray groupsArray; for(const auto &groupEntry : groups()) { QJsonObject info; putString(info, QStringLiteral("name"), groupEntry.first); QJsonArray pkgNames; for(const auto *pkg : groupEntry.second) { pkgNames << pkg->name(); } putString(info, QStringLiteral("pkgs"), pkgNames); groupsArray << info; } info.insert(QStringLiteral("groups"), groupsArray); return info; } /*! * \brief Writes the repository information to the specified cache stream. */ void Repository::writeToCacheStream(QDataStream &out) { out << static_cast(0x7265706F); // magic number out << static_cast(0x0); // version out << static_cast(type()); out << static_cast(m_packages.size()); for(const auto &pkg : m_packages) { pkg.second->writeToCacheStream(out); } // write specific header auto headerStart = out.device()->pos(); out.device()->seek(headerStart + 4); writeSpecificCacheHeader(out); auto headerEnd = out.device()->pos(); out.device()->seek(headerStart); out << static_cast(headerEnd - headerStart - 4); out.device()->seek(headerEnd); // no extended header out << static_cast(0x0); } /*! * \brief Restores the repository information from cache. */ void Repository::restoreFromCacheStream(QDataStream &in, bool skipOutdated) { quint32 magic; in >> magic; if(magic == 0x7265706F) { // read version quint32 version; in >> version; // read type quint32 denotedType; in >> denotedType; if(denotedType == static_cast(type())) { // read packages quint32 packageCount; in >> packageCount; bool good = true; const auto now = DateTime::now(); for(quint32 i = 0; i < packageCount && good && in.status() == QDataStream::Ok; ++i) { if(auto package = emptyPackage()) { package->restoreFromCacheStream(in); if(!package->name().isEmpty()) { if(!skipOutdated || !((now - package->timeStamp()) > maxPackageAge())) { m_packages[package->name()] = move(package); } else { cerr << shchar << "Info: Cache entry for package \"" << package->name().toLocal8Bit().data() << "\" is outdated and won't be restored." << endl; } } else { good = false; } } else { good = false; } } if(in.status() == QDataStream::Ok) { // specific header quint32 headerSize; in >> headerSize; quint64 headerEnd = in.device()->pos() + headerSize; restoreSpecificCacheHeader(in); in.device()->seek(headerEnd); if(in.status() == QDataStream::Ok) { // skip extended header in >> headerSize; quint64 headerEnd = in.device()->pos() + headerSize; in.device()->seek(headerEnd); } else { cerr << shchar << "Failed to restore cache for repository \"" << m_name.toLocal8Bit().data() << "\": unable to parse specific cache header" << endl; } } else { cerr << shchar << "Failed to restore cache for repository \"" << m_name.toLocal8Bit().data() << "\": unable to parse packages" << endl; } } else { cerr << shchar << "Failed to restore cache for repository \"" << m_name.toLocal8Bit().data() << "\": denoted type does not match expected type" << endl; } } else { cerr << shchar << "Failed to restore cache for repository \"" << m_name.toLocal8Bit().data() << "\": bad magic number" << endl; } } /*! * \brief Writes the repo-type-specific cache header. */ void Repository::writeSpecificCacheHeader(QDataStream &out) { Q_UNUSED(out) } /*! * \brief Returns an new, empty package. * \remarks Used to when restoring packages from cache. */ unique_ptr Repository::emptyPackage() { return unique_ptr(); } /*! * \brief Restores the repo-type-specific cache header. */ void Repository::restoreSpecificCacheHeader(QDataStream &in) { Q_UNUSED(in) } /*! * \brief Cleans the repository from outdated packages. * \remarks Does nothing if maxPackageAge() is infinity (which is the default). */ void Repository::cleanOutdatedPackages() { if(maxPackageAge().isInfinity()) { return; } auto now = DateTime::now(); for(auto i = m_packages.begin(); i != m_packages.end(); ) { const Package &pkg = *i->second; if((now - pkg.timeStamp()) > maxPackageAge()) { i = m_packages.erase(i); } else { ++i; } } } /*! * \brief Returns whether the repository has outdated packages. * \sa cleanOutdatedPackages() */ bool Repository::hasOutdatedPackages() { if(maxPackageAge().isInfinity()) { return false; } auto now = DateTime::now(); for(auto i = m_packages.begin(); i != m_packages.end(); ) { const Package &pkg = *i->second; if((now - pkg.timeStamp()) > maxPackageAge()) { return true; } } return false; } /*! * \brief Parses the specified .PKGINFO file. */ void Repository::parsePkgInfo(const QByteArray &pkgInfo, QString &name, QList > packageInfo) { // define states enum { FieldName, // reading field name (initial state) EquationSign, // expecting equation sign Pad, // expecting padding FieldValue, // reading field value Comment // reading comment } state = FieldName; // define variables to store parsing results QByteArray currentFieldName; currentFieldName.reserve(16); QByteArray currentFieldValue; currentFieldValue.reserve(32); packageInfo.reserve(16); // state machine: consumes each char of .SRCINFO for(const char c : pkgInfo) { switch(state) { case FieldName: switch(c) { case '#': // discard truncated line currentFieldName.clear(); state = Comment; case ' ': // field name complete, expect equation sign if(!currentFieldName.isEmpty()) { state = EquationSign; } break; case '\n': case '\r': case '\t': // discard truncated line currentFieldName.clear(); break; default: currentFieldName.append(c); } break; case EquationSign: switch(c) { case '=': state = Pad; break; case '\n': case '\r': case '\t': // unexpected new line -> discard truncated line currentFieldName.clear(); break; default: ; // ignore unexpected characters } break; case Pad: switch(c) { case ' ': state = FieldValue; break; case '\n': case '\r': case '\t': // unexpected new line -> discard truncated line currentFieldName.clear(); break; default: ; // ignore unexpected characters } break; case FieldValue: switch(c) { case '\n': case '\r': state = FieldName; if(!currentFieldValue.isEmpty() && "pkgname" == currentFieldName) { // put current info to current package name = currentFieldValue; } packageInfo << QPair(currentFieldName, currentFieldValue); currentFieldName.clear(); currentFieldValue.clear(); break; default: currentFieldValue.append(c); } break; case Comment: switch(c) { case '\n': case '\r': case '\t': state = FieldName; break; default: ; // ignore outcommented characters } break; } } } /*! * \brief Parses the specified package \a descriptions (desc/depends/files file). * * Stores the results in \a fields. The package name is also stored in \a name. */ void Repository::parseDescriptions(const QList &descriptions, QString &name, QList > &fields) { // define variables to store parsing results fields.reserve(32); QByteArray currentFieldName; currentFieldName.reserve(16); QByteArray currentFieldValue; currentFieldValue.reserve(16); QStringList currentFieldValues; for(const QByteArray &description : descriptions) { // define states enum { FieldName, // reading field name NewLine, // expecting new line (after field name) Next, // start reading next field value / next field name (initial state) FieldValue, // reading field value } state = Next; // state machine: consumes each char of desc for(const char c : description) { switch(state) { case FieldName: switch(c) { case '%': state = NewLine; break; default: currentFieldName.append(c); } break; case NewLine: switch(c) { case '\n': case '\r': state = Next; break; default: ; // ignore unexpected characters } break; case Next: switch(c) { case '\n': case '\r': case '\t': case ' ': break; case '%': state = FieldName; // next field -> put current field if(!currentFieldName.isEmpty()) { fields << QPair(currentFieldName, currentFieldValues); currentFieldName.clear(); currentFieldValues.clear(); } break; default: state = FieldValue; currentFieldValue.append(c); } break; case FieldValue: switch(c) { case '\n': case '\r': state = Next; currentFieldValues << currentFieldValue; if(!currentFieldValue.isEmpty() && "NAME" == currentFieldName) { name = currentFieldValues.back(); } currentFieldValue.clear(); break; default: currentFieldValue.append(c); } } } // all characters read switch(state) { case FieldValue: currentFieldValues << currentFieldValue; if(!currentFieldValue.isEmpty() && "NAME" == currentFieldName) { name = currentFieldValues.back(); } default: ; } // put last field if(!currentFieldName.isEmpty()) { fields << QPair(currentFieldName, currentFieldValues); currentFieldName.clear(); currentFieldValues.clear(); } } } /*! * \brief Adds packages parsed from the specified .SRCINFO file. * \remarks Updates existing packages. * \returns Returns the added/updated packages. In the case of a split package more then * one package is returned. */ QList Repository::addPackagesFromSrcInfo(const QByteArray &srcInfo) { // define states enum { FieldName, // reading field name (initial state) EquationSign, // expecting equation sign Pad, // expecting padding FieldValue, // reading field value Comment // reading comment } state = FieldName; // define variables to store parsing results QByteArray currentFieldName; currentFieldName.reserve(16); QByteArray currentFieldValue; currentFieldValue.reserve(32); QString packageBase; packageBase.reserve(32); QList > baseInfo; baseInfo.reserve(16); QList > packageInfo; packageInfo.reserve(16); QList packages; Package *currentPackage = nullptr; // state machine: consumes each char of .SRCINFO for(const char c : srcInfo) { switch(state) { case FieldName: switch(c) { case '#': // discard truncated line currentFieldName.clear(); state = Comment; case ' ': // field name complete, expect equation sign if(!currentFieldName.isEmpty()) { state = EquationSign; } break; case '\n': case '\r': case '\t': // discard truncated line currentFieldName.clear(); break; default: currentFieldName.append(c); } break; case EquationSign: switch(c) { case '=': state = Pad; break; case '\n': case '\r': case '\t': // unexpected new line -> discard truncated line currentFieldName.clear(); break; default: ; // ignore unexpected characters } break; case Pad: switch(c) { case ' ': state = FieldValue; break; case '\n': case '\r': case '\t': // unexpected new line -> discard truncated line currentFieldName.clear(); break; default: ; // ignore unexpected characters } break; case FieldValue: switch(c) { case '\n': case '\r': state = FieldName; if("pkgbase" == currentFieldName) { // pkgbase packageBase = currentFieldValue; } else if("pkgname" == currentFieldName) { // next package if(packageBase.isEmpty()) { // no pkgbase specified -> use the first pkgname as pkgbase packageBase = currentFieldName; } // put current info to current package if(currentPackage) { currentPackage->putInfo(baseInfo, packageInfo, true); // TODO: add groups packages << currentPackage; } // find next package auto &pkg = m_packages[currentFieldValue]; if(!pkg) { pkg = emptyPackage(); } currentPackage = pkg.get(); packageInfo.clear(); } // add field to ... if(currentPackage) { // ... concrete package info if there's already a concrete package packageInfo << QPair(currentFieldName, currentFieldValue); } else { // ... base info if still parsing general info baseInfo << QPair(currentFieldName, currentFieldValue); } currentFieldName.clear(); currentFieldValue.clear(); break; default: currentFieldValue.append(c); } break; case Comment: switch(c) { case '\n': case '\r': case '\t': state = FieldName; break; default: ; // ignore outcommented characters } break; } } if(currentPackage) { currentPackage->putInfo(baseInfo, packageInfo, true); packages << currentPackage; } return packages; } /*! * \brief Adds packages parsed from the specified desc/depends/files file. * \remarks * - Updates the package if it already exists. * - If \a name is empty and the description doesn't provide a name either, the package can not be added. * \returns Returns the added/updated package or nullptr if no package could be added. */ Package *Repository::addPackageFromDescription(QString name, const QList &descriptions, PackageOrigin origin) { // parse fields QList > fields; parseDescriptions(descriptions, name, fields); // check whether name is empty if(name.isEmpty()) { return nullptr; } // find/create package for description auto pkg = emptyPackage(); Package *pkgRawPtr = pkg.get(); pkgRawPtr->putDescription(name, fields, origin); { QWriteLocker locker(&m_lock); m_packages[name] = move(pkg); } return pkgRawPtr; } /*! * \brief Internally called to add the busy flag. */ void Repository::addBusyFlag() { m_isBusy.store(1); } /*! * \brief Internally called to remove the busy flag. */ void Repository::removeBusyFlag() { m_isBusy.store(0); emit available(); } } // namespace PackageManagement