repoindex/lib/alpm/alpmdatabase.cpp

#include "./alpmdatabase.h"
#include "./upgradelookup.h"
#include "./alpmpackage.h"
#include "./utilities.h"
#include "./config.h"

#include "../network/networkaccessmanager.h"

#include <KTar>
#include <KArchiveDirectory>

#include <QList>
#include <QJsonObject>
#include <QStringBuilder>
#include <QtConcurrent>
#include <QNetworkAccessManager>
#include <QNetworkRequest>
#include <QNetworkReply>

#include <memory>
#include <iostream>

using namespace std;
using namespace CppUtilities;

namespace RepoIndex {

using namespace Utilities;

/*!
 * \class AlpmDatabase
 * \brief The AlpmDatabase class wraps an ALPM data base struct and holds additional meta information.
 *
 * All packages returned by the AlpmDatabase class are AlpmPackage instances.
 */

class LoadPackage
{
public:
    LoadPackage(AlpmDatabase *database, PackageOrigin origin, DateTime descriptionsLastModified) :
        m_db(database),
        m_origin(origin),
        m_descriptionsLastModified(descriptionsLastModified)
    {}

    void operator()(const QPair<QString, QList<QByteArray> > &description)
    {
        m_db->addPackageFromDescription(description.first, description.second, m_origin, m_descriptionsLastModified);
    }

private:
    AlpmDatabase *const m_db;
    const PackageOrigin m_origin;
    const DateTime m_descriptionsLastModified;
};

DatabaseError AlpmDatabase::loadDescriptions(QList<QPair<QString, QList<QByteArray> > > &descriptions, CppUtilities::DateTime *lastModified)
{
    if(!databasePath().isEmpty()) {
        QFileInfo pathInfo(databasePath());
        if(pathInfo.isDir()) {
            if(lastModified) {
                // just use current date here since this is usually the local db
                *lastModified = DateTime::gmtNow();
            }
            static const QStringList relevantFiles = QStringList() << QStringLiteral("desc") << QStringLiteral("files");
            QDir dbDir(databasePath());
            QStringList pkgDirNames = dbDir.entryList(QDir::Dirs | QDir::Readable | QDir::Executable | QDir::NoDotAndDotDot);
            descriptions.reserve(pkgDirNames.size());
            for(QString &pkgDirName : pkgDirNames) {
                if(dbDir.cd(pkgDirName)) {
                    Utilities::stripVersion(pkgDirName);
                    const QStringList descFileNames = dbDir.entryList(relevantFiles, QDir::Files | QDir::Readable | QDir::NoDotAndDotDot);
                    QList<QByteArray> descData;
                    descData.reserve(descFileNames.size());
                    for(const QString &descFileName : descFileNames) {
                        QFile descFile(dbDir.absoluteFilePath(descFileName));
                        if(descFile.open(QFile::ReadOnly)) {
                            descData << descFile.readAll();
                        } else {
                            return DatabaseError::UnableToOpenDescFile;
                        }
                    }
                    if(!descData.isEmpty()) {
                        descriptions << qMakePair(pkgDirName, descData);
                    }
                    dbDir.cdUp();
                } else {
                    return DatabaseError::UnableToEnterDirectory;
                }
            }
        } else if(pathInfo.isFile()) {
            if(lastModified) {
                *lastModified = DateTime::fromTimeStampGmt(pathInfo.lastModified().toUTC().toSecsSinceEpoch());
            }
            KTar tar(databasePath());
            const KArchiveDirectory *dbDir;
            if(tar.open(QIODevice::ReadOnly) && (dbDir = tar.directory())) {
                QStringList pkgDirNames = dbDir->entries();
                descriptions.reserve(pkgDirNames.size());
                for(QString &pkgDirName : pkgDirNames) {
                    if(const auto *pkgEntry = dbDir->entry(pkgDirName)) {
                        if(pkgEntry->isDirectory()) {
                            Utilities::stripVersion(pkgDirName);
                            const auto *pkgDir = static_cast<const KArchiveDirectory *>(pkgEntry);
                            const QStringList descFileNames = pkgDir->entries();
                            QList<QByteArray> descData;
                            descData.reserve(descFileNames.size());
                            for(const QString &descFileName : descFileNames) {
                                if(const auto *descEntry = pkgDir->entry(descFileName)) {
                                    if(descEntry->isFile()) {
                                        descData << static_cast<const KArchiveFile *>(descEntry)->data();
                                    } else {
                                        // there shouldn't be any subdirs anyways
                                    }
                                }
                            }
                            if(!descData.isEmpty()) {
                                descriptions << qMakePair(pkgDirName, descData);
                            }
                        } else {
                            // there shouldn't be any files anyways
                        }
                    }
                }
            } else {
                return DatabaseError::UnableToOpenArchive;
            }
        } else {
            return DatabaseError::NotFound;
        }
        return DatabaseError::NoError;
    } else {
        return DatabaseError::NotFound;
    }
}

AlpmPackageLoader::AlpmPackageLoader(AlpmDatabase *repository, PackageOrigin origin) :
    m_db(repository)
{
    if((m_error = repository->loadDescriptions(m_descriptions, &m_descriptionsLastModified)) == DatabaseError::NoError) {
        m_future = QtConcurrent::map(m_descriptions, LoadPackage(repository, origin, m_descriptionsLastModified));
    }
}

/*!
 * \brief Creates a new instance wrapping the specified database struct.
 */
AlpmDatabase::AlpmDatabase(const QString &name, const QString &dbPath, RepositoryUsage usage, SignatureLevel sigLevel, std::uint32_t index, QObject *parent) :
    Repository(name, index, parent),
    m_dbPath(dbPath)
{
    m_usage = usage;
    m_sigLevel = sigLevel;
}

AlpmPackageLoader *AlpmDatabase::internalInit()
{
    // set description, determine origin
    PackageOrigin origin;
    if(m_name.compare(QLatin1String("local"), Qt::CaseInsensitive) == 0) {
        m_description = QStringLiteral("The local database");
        origin = PackageOrigin::LocalDb;
    } else {
        if((m_usage & RepositoryUsage::Sync) || (m_usage & RepositoryUsage::Install) || (m_usage & RepositoryUsage::Upgrade)) {
            m_description = QStringLiteral("Sync database »%1«").arg(m_name);
        } else {
            m_description = QStringLiteral("Database »%1«").arg(m_name);
        }
        origin = PackageOrigin::SyncDb;
    }

    // initialization of packages is done concurrently via AlpmPackageLoader
    return new AlpmPackageLoader(this, origin);

    // without concurrency
    //QList<QPair<QString, QList<QByteArray> > > descriptions;
    //loadDescriptions(descriptions);
    //for(const auto &description : descriptions) {
    //    addPackageFromDescription(description.first, description.second, origin);
    //}
    //emit initialized();
    //return nullptr;
}

RepositoryType AlpmDatabase::type() const
{
    return RepositoryType::AlpmDatabase;
}

PackageDetailAvailability AlpmDatabase::requestsRequired(PackageDetail packageDetail) const
{
    switch(packageDetail) {
    case PackageDetail::Basics:
    case PackageDetail::Dependencies:
    case PackageDetail::PackageInfo:
    case PackageDetail::AllAvailable:
        return PackageDetailAvailability::Immediately;
    default:
        return PackageDetailAvailability::Never;
    }
}

QNetworkRequest AlpmDatabase::regularDatabaseRequest()
{
    return QNetworkRequest(m_filesRedirs.isEmpty()
                ? (QUrl(serverUrls().front() % QChar('/') % name() % QStringLiteral(".db")))
                : m_regularRedirs.back());
}

QNetworkRequest AlpmDatabase::filesDatabaseRequest()
{
    return QNetworkRequest(m_filesRedirs.isEmpty()
                ? (QUrl(serverUrls().front() % QChar('/') % name() % QStringLiteral(".files")))
                : m_filesRedirs.back());
}

/*!
 * \brief Downloads the database from the server.
 * \param targetDir Specifies the directory to store the downloaded database file. Shall not include the filename.
 * \remarks
 *  - The download is performed asynchronously - this method returns immediately.
 *  - After successfull download the database path is update to the path of the new file and the
 *    repository is reinitiated.
 *  - Does nothing if there is not at least one server URL available.
 *  - Status messages are printed via cerr.
 */
bool AlpmDatabase::downloadDatabase(const QString &targetDir, bool filesDatabase)
{
    QWriteLocker locker(lock());
    if(serverUrls().isEmpty()) {
        return false; // no server URLs available
    }
    addBusyFlag();
    cerr << shchar << "Downloading " << (filesDatabase ? "files" : "regular") << " database for [" << name().toLocal8Bit().data() << "] from  mirror " << serverUrls().front().toLocal8Bit().data() << " ..." << endl;
    QNetworkReply *reply = networkAccessManager().get(filesDatabase ? filesDatabaseRequest() : regularDatabaseRequest());
    reply->setProperty("filesDatabase", filesDatabase);
    m_downloadTargetDir = targetDir.isEmpty() ? QString(QChar('.')) : targetDir;
    connect(reply, &QNetworkReply::finished, this, &AlpmDatabase::databaseDownloadFinished);
    return true;
}

/*!
 * \brief Refreshes the database by downloading it from the server
 *        or just reinitializing it if there is not at least one server URL available.
 * \param targetDir Specifies the directory to store the downloaded database file. Shall not include the filename.
 *                  Ignored when downloading is not possible.
 * \remarks Effectively updates the database file of sync databases and just refreshes
 *          local databases.
 */
void AlpmDatabase::refresh(const QString &targetDir)
{
    if(!downloadDatabase(targetDir, true)) {
        init();
    }
}

std::unique_ptr<Package> AlpmDatabase::emptyPackage()
{
    return make_unique<AlpmPackage>(this);
}

/*!
 * \brief Internally called to handle finishing of database download.
 * \remarks Connected in downloadDatabase().
 */
void AlpmDatabase::databaseDownloadFinished()
{
    auto *reply = static_cast<QNetworkReply *>(sender());
    reply->deleteLater();
    bool filesDatabase =  reply->property("filesDatabase").toBool();
    const QString redirTarget = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toString();
    QReadLocker locker(lock());
    switch(reply->error()) {
    case QNetworkReply::NoError: {
        if(!redirTarget.isEmpty()) {
            if((!filesDatabase && m_regularRedirs.contains(redirTarget)) || (filesDatabase && m_filesRedirs.contains(redirTarget))) {
                cerr << "Redirection-loop for database file [" << name().toLocal8Bit().data() << "]" << endl;
                removeBusyFlag();
                return;
            }

            (filesDatabase ? m_filesRedirs : m_regularRedirs) << redirTarget;
            cerr << "Redirection target for database file [" << name().toLocal8Bit().data() << "] available: " << redirTarget.toLocal8Bit().data() << endl;

            locker.unlock();
            downloadDatabase(m_downloadTargetDir, filesDatabase);
        }
        m_filesRedirs.clear();

        QString newDatabasePath;
        cerr << "Downloaded database file for [" << name().toLocal8Bit().data() << "] successfully." << endl;
        newDatabasePath = m_downloadTargetDir % QChar('/') % name() % (filesDatabase ? QStringLiteral(".files") : QStringLiteral(".db"));
        if(QFile::exists(newDatabasePath)) {
            QString backupFile(newDatabasePath + QStringLiteral(".bak"));
            QFile::remove(backupFile);
            if(!QFile::rename(newDatabasePath, backupFile)) {
                cerr << "An IO error occured when storing database file for [" << name().toLocal8Bit().data() << "]: Unable to rename present database file." << endl;
                reply = nullptr;
                removeBusyFlag();
                return;
            }
        }
        locker.unlock();
        QFile outputFile(newDatabasePath);
        if(outputFile.open(QFile::WriteOnly) && outputFile.write(reply->readAll())) {
            outputFile.close();
            {
                QWriteLocker locker(lock());
                m_dbPath = newDatabasePath;
            }
            init();
        } else {
            locker.relock();
            cerr << "An IO error occured when storing database file for [" << name().toLocal8Bit().data() << "]: Unable to create/write output file." << endl;
            removeBusyFlag();
        }
        break;
    } default:
        cerr << "An error occured when dwonloading database file for [" << name().toLocal8Bit().data() << "]: " << reply->errorString().toLocal8Bit().data() << endl;
        if(filesDatabase && reply->error() == QNetworkReply::ContentNotFoundError) {
            cerr << "-> Attempting to download regular database file instead of files database file." << endl;
            locker.unlock();
            downloadDatabase(m_downloadTargetDir, false);
        } else {
            removeBusyFlag();
        }
    }
}

} // namespace Alpm