#include "./buildactionprivate.h"
#include "./subprocess.h"

#include "../helper.h"
#include "../json.h"
#include "../serversetup.h"

#include "../webclient/aur.h"

#include "../webapi/params.h"

#include "../../libpkg/parser/database.h"
#include "../../libpkg/parser/utils.h"

#include <c++utilities/conversion/stringbuilder.h>
#include <c++utilities/conversion/stringconversion.h>
#include <c++utilities/io/ansiescapecodes.h>
#include <c++utilities/io/misc.h>

#include <boost/asio/read.hpp>

#include <boost/process/search_path.hpp>
#include <boost/process/start_dir.hpp>

#include <rapidjson/filewritestream.h>
#include <rapidjson/prettywriter.h>

#include <filesystem>
#include <iostream>

using namespace std;
using namespace CppUtilities;
using namespace CppUtilities::EscapeCodes;

namespace LibRepoMgr {

bool PackageBuildProgress::hasBeenAnyProgressMade() const
{
    return checksumsUpdated || hasSources || !finished.isNull() || addedToRepo || stagingNeeded != PackageStagingNeeded::Undetermined;
}

void PackageBuildProgress::resetProgress()
{
    checksumsUpdated = false;
    hasSources = false;
    finished = DateTime();
    addedToRepo = false;
    stagingNeeded = PackageStagingNeeded::Undetermined;
}

void PackageBuildProgress::resetChrootSettings()
{
    chrootDirectory.clear();
    chrootUser.clear();
    skipChrootUpgrade = skipChrootCleanup = keepPreviousSourceTree = false;
}

PrepareBuild::PrepareBuild(ServiceSetup &setup, const std::shared_ptr<BuildAction> &buildAction)
    : InternalBuildAction(setup, buildAction)
{
}

void PrepareBuild::run()
{
    // check whether a directory has been specified
    if (m_buildAction->directory.empty()) {
        reportError("Unable to create working directory: no directory name specified");
        return;
    }

    // check flags and settings
    const auto flags = static_cast<PrepareBuildFlags>(m_buildAction->flags);
    m_forceBumpPackageVersion = flags & PrepareBuildFlags::ForceBumpPkgRel;
    m_cleanSourceDirectory = flags & PrepareBuildFlags::CleanSrcDir;
    m_keepOrder = flags & PrepareBuildFlags::KeepOrder;
    m_keepPkgRelAndEpoch = flags & PrepareBuildFlags::KeepPkgRelAndEpoch;
    m_resetChrootSettings = flags & PrepareBuildFlags::ResetChrootSettings;
    if (m_forceBumpPackageVersion && m_keepPkgRelAndEpoch) {
        reportError("Can not force-bump pkgrel and keeping it at the same time.");
        return;
    }
    auto &metaInfo = m_setup.building.metaInfo;
    auto metaInfoLock = metaInfo.lockToRead();
    const auto &typeInfo = metaInfo.typeInfoForId(BuildActionType::PrepareBuild);
    const auto pkgbuildsDirsSetting = std::string(typeInfo.settings[static_cast<std::size_t>(PrepareBuildSettings::PKGBUILDsDirs)].param);
    metaInfoLock.unlock();
    if (const auto i = m_buildAction->settings.find(pkgbuildsDirsSetting); i != m_buildAction->settings.end()) {
        m_pkgbuildsDirs = splitString<std::vector<std::string>>(i->second, ":", EmptyPartsTreat::Omit);
    }

    // read values from global config
    auto setupReadLock = m_setup.lockToRead();
    m_makePkgPath = findExecutable(m_setup.building.makePkgPath);
    copySecondVectorIntoFirstVector(m_pkgbuildsDirs, m_setup.building.pkgbuildsDirs);
    m_ignoreLocalPkgbuildsRegex = m_setup.building.ignoreLocalPkgbuildsRegex;
    m_workingDirectory = determineWorkingDirectory(buildDataWorkingDirectory);
    setupReadLock.unlock();

    // check executables
    if (!checkExecutable(m_makePkgPath)) {
        reportError("Unable to find makepkg executable \"" % m_setup.building.makePkgPath + "\" in PATH.");
        return;
    }

    // init build action
    auto configReadLock
        = init(BuildActionAccess::ReadConfig, RequiredDatabases::MaybeSource | RequiredDatabases::OneDestination, RequiredParameters::Packages);
    if (std::holds_alternative<std::monostate>(configReadLock)) {
        return;
    }

    // find dependencies of the destination repository
    auto *const destinationDb = (*m_destinationDbs.begin());
    auto databaseDependencyOrderRes = m_setup.config.computeDatabaseDependencyOrder(*destinationDb);
    if (std::holds_alternative<std::string>(databaseDependencyOrderRes)) {
        configReadLock = std::monostate{};
        reportError("Unable to find the destination DB's dependencies: " + get<string>(databaseDependencyOrderRes));
        return;
    }
    auto &databaseDependencyOrder = get<vector<LibPkg::Database *>>(databaseDependencyOrderRes);
    for (const auto *destinationDbOrDependency : databaseDependencyOrder) {
        m_requiredDbs.emplace(destinationDbOrDependency->name);
    }

    // find dependencies of the staging repository
    auto *stagingDb = m_setup.config.findDatabase(destinationDb->name + "-staging", destinationDb->arch);
    auto stagingDbOrder = std::variant<std::vector<LibPkg::Database *>, std::string>();
    auto hasStagingDbOrder = false;
    if (!stagingDb && endsWith(destinationDb->name, "-testing")) {
        stagingDb = m_setup.config.findDatabase(
            argsToString(std::string_view(destinationDb->name.data(), destinationDb->name.size() - 8), "-staging"), destinationDb->arch);
    }
    if (!stagingDb && endsWith(destinationDb->name, "-staging")) {
        // use the destination DB as staging DB if it ends with "-staging"; we can assume it was intended to use it and the auto-staging
        // was just passed as usual
        stagingDb = destinationDb;
        stagingDbOrder = databaseDependencyOrder;
        hasStagingDbOrder = true;
    }
    if (stagingDb) {
        if (!hasStagingDbOrder) {
            stagingDbOrder = m_setup.config.computeDatabaseDependencyOrder(*stagingDb);
        }
        if (std::holds_alternative<std::string>(stagingDbOrder)) {
            m_warnings.emplace_back("Unable to find the staging DB's dependencies: " + get<string>(stagingDbOrder));
        }
    } else {
        m_warnings.emplace_back("Unable to find staging DB for \"" + destinationDb->name + "\". Auto-staging will not work.");
    }

    // set target and staging info
    m_targetDbName = destinationDb->name;
    m_targetArch = destinationDb->arch;
    if (stagingDb) {
        m_stagingDbName = stagingDb->name;
    }

    // make set of databases considered as given from the specified source databases
    // notes: 1. If *no* source databases are specified the destination database and all databases it is based on are considered as "given" and
    //           therefore not included in the build unless their package names are explicitly specified. All direct and transitive dependencies
    //           of the destination repository and the destination repository itself are enabled within the build root.
    //        2. If source databases are specified only packages contained by these databases will be considered as "given". Any other packages
    //           will be included in the build. Dependencies of the destination repository and the destination repository itself are only enabled within
    //           the build root when also listed as source repositories.
    //        3. All specified source repositories should be equal to the destination repository or a direct or transitive dependency of the destination
    //           repository. If that is not the case the built packages might depend on packages which are not available at install time. Hence a warning
    //           is emitted in that case.
    std::vector<std::string> missingBaseDbs;
    for (auto *const sourceDb : m_sourceDbs) {
        const auto &[sourceDbName, added] = m_baseDbs.emplace(sourceDb->name);
        if (!added) {
            continue;
        }
        if (m_requiredDbs.find(*sourceDbName) == m_requiredDbs.end()) {
            missingBaseDbs.emplace_back(*sourceDbName);
            databaseDependencyOrder.emplace_back(sourceDb); // appending the "odd" source database (see note 3. above) should be sufficient
        }
    }
    if (!missingBaseDbs.empty()) {
        m_warnings.emplace_back(
            "The following source DBs are not the destination DB or a (transitive) dependency of it: " + joinStrings(missingBaseDbs, ", "));
    }

    // compose the list of repositories to enable within the build root (for staging)
    populateDbConfig(databaseDependencyOrder, false);
    if (std::holds_alternative<std::vector<LibPkg::Database *>>(stagingDbOrder)) {
        populateDbConfig(std::get<std::vector<LibPkg::Database *>>(stagingDbOrder), true);
    }

    // try to find denoted packages to determine the actual package base and the existing version
    // -> We might get something like "my-repo/foo" and "my-repo/bar" where "foo" and "bar" are only split
    //    packages from the base package "foobar" when blindly trying to "rebuild" packages from a binary
    //    repository. Luckily database files store the corresponding base packages.
    // -> That trick doesn't work for new packages of course.
    //    FIXME: Maybe the AUR RPC provides useful information (which would at least help with AUR packages).
    // -> The existing version is useful to know when we need to bump pkgrel or even epoch.
    size_t currentIndex = 0;
    for (const auto &packageName : m_buildAction->packageNames) {
        // pick some package matching the specified packageName which has a source info; preferring packages
        // from the destination db
        auto existingPackages = m_setup.config.findPackages(packageName);
        auto foundSourcePackage = false;
        auto packageBuildData = PackageBuildData();
        for (auto &existingPackage : existingPackages) {
            // skip if package not relevant
            if (!isExistingPackageRelevant(packageName, existingPackage, packageBuildData, *destinationDb)) {
                continue;
            }
            // skip if package has no source info
            const auto &sourceInfo = existingPackage.pkg->sourceInfo;
            if (!sourceInfo || sourceInfo->name.empty()) {
                continue;
            }
            //
            auto &buildData = m_buildDataByPackage[sourceInfo->name];
            buildData.specifiedIndex = currentIndex++;
            buildData.existingPackages.emplace_back(existingPackage.pkg);
            if (buildData.existingVersion.empty()
                || LibPkg::PackageVersion::isNewer(LibPkg::PackageVersion::compare(existingPackage.pkg->version, buildData.existingVersion))) {
                buildData.existingVersion = existingPackage.pkg->version;
            }
            // assume we don't have sources if going to clean up the source directory
            // FIXME: It is a little bit inconsistent that in other places existing PackageBuildData is overridden and here it is re-used.
            if (m_cleanSourceDirectory) {
                buildData.hasSource = false;
            }
            foundSourcePackage = true;
            if (std::get<LibPkg::Database *>(existingPackage.db) == destinationDb) {
                break;
            }
        }
        // add an empty BuildPackage nevertheless assuming packageName specifies the base package as-is
        if (!foundSourcePackage) {
            packageBuildData.specifiedIndex = currentIndex++;
            m_buildDataByPackage[packageName] = std::move(packageBuildData);
        }
    }

    configReadLock = std::monostate{};

    // make working directory
    try {
        filesystem::create_directories(m_workingDirectory);
    } catch (const filesystem::filesystem_error &e) {
        reportError("Unable to create working directory " % m_workingDirectory % ": " + e.what());
        return;
    }

    if (reportAbortedIfAborted()) {
        return;
    }

    // locate PKGBUILDs locally or download them from AUR
    fetchMissingBuildData();
}

/*!
 * \brief Populates m_dbConfig (or m_dbStagingConfig if \a forStaging) with the specified \a dbOrder.
 */
void PrepareBuild::populateDbConfig(const std::vector<LibPkg::Database *> &dbOrder, bool forStaging)
{
    auto &dbConfig = forStaging ? m_stagingDbConfig : m_dbConfig;
    for (auto i = dbOrder.rbegin(), end = dbOrder.rend(); i != end; ++i) {
        const auto *const db = *i;
        auto scopeData = IniFile::ScopeData{};
        scopeData.emplace(make_pair("SigLevel", LibPkg::signatureLevelToString(db->signatureLevel)));
        for (const auto &mirror : db->mirrors) {
            scopeData.emplace(make_pair("Server", mirror));
        }
        if (m_sourceDbs.empty() || m_baseDbs.find(db->name) != m_baseDbs.cend() || (forStaging && endsWith(db->name, "-staging"))) {
            dbConfig.emplace_back(IniFile::Scope(db->name, move(scopeData)));
        }
    }
}

/*!
 * \brief Determines whether the existing package \a package is relevant for providing the dependency with \a dependencyName considering the
 * specified \a destinationDb.
 * \remarks The \a package is added to the existing packages in \a packageBuildData in case it is useful for further processing even though
 * not for providing the dependency (e.g. bumping pkgrel).
 */
bool PrepareBuild::isExistingPackageRelevant(
    const string &dependencyName, LibPkg::PackageSearchResult &package, PackageBuildData &packageBuildData, const LibPkg::Database &destinationDb)
{
    // skip db if arch isn't matching the destination db arch
    const auto *const db = std::get<LibPkg::Database *>(package.db);
    if (db->arch != destinationDb.arch) {
        return false;
    }
    // add an exact package match to the build data if it is present in the destination db or any dependency because it is
    // still relevant for bumping versions, even if we ignore it because it is not present in the base DBs
    const auto dependencyPresentInRequiredDbs = m_requiredDbs.find(db->name) != m_requiredDbs.end();
    if (dependencyPresentInRequiredDbs && dependencyName == package.pkg->name) {
        if (packageBuildData.existingVersion.empty()
            || LibPkg::PackageVersion::isNewer(LibPkg::PackageVersion::compare(package.pkg->version, packageBuildData.existingVersion))) {
            packageBuildData.existingVersion = package.pkg->version;
        }
        packageBuildData.existingPackages.emplace_back(package.pkg);
    }
    // skip if package is not from any database available within that build
    return (m_baseDbs.empty() && dependencyPresentInRequiredDbs) || (m_baseDbs.find(db->name) != m_baseDbs.end());
}

void PrepareBuild::addPackageToLogLine(std::string &logLine, const std::string &packageName)
{
    logLine += ' ';
    logLine += packageName;
}

void PrepareBuild::makeSrcInfo(
    std::shared_ptr<WebClient::AurSnapshotQuerySession> &multiSession, const std::string &sourceDirectory, const std::string &packageName)
{
    auto processSession = make_shared<ProcessSession>(
        m_setup.building.ioContext, [multiSession, &sourceDirectory, &packageName](boost::process::child &&child, ProcessResult &&result) {
            processSrcInfo(*multiSession, sourceDirectory, packageName, std::move(child), std::move(result));
        });
    processSession->launch(boost::process::start_dir(sourceDirectory), m_makePkgPath.string(), "--printsrcinfo");
}

void PrepareBuild::processSrcInfo(WebClient::AurSnapshotQuerySession &multiSession, const std::string &sourceDirectory,
    const std::string &packageName, boost::process::child &&child, ProcessResult &&result)
{
    if (result.errorCode) {
        multiSession.addResponse(WebClient::AurSnapshotResult{ .packageName = packageName,
            .errorOutput = move(result.error),
            .error = argsToString("Unable to invoke makepkg --printsourceinfo: ", result.errorCode.message()) });
        return;
    }
    if (child.exit_code() != 0) {
        multiSession.addResponse(WebClient::AurSnapshotResult{ .packageName = packageName,
            .errorOutput = move(result.error),
            .error = argsToString("makepkg --printsourceinfo exited with non-zero exit code: ", child.exit_code()) });
        return;
    }

    const auto srcInfoPath = sourceDirectory + "/.SRCINFO";
    try {
        writeFile(srcInfoPath, result.output);
    } catch (const std::ios_base::failure &failure) {
        multiSession.addResponse(
            WebClient::AurSnapshotResult{ .packageName = packageName, .error = "Unable to write " % srcInfoPath % ": " + failure.what() });
        return;
    }

    addResultFromSrcInfo(multiSession, packageName, result.output);
}

void PrepareBuild::addResultFromSrcInfo(WebClient::AurSnapshotQuerySession &multiSession, const std::string &packageName, const std::string &srcInfo)
{
    auto snapshotResult = WebClient::AurSnapshotResult{ .packageName = packageName, .packages = LibPkg::Package::fromInfo(srcInfo, false) };
    if (snapshotResult.packages.empty() || snapshotResult.packages.front()->name.empty()) {
        snapshotResult.error = "Unable to parse .SRCINFO: no package name present";
    } else if (!(snapshotResult.sourceInfo = snapshotResult.packages.front()->sourceInfo)) {
        snapshotResult.error = "Unable to parse .SRCINFO: no source info present";
    }
    multiSession.addResponse(std::move(snapshotResult));
}

void PrepareBuild::fetchMissingBuildData()
{
    auto multiSession
        = WebClient::AurSnapshotQuerySession::create(m_setup.building.ioContext, bind(&PrepareBuild::computeDependencies, this, placeholders::_1));
    vector<WebClient::AurSnapshotQueryParams> snapshotQueries;

    // prepare logging
    auto logLines = vector<string>{
        "  -> Generating .SRCINFO for local PKGBUILDs:", "  -> Retrieving sources from AUR:", "  -> Using existing source directories:"
    };
    auto needToGeneratedSrcInfo = false;
    auto usingExistingSourceDirectories = false;

    // fetch build data (PKGBUILD, .SRCINFO, ...) for all the packages we want to build
    for (auto &[packageName, buildData] : m_buildDataByPackage) {

        // skip packages where the source has already been fetched in a previous invocation or which have already failed on a previous invocation
        if (buildData.hasSource || !buildData.error.empty()) {
            continue;
        }

        buildData.sourceDirectory = m_workingDirectory % '/' % packageName + "/src";

        // skip fetching if source directory already exists (from previous run or manual provisioning)
        // clean existing source directory if cleanup is enabled
        try {
            const auto sourceDirectoryExists = filesystem::exists(buildData.sourceDirectory);
            if (m_cleanSourceDirectory && sourceDirectoryExists) {
                filesystem::remove_all(buildData.sourceDirectory);
            } else if (sourceDirectoryExists) {
                // verify the PKGBUILD file exists
                const auto pkgbuildPath = buildData.sourceDirectory + "/PKGBUILD";
                if (!filesystem::exists(pkgbuildPath)) {
                    multiSession->addResponse(WebClient::AurSnapshotResult{ .packageName = packageName,
                        .error = "Existing source directory \"" % buildData.sourceDirectory + "\" does not contain a PKGBUILD file." });
                    continue;
                }
                // generate the .SRCINFO file if not already present; otherwise read the existing .SRCINFO
                const auto srcInfoPath = buildData.sourceDirectory + "/.SRCINFO";
                if (!filesystem::exists(srcInfoPath) || filesystem::last_write_time(srcInfoPath) < filesystem::last_write_time(pkgbuildPath)) {
                    makeSrcInfo(multiSession, buildData.sourceDirectory, packageName);
                    needToGeneratedSrcInfo = true;
                } else {
                    string srcInfo;
                    try {
                        srcInfo = readFile(srcInfoPath, 0x10000);
                    } catch (const std::ios_base::failure &e) {
                        multiSession->addResponse(WebClient::AurSnapshotResult{ .packageName = packageName,
                            .error = "Unable to read .SRCINFO \"" % srcInfoPath % "\" from existing source directory: " + e.what() });
                        continue;
                    }
                    addResultFromSrcInfo(*multiSession, packageName, srcInfo);
                }
                usingExistingSourceDirectories = true;
                addPackageToLogLine(logLines[0], packageName);
                addPackageToLogLine(logLines[2], packageName);
                buildData.hasSource = true;
                continue;
            }
        } catch (const filesystem::filesystem_error &e) {
            multiSession->addResponse(WebClient::AurSnapshotResult{ .packageName = packageName,
                .error = "Unable to check existing source directory \"" % buildData.sourceDirectory % "\": " + e.what() });
            continue;
        }

        // clear the original source directory which is possibly still assigned from a previous run
        buildData.originalSourceDirectory.clear();

        // skip next block if package name matches configured pattern
        LibPkg::PackageNameData packageNameData;
        if (regex_match(packageName, m_ignoreLocalPkgbuildsRegex)) {
            goto addAurQuery;
        }

        // copy PKGBUILD from local PKGBUILDs directory and generate .SRCINFO from it via makepkg
        if (!m_pkgbuildsDirs.empty()) {
            packageNameData = LibPkg::PackageNameData::decompose(packageName);
        }
        for (const auto &pkgbuildsDir : m_pkgbuildsDirs) {
            const auto variant = packageNameData.variant();
            auto pkgbuildPath = pkgbuildsDir % '/' % packageNameData.actualName % '/' % variant;
            try {
                if (!filesystem::exists(pkgbuildPath + "/PKGBUILD")) {
                    continue;
                }
                buildData.originalSourceDirectory = tupleToString(pkgbuildPath);
                filesystem::create_directories(buildData.sourceDirectory);
                filesystem::copy(buildData.originalSourceDirectory, buildData.sourceDirectory, std::filesystem::copy_options::recursive);

            } catch (const filesystem::filesystem_error &e) {
                multiSession->addResponse(WebClient::AurSnapshotResult{ .packageName = packageName,
                    .error
                    = "Unable to copy files from PKGBUILDs directory " % pkgbuildsDir % " to " % buildData.sourceDirectory % ": " + e.what() });
                continue;
            }
            makeSrcInfo(multiSession, buildData.sourceDirectory, packageName);
            needToGeneratedSrcInfo = true;
            addPackageToLogLine(logLines[0], packageName);
            buildData.hasSource = true;
            break;
        }

    addAurQuery:
        // download latest snapshot containing PKGBUILD and .SRCINFO from AUR
        if (!buildData.hasSource) {
            snapshotQueries.emplace_back(WebClient::AurSnapshotQueryParams{
                .packageName = &packageName,
                .targetDirectory = &buildData.sourceDirectory,
            });
            addPackageToLogLine(logLines[1], packageName);
        }
    }

    // schedule async AUR downloads
    if (!snapshotQueries.empty()) {
        WebClient::queryAurSnapshots(m_buildAction->log(), m_setup, snapshotQueries, m_setup.building.ioContext, multiSession);
    }

    // log status
    if (!needToGeneratedSrcInfo) {
        logLines[0].clear();
    }
    if (snapshotQueries.empty()) {
        logLines[1].clear();
    }
    if (!usingExistingSourceDirectories) {
        logLines[2].clear();
    }
    m_buildAction->appendOutput(Phrases::InfoMessage, "Fetching missing build data\n", joinStrings(logLines, string(), true, string(), "\n"));
}

bool PrepareBuild::pullFurtherDependencies(const std::vector<LibPkg::Dependency> &dependencies)
{
    // pull further dependencies; similar to initial dependency lookup in run()
    auto dependencyAdded = false;
    const auto *const destinationDb = *m_destinationDbs.begin();
    for (const auto &dependency : dependencies) {
        auto dependencyExists = false;

        // skip if the dependency is already in the list of packages to be built (check for cycles is done later when computing batches)
        if (m_buildDataByPackage.find(dependency.name) != m_buildDataByPackage.end()) {
            continue;
        }
        for (const auto &[packageName, buildData] : m_buildDataByPackage) {
            for (const auto &package : buildData.packages) {
                if (package->providesDependency(dependency)) {
                    dependencyExists = true;
                    break;
                }
            }
            if (dependencyExists) {
                break;
            }
        }
        if (dependencyExists) {
            continue;
        }

        // skip dependency if it is already present in one of the dependencies considered as "given" (see note in run() function)
        auto packageBuildData = PackageBuildData();
        auto existingPackages = m_setup.config.findPackages(dependency);
        for (auto &package : existingPackages) {
            // skip if package not relevant
            if (!isExistingPackageRelevant(dependency.name, package, packageBuildData, *destinationDb)) {
                continue;
            }
            dependencyExists = true;
        }
        if (dependencyExists) {
            continue;
        }

        // assume the dependency is provided by a package with the pkgbase of the dependency denotation
        // FIXME: support split packages
        m_buildDataByPackage[dependency.name] = std::move(packageBuildData);
        dependencyAdded = true;
    }
    return dependencyAdded;
}

struct BatchItem {
    struct MissingDependency {
        LibPkg::Dependency dependency;

        LibPkg::Dependency requiredBy;
    };

    const std::string *const name;
    const PackageBuildData *const buildData;
    std::unordered_map<const BatchItem *, std::unordered_set<const BatchItem *>> neededItems;
    LibPkg::DependencySet requiredDependencies;
    LibPkg::DependencySet providedDependencies;
    std::vector<MissingDependency> missingDependencies;
    bool done = false;

    void determineNeededItems(LibPkg::Config &config, const std::unordered_set<string> &requiredDbs, const LibPkg::Database *destinationDb,
        unordered_map<string, BatchItem> &batchItems);
    template <typename... DependencyParams>
    bool addNeededItemForRequiredDependencyFromOtherItems(
        const unordered_map<string, BatchItem> &otherItems, DependencyParams &&...requiredDependencyParams);
    static bool addItemsToBatch(Batch &currentBatch, BatchMap &items);

private:
    void addNeededBatchItemsForPackage(LibPkg::Config &config, const std::unordered_set<string> &requiredDbs, const LibPkg::Database *destinationDb,
        unordered_map<string, BatchItem> &batchItems, unordered_set<const LibPkg::Package *> &visitedPackages, LibPkg::Package &package);
};

template <typename... DependencyParams>
bool BatchItem::addNeededItemForRequiredDependencyFromOtherItems(
    const unordered_map<string, BatchItem> &otherItems, DependencyParams &&...requiredDependencyParams)
{
    // skip if the dependency of a package is provided by the package itself (can happen when dealing with split packages)
    if (providedDependencies.provides(std::forward<DependencyParams>(requiredDependencyParams)...)) {
        return true;
    }

    // go through all the items to find one which provides the dependency; continue to look for alternatives as well
    const BatchItem *relevantItem = nullptr;
    std::unordered_set<const BatchItem *> alternativeItems;
    for (const auto &item : otherItems) {
        if (&item.second == this) {
            continue;
        }
        if (item.second.providedDependencies.provides(std::forward<DependencyParams>(requiredDependencyParams)...)) {
            if (!relevantItem) {
                relevantItem = &item.second;
            } else {
                alternativeItems.emplace(&item.second);
            }
        }
    }

    if (relevantItem) {
        // FIXME: actually, we need to handle the case when the item is already present as the alternative items might differ
        neededItems.emplace(relevantItem, move(alternativeItems));
        return true;
    }
    return false;
}

void BatchItem::determineNeededItems(LibPkg::Config &config, const std::unordered_set<std::string> &requiredDbs,
    const LibPkg::Database *destinationDb, unordered_map<string, BatchItem> &batchItems)
{
    unordered_set<const LibPkg::Package *> visitedPackages;

    for (const auto &require : requiredDependencies) {
        // check whether its provided directly by some other item
        if (addNeededItemForRequiredDependencyFromOtherItems(batchItems, require.first, require.second)) {
            continue;
        }

        // check by which other package the item is provided
        // skip dependency if it is already present in one of the dbs
        auto dependency = LibPkg::Dependency(require.first, require.second.version, require.second.mode);
        auto foundPackage = false;
        const auto existingPackages = config.findPackages(dependency);
        for (const auto &package : existingPackages) {
            // skip if package is not from any database available within that build
            const auto *const db = std::get<LibPkg::Database *>(package.db);
            if (requiredDbs.find(db->name) == requiredDbs.end()) {
                continue;
            }
            if (db->arch != destinationDb->arch) {
                continue;
            }
            addNeededBatchItemsForPackage(config, requiredDbs, destinationDb, batchItems, visitedPackages, *package.pkg);
            foundPackage = true;
        }
        if (!foundPackage) {
            missingDependencies.emplace_back(MissingDependency{ .dependency = move(dependency), .requiredBy = LibPkg::Dependency(*name) });
        }
    }
}

bool BatchItem::addItemsToBatch(Batch &currentBatch, BatchMap &items)
{
    auto allDone = true;

    for (auto &[packageName, batchItem] : items) {
        // skip items which are already done
        if (batchItem.done) {
            continue;
        }

        allDone = false;

        // skip if not all dependencies of the items are done
        auto allDepsDone = true;
        for (const auto &[neededItem, alternatives] : batchItem.neededItems) {
            if (neededItem->done) {
                continue;
            }
            auto oneAlternativeDone = false;
            for (const auto &alternative : alternatives) {
                if (alternative->done) {
                    oneAlternativeDone = true;
                    break;
                }
            }
            if (oneAlternativeDone) {
                continue;
            }
            allDepsDone = false;
            break;
        }
        if (!allDepsDone) {
            continue;
        }

        // add item to current batch
        currentBatch.emplace_back(&batchItem);
    }

    return allDone;
}

void BatchItem::addNeededBatchItemsForPackage(LibPkg::Config &config, const std::unordered_set<string> &requiredDbs,
    const LibPkg::Database *destinationDb, unordered_map<string, BatchItem> &batchItems, unordered_set<const LibPkg::Package *> &visitedPackages,
    LibPkg::Package &package)
{
    // stop if there's a cycle (not a critical error here; we've just seen enough; only cycles between packages we actually want to build are interesting)
    const auto [i, notVisitedBefore] = visitedPackages.emplace(&package);
    if (!notVisitedBefore) {
        return;
    }

    // check whether the dependencies of that package lead to another build item
    for (const auto &dep : package.dependencies) {
        // check whether the dependency is directly one of the build items
        if (addNeededItemForRequiredDependencyFromOtherItems(batchItems, dep)) {
            continue;
        }

        // check other packages recursively
        auto foundPackage = false;
        const auto existingPackages = config.findPackages(dep);
        for (const auto &existingPackage : existingPackages) {
            // skip if package is not from any database available within that build
            const auto *const db = std::get<LibPkg::Database *>(existingPackage.db);
            if (requiredDbs.find(db->name) == requiredDbs.end()) {
                continue;
            }
            if (db->arch != destinationDb->arch) {
                continue;
            }
            addNeededBatchItemsForPackage(config, requiredDbs, destinationDb, batchItems, visitedPackages, *existingPackage.pkg);
            foundPackage = true;
        }
        if (!foundPackage) {
            missingDependencies.emplace_back(MissingDependency{ .dependency = dep, .requiredBy = LibPkg::Dependency(package.name) });
        }
    }
}

static bool sortBatch(const BatchItem *lhs, const BatchItem *rhs)
{
    if (lhs->buildData->specifiedIndex != numeric_limits<size_t>::max() && rhs->buildData->specifiedIndex != numeric_limits<size_t>::max()) {
        return lhs->buildData->specifiedIndex < rhs->buildData->specifiedIndex;
    } else if (lhs->buildData->specifiedIndex != numeric_limits<size_t>::max()) {
        return true;
    } else if (rhs->buildData->specifiedIndex != numeric_limits<size_t>::max()) {
        return false;
    } else {
        return *lhs->name < *rhs->name;
    }
}

void PrepareBuild::bumpVersions()
{
    // bump pkgrel or even epoch to ensure the new packages are actually considered newer by pacman when updating
    if (m_keepPkgRelAndEpoch) {
        return;
    }
    for (const auto &[packageName, buildData] : m_buildDataByPackage) {
        auto &existingVersionStr = buildData.existingVersion;
        if (existingVersionStr.empty() && !m_forceBumpPackageVersion) {
            continue;
        }
        auto existingVersion = existingVersionStr.empty() ? LibPkg::PackageVersion{} : LibPkg::PackageVersion::fromString(existingVersionStr);
        LibPkg::PackageAmendment amendment;
        LibPkg::PackageVersion newVersion;
        for (const auto &package : buildData.packages) {
            newVersion = LibPkg::PackageVersion::fromString(package->version);
            if (existingVersionStr.empty()) {
                existingVersion = newVersion;
                break;
            }
            switch (newVersion.compare(existingVersion)) {
            case LibPkg::PackageVersionComparison::Equal:
            case LibPkg::PackageVersionComparison::PackageUpgradeOnly:
                amendment.bumpDownstreamVersion = LibPkg::PackageAmendment::VersionBump::PackageVersion;
                m_warnings.emplace_back("Bumping pkgrel of " % package->name % "; version " % existingVersionStr + " already exists");
                break;
            case LibPkg::PackageVersionComparison::SoftwareUpgrade:
                if (package->decomposeName().isVcsPackage()) {
                    // skip bumping epoch of VCS packages; the pkgver is supposed to be adjusted in pkgver()
                    // note: Not skipping this in the pkgrel handling in the case above because when pkgver() returns a new version, pkgrel is
                    //       reset automatically so there's no harm in bumping it and it might actually be needed if there's no no version.
                    m_warnings.emplace_back("Version of package " % package->name % " is" % package->version
                            % " which is older than existing version " % existingVersionStr
                        + "; NOT bumping the epoch because it is a VCS package; be sure pkgver() actually yields a new version");
                    break;
                }
                amendment.bumpDownstreamVersion = LibPkg::PackageAmendment::VersionBump::Epoch;
                m_warnings.emplace_back("Bumping epoch of " % package->name % "; its version " % package->version % " is older than existing version "
                    + existingVersionStr);
                goto breakLoop;
            default:;
            }
        }
    breakLoop:
        if (m_forceBumpPackageVersion && amendment.isEmpty()) {
            amendment.bumpDownstreamVersion = LibPkg::PackageAmendment::VersionBump::PackageVersion;
            m_warnings.emplace_back("Bumping pkgrel of " % packageName + " (and its split packages); forcing pkgrel bump has been enabled");
        }
        if (!amendment.isEmpty()) {
            auto amendedVersions = LibPkg::amendPkgbuild(buildData.sourceDirectory + "/PKGBUILD", existingVersion, amendment);
            if (!amendedVersions.newEpoch.empty()) {
                newVersion.epoch = std::move(amendedVersions.newEpoch);
            }
            if (!amendedVersions.newPkgRel.empty()) {
                newVersion.package = std::move(amendedVersions.newPkgRel);
            }
            const auto newVersionStr = newVersion.toString();
            for (const auto &package : buildData.packages) {
                package->version = newVersionStr;
            }
            m_warnings.emplace_back("New version of " % packageName % " (and its split packages) is " + newVersionStr);
        }
    }
}

void PrepareBuild::computeDependencies(WebClient::AurSnapshotQuerySession::ContainerType &&responses)
{
    if (reportAbortedIfAborted()) {
        return;
    }

    // find databases again
    auto configReadLock = m_setup.config.lockToRead();
    if (auto error = findDatabases(); !error.empty()) {
        reportError(move(error));
        return;
    }

    // populate build data from responses and add further dependencies
    auto furtherDependenciesNeeded = false;
    auto sourcesMissing = false;
    for (auto &response : responses) {
        if (response.packageName.empty()) {
            auto &buildData = m_buildDataByPackage["?"];
            auto &error = buildData.error;
            if (!error.empty()) {
                error += '\n';
            }
            error = response.error.empty() ? "got response with no package name (internal error)" : move(response.error);
            sourcesMissing = true;
            continue;
        }
        auto &buildData = m_buildDataByPackage[response.packageName];
        buildData.sourceInfo = move(response.sourceInfo);
        buildData.packages = move(response.packages);
        buildData.error = move(response.error);
        buildData.hasSource = buildData.sourceInfo && !buildData.packages.empty() && buildData.error.empty();
        if (buildData.hasSource) {
            const auto buildActionsWriteLock = m_setup.building.lockToWrite();
            m_buildAction->artefacts.emplace_back(buildData.sourceDirectory + "/PKGBUILD"); // FIXME: add all files as artefacts
            continue;
        }
        if (buildData.error.empty()) {
            buildData.error = "no build data available";
        }
        sourcesMissing = true;
    }
    for (auto &response : responses) {
        if (response.packageName.empty()) {
            continue;
        }
        auto &buildData = m_buildDataByPackage[response.packageName];
        if (!buildData.hasSource) {
            continue;
        }
        furtherDependenciesNeeded = pullFurtherDependencies(buildData.sourceInfo->makeDependencies) || furtherDependenciesNeeded;
        furtherDependenciesNeeded = pullFurtherDependencies(buildData.sourceInfo->checkDependencies) || furtherDependenciesNeeded;
        for (const auto &package : buildData.packages) {
            furtherDependenciesNeeded = pullFurtherDependencies(package->dependencies) || furtherDependenciesNeeded;
        }
    }
    configReadLock.unlock();

    if (reportAbortedIfAborted()) {
        return;
    }

    // pull missing dependencies if all sources could be retrieved so far
    if (!sourcesMissing && furtherDependenciesNeeded) {
        fetchMissingBuildData();
        return;
    }

    // check for errors
    set<string> failedPackages;
    for (const auto &buildData : m_buildDataByPackage) {
        if (!buildData.second.error.empty()) {
            failedPackages.emplace(buildData.first);
        }
    }
    if (!failedPackages.empty()) {
        auto errorMessage = "Unable to retrieve the following packages (see result data for details): " + joinStrings(failedPackages, " ");
        m_buildAction->appendOutput(Phrases::ErrorMessage, errorMessage, '\n');
        auto resultData = makeResultData(std::move(errorMessage));
        auto buildActionWriteLock = m_setup.building.lockToWrite();
        m_buildAction->resultData = std::move(resultData);
        reportError();
        return;
    }

    bumpVersions();

    if (reportAbortedIfAborted()) {
        return;
    }

    if (m_keepOrder) {
        deduceBatchesFromSpecifiedOrder();
    } else {
        computeBatches();
    }

    auto resultData = makeResultData();
    auto buildActionWriteLock = m_setup.building.lockToWrite();
    m_buildAction->resultData = std::move(resultData);
    if (resultData.error.empty() && resultData.cyclicLeftovers.empty()) {
        reportSuccess();
    } else {
        reportError();
    }
}

std::unordered_map<std::string, BatchItem> PrepareBuild::prepareBatches()
{
    std::unordered_map<std::string, BatchItem> batchItems;
    for (const auto &[packageName, buildData] : m_buildDataByPackage) {
        auto [i, newItem] = batchItems.try_emplace(packageName, BatchItem{ .name = &packageName, .buildData = &buildData });
        auto &batchItem = i->second;
        const auto &sourceInfo = buildData.sourceInfo;
        for (const auto &package : buildData.packages) {
            for (const auto &deps : { sourceInfo->makeDependencies, sourceInfo->checkDependencies, package->dependencies }) {
                for (const auto &dep : deps) {
                    batchItem.requiredDependencies.add(dep, package);
                }
            }
            for (auto &dep : package->provides) {
                batchItem.providedDependencies.add(dep, package);
            }
        }
    }
    return batchItems;
}

void PrepareBuild::deduceBatchesFromSpecifiedOrder()
{
    m_buildAction->appendOutput(Phrases::InfoMessage, "Fetched sources; making batches for specified order ...\n"sv);

    auto batchItems = prepareBatches();
    BatchList batches;
    for (auto &[packageName, batchItem] : batchItems) {
        batches.emplace_back(std::vector<BatchItem *>{ &batchItem });
    }
    std::sort(batches.begin(), batches.end(), [](const Batch &lhs, const Batch &rhs) { return sortBatch(lhs.front(), rhs.front()); });
    addBatchesToResult(std::move(batches), Batch());
}

void PrepareBuild::computeBatches()
{
    m_buildAction->appendOutput(Phrases::InfoMessage, "Fetched sources; computing build batches ...\n"sv);

    // prepare computing batches
    auto batchItems = prepareBatches();
    auto configReadLock2 = m_setup.config.lockToRead();
    if (auto error = findDatabases(); !error.empty()) {
        reportError(move(error));
        return;
    }

    // add dependency relations to batch items
    for (auto &batchItem : batchItems) {
        batchItem.second.determineNeededItems(m_setup.config, m_requiredDbs, *m_destinationDbs.begin(), batchItems);
    }
    configReadLock2.unlock();

    // FIXME: check for missing dependencies

    // add batch items into batches
    BatchList batches;
    bool allDone;
    size_t addedItems;
    do {
        if (reportAbortedIfAborted()) {
            return;
        }
        auto &newBatch = batches.emplace_back();
        allDone = BatchItem::addItemsToBatch(newBatch, batchItems);
        addedItems = newBatch.size();
        if (!addedItems) {
            batches.pop_back();
        }
        for (auto &addedItem : newBatch) {
            addedItem->done = true;
        }
    } while (!allDone && addedItems);

    // check for cyclic leftovers
    Batch cyclicLeftovers;
    if (!allDone) {
        for (auto &item : batchItems) {
            if (!item.second.done) {
                cyclicLeftovers.emplace_back(&item.second);
            }
        }
        if (!cyclicLeftovers.empty()) {
            m_warnings.emplace_back("Dependency cycles have been detected. Add a bootstrap package to the package list to resolve this.");
        }
    }

    // sort batches so the order in which packages have been specified is preserved within the batches
    for (auto &batch : batches) {
        std::sort(batch.begin(), batch.end(), sortBatch);
    }
    std::sort(cyclicLeftovers.begin(), cyclicLeftovers.end(), sortBatch);

    // add batches to result
    addBatchesToResult(std::move(batches), std::move(cyclicLeftovers));
}

void PrepareBuild::addBatchesToResult(BatchList &&batches, Batch &&cyclicLeftovers)
{
    const auto returnItemName = [](const BatchItem *item) { return *item->name; };
    m_batches.reserve(batches.size());
    std::transform(batches.cbegin(), batches.cend(), std::back_inserter(m_batches), [&returnItemName](const Batch &batch) {
        std::vector<std::string> newBatch;
        newBatch.reserve(batch.size());
        std::transform(batch.cbegin(), batch.cend(), std::back_inserter(newBatch), returnItemName);
        return newBatch;
    });
    m_cyclicLeftovers.reserve(cyclicLeftovers.size());
    std::transform(cyclicLeftovers.cbegin(), cyclicLeftovers.cend(), std::back_inserter(m_cyclicLeftovers), returnItemName);
}

BuildPreparation PrepareBuild::makeResultData(std::string &&error)
{
    auto resultData = BuildPreparation{
        .buildData = std::move(m_buildDataByPackage),
        .dbConfig = std::move(m_dbConfig),
        .stagingDbConfig = std::move(m_stagingDbConfig),
        .targetDb = std::move(m_targetDbName),
        .targetArch = std::move(m_targetArch),
        .stagingDb = std::move(m_stagingDbName),
        .batches = std::move(m_batches),
        .cyclicLeftovers = std::move(m_cyclicLeftovers),
        .warnings = std::move(m_warnings),
        .error = std::move(error),
        .manuallyOrdered = m_keepOrder,
    };

    // write results into the working directory so the actual build can pick it up
    try {
        dumpJsonDocument(
            [this, &resultData] {
                const auto configLock = m_setup.config.lockToRead();
                return resultData.toJsonDocument();
            },
            m_workingDirectory, buildPreparationFileName, DumpJsonExistingFileHandling::Backup);
    } catch (const std::runtime_error &e) {
        const auto what = string_view(e.what());
        if (resultData.error.empty()) {
            resultData.error = what;
        } else {
            resultData.warnings.emplace_back(what);
        }
        m_buildAction->appendOutput(Phrases::ErrorMessage, what, '\n');
    }

    // write build progress skeletion (a BuildProgress with a default-initialized PackageBuildProgress for each package)
    try {
        BuildProgress progress;
        try {
            restoreJsonObject(progress, m_workingDirectory, buildProgressFileName, RestoreJsonExistingFileHandling::Skip);
        } catch (const std::runtime_error &e) {
            m_buildAction->appendOutput(Phrases::ErrorMessage, "Unable to read existing build-progress.json: ", e.what(), '\n');
        }
        if (dumpJsonDocument(
                [this, &resultData, &progress] {
                    // reset database-specific fields as the database configuration might have changed
                    progress.targetDbFilePath.clear();
                    progress.targetRepoPath.clear();
                    progress.stagingDbFilePath.clear();
                    progress.stagingRepoPath.clear();
                    for (const auto &[packageName, buildData] : resultData.buildData) {
                        auto &buildProgress = progress.progressByPackage[packageName];
                        // reset the build progress if the PKGBUILD has been updated
                        if (!buildProgress.hasBeenAnyProgressMade()) {
                            continue;
                        }
                        if (buildProgress.buildDirectory.empty()) {
                            buildProgress.resetProgress();
                            if (m_resetChrootSettings) {
                                buildProgress.resetChrootSettings();
                            }
                            continue;
                        }
                        const std::filesystem::path srcDirPkgbuild = buildData.sourceDirectory + "/PKGBUILD";
                        const std::filesystem::path buildDirPkgbuild = buildProgress.buildDirectory + "/PKGBUILD";
                        try {
                            if (!std::filesystem::exists(buildDirPkgbuild)
                                || std::filesystem::last_write_time(srcDirPkgbuild) > std::filesystem::last_write_time(buildDirPkgbuild)) {
                                buildProgress.resetProgress();
                                if (m_resetChrootSettings) {
                                    buildProgress.resetChrootSettings();
                                }
                            }
                        } catch (const std::filesystem::filesystem_error &e) {
                            m_buildAction->appendOutput(
                                Phrases::ErrorMessage, "Unable to determine whether PKGBUILD has been updated: ", e.what(), '\n');
                        }
                    }
                    return progress.toJsonDocument();
                },
                m_workingDirectory, buildProgressFileName, DumpJsonExistingFileHandling::Backup)
                .empty()) {
            m_buildAction->appendOutput(Phrases::ErrorMessage, "Unable to write build-progress.json\n"sv);
        }
    } catch (const std::runtime_error &e) {
        const auto what = string_view(e.what());
        resultData.warnings.emplace_back(what);
        m_buildAction->appendOutput(Phrases::ErrorMessage, what);
    }

    return resultData;
}

} // namespace LibRepoMgr