320 lines
12 KiB
C++
320 lines
12 KiB
C++
#include "./userrepository.h"
|
|
#include "./networkaccessmanager.h"
|
|
|
|
#include "../alpm/aurpackage.h"
|
|
#include "../alpm/config.h"
|
|
|
|
#include <KTar>
|
|
#include <KArchiveFile>
|
|
#include <KFilterDev>
|
|
|
|
#include <QStringBuilder>
|
|
#include <QUrlQuery>
|
|
#include <QNetworkRequest>
|
|
#include <QNetworkReply>
|
|
#include <QNetworkAccessManager>
|
|
#include <QJsonDocument>
|
|
#include <QJsonObject>
|
|
#include <QJsonArray>
|
|
#include <QDir>
|
|
#include <QTemporaryFile>
|
|
#include <QStringBuilder>
|
|
|
|
#include <iostream>
|
|
#include <memory>
|
|
|
|
using namespace std;
|
|
using namespace CppUtilities;
|
|
|
|
namespace RepoIndex {
|
|
|
|
const char *requestTypeProp = "type";
|
|
const QString rpcRequestTypeKey(QStringLiteral("type"));
|
|
const QString rpcRequestTypeSearch(QStringLiteral("search"));
|
|
const QString rpcRequestTypeSuggest(QStringLiteral("suggest"));
|
|
const QString rpcRequestTypeMultiInfo(QStringLiteral("multiinfo"));
|
|
const QString rpcArgKey(QStringLiteral("arg"));
|
|
const QString rpcArgArray(QStringLiteral("arg[]"));
|
|
const QString pkgbuildRequestType(QString("pkgbuild"));
|
|
|
|
QString UserRepository::m_aurBaseUrl = QStringLiteral("https://aur.archlinux.org");
|
|
QUrl UserRepository::m_aurRpcUrl = QUrl(QStringLiteral("https://aur.archlinux.org/rpc.php"));
|
|
QUrl UserRepository::m_aurPkgbuildUrl = QUrl(QStringLiteral("https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD"));
|
|
QUrl UserRepository::m_aurSrcInfoUrl = QUrl(QStringLiteral("https://aur.archlinux.org/cgit/aur.git/plain/.SRCINFO"));
|
|
QString UserRepository::m_aurSnapshotPath = QStringLiteral("https://aur.archlinux.org/cgit/aur.git/snapshot/%1.tar.gz");
|
|
|
|
AurPackageReply::AurPackageReply(const QList<QNetworkReply *> &networkReplies, const QStringList &requestedPackages, UserRepository *userRepo) :
|
|
PackageReply(networkReplies, requestedPackages, userRepo),
|
|
m_userRepo(userRepo)
|
|
{}
|
|
|
|
void AurPackageReply::processData(QNetworkReply *reply)
|
|
{
|
|
if(reply->error() == QNetworkReply::NoError) {
|
|
QJsonParseError error;
|
|
const auto doc = QJsonDocument::fromJson(reply->readAll(), &error);
|
|
if(error.error == QJsonParseError::NoError) {
|
|
auto now = DateTime::gmtNow();
|
|
QWriteLocker locker(m_repo->lock());
|
|
auto &packages = m_repo->packages();
|
|
for(const auto &result : doc.object().value(QStringLiteral("results")).toArray()) {
|
|
QJsonObject obj = result.toObject();
|
|
QString packageName = obj.value(QStringLiteral("Name")).toString();
|
|
if(!packageName.isEmpty()) {
|
|
auto &package = packages[packageName];
|
|
if(package) {
|
|
static_cast<AurPackage *>(package.get())->putJson(obj);
|
|
} else {
|
|
package = make_unique<AurPackage>(obj, static_cast<UserRepository *>(m_userRepo));
|
|
}
|
|
package->setTimeStamp(now);
|
|
}
|
|
}
|
|
} else {
|
|
m_error = QStringLiteral("Unable to parse JSON received from AUR: ") % error.errorString() % QStringLiteral(" at character ") % QString::number(error.offset);
|
|
}
|
|
} else {
|
|
m_error = QStringLiteral("Unable to request data from AUR via AurJson: ") + reply->errorString();
|
|
}
|
|
}
|
|
|
|
AurFullPackageReply::AurFullPackageReply(const QList<QNetworkReply *> &networkReplies, const QStringList &requestedPackages, UserRepository *userRepo) :
|
|
PackageReply(networkReplies, requestedPackages, userRepo),
|
|
m_userRepo(userRepo)
|
|
{}
|
|
|
|
void AurFullPackageReply::processData(QNetworkReply *reply)
|
|
{
|
|
if(reply->error() == QNetworkReply::NoError) {
|
|
//QBuffer buffer;
|
|
//buffer.setData(reply->readAll());
|
|
QTemporaryFile tmpFile;
|
|
tmpFile.setFileTemplate(QDir::tempPath() % QChar('/') % QStringLiteral("repoindex-XXXXXX.tar.gz"));
|
|
tmpFile.open();
|
|
tmpFile.write(reply->readAll());
|
|
tmpFile.seek(0);
|
|
//QFile testFile("/tmp/test.tar.gz");
|
|
//testFile.open(QFile::WriteOnly);
|
|
//testFile.write(buffer.data());
|
|
//testFile.close();
|
|
//testFile.open(QFile::ReadOnly);
|
|
//buffer.open(QBuffer::ReadOnly);
|
|
//KCompressionDevice dev(&tmpFile, false, KCompressionDevice::GZip);
|
|
KTar tar(tmpFile.fileName());
|
|
if(tar.open(QIODevice::ReadOnly)) {
|
|
const KArchiveDirectory *baseDir;
|
|
const KArchiveEntry *srcInfoEntry = nullptr;
|
|
if(!tar.directory()->entries().isEmpty()) {
|
|
const auto *baseEntry = tar.directory()->entry(tar.directory()->entries().front());
|
|
if(baseEntry && baseEntry->isDirectory()) {
|
|
baseDir = static_cast<const KArchiveDirectory *>(baseEntry);
|
|
srcInfoEntry = baseDir->entry(QStringLiteral(".SRCINFO"));
|
|
}
|
|
}
|
|
if(srcInfoEntry && srcInfoEntry->isFile()) {
|
|
const auto srcInfo = static_cast<const KArchiveFile *>(srcInfoEntry)->data();
|
|
QWriteLocker locker(m_userRepo->lock());
|
|
const auto packages = m_userRepo->addPackagesFromSrcInfo(srcInfo, DateTime::gmtNow());
|
|
// TODO: error handling
|
|
for(const auto &entryName : baseDir->entries()) {
|
|
if(entryName != QLatin1String(".SRCINFO")) {
|
|
const auto *entry = baseDir->entry(entryName);
|
|
if(entry->isFile()) {
|
|
const auto data = static_cast<const KArchiveFile *>(entry)->data();
|
|
for(Package *package : packages) {
|
|
package->putSourceFile(entry->name(), data);
|
|
}
|
|
} else {
|
|
// don't think it is required to read sub directories recursively (TODO?)
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
m_error = QStringLiteral("Aur tarball does not contain \".SRCINFO\".");
|
|
}
|
|
} else {
|
|
m_error = QStringLiteral("Unable to open tarball reply.");
|
|
}
|
|
} else {
|
|
m_error = QStringLiteral("Unable to request tarball from AUR: ") + reply->errorString();
|
|
}
|
|
if(!m_error.isEmpty()) {
|
|
cerr << shchar << m_error.toLocal8Bit().data() << endl;
|
|
}
|
|
}
|
|
|
|
AurSuggestionsReply::AurSuggestionsReply(QNetworkReply *networkReply, const QString &term, UserRepository *repo) :
|
|
SuggestionsReply(networkReply, term, repo)
|
|
{}
|
|
|
|
void AurSuggestionsReply::processData(QNetworkReply *reply)
|
|
{
|
|
if(reply->error() == QNetworkReply::NoError) {
|
|
QJsonParseError error;
|
|
//QByteArray data = m_networkReply->readAll();
|
|
//cerr << shchar << "AUR reply: " << data.data() << endl;
|
|
//const QJsonDocument doc = QJsonDocument::fromJson(data, &error);
|
|
const auto doc = QJsonDocument::fromJson(reply->readAll(), &error);
|
|
if(error.error == QJsonParseError::NoError) {
|
|
QWriteLocker locker(m_repo->lock());
|
|
auto &packages = m_repo->packages();
|
|
if(doc.isObject()) {
|
|
for(const auto &result : doc.object().value(QStringLiteral("results")).toArray()) {
|
|
QJsonObject obj = result.toObject();
|
|
QString packageName = obj.value(QStringLiteral("Name")).toString();
|
|
if(!packageName.isEmpty()) {
|
|
auto &package = packages[packageName];
|
|
if(package) {
|
|
static_cast<AurPackage *>(package.get())->putJson(obj);
|
|
} else {
|
|
package = make_unique<AurPackage>(obj, static_cast<UserRepository *>(m_repo));
|
|
}
|
|
}
|
|
}
|
|
} else if(doc.isArray()) {
|
|
for(const auto &suggestion : doc.array()) {
|
|
const auto suggestionString = suggestion.toString();
|
|
if(!suggestionString.isEmpty()) {
|
|
auto &package = packages[suggestionString];
|
|
if(!package) {
|
|
package = make_unique<AurPackage>(suggestionString, static_cast<UserRepository *>(m_repo));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
m_error = QStringLiteral("Unable to parse JSON received from AUR: ") % error.errorString() % QStringLiteral(" at character ") % QString::number(error.offset);
|
|
}
|
|
} else {
|
|
m_error = QStringLiteral("Unable to request data from AUR: ") + reply->errorString();
|
|
}
|
|
}
|
|
|
|
UserRepository::UserRepository(QObject *parent) :
|
|
Repository(QStringLiteral("AUR"), invalidIndex, parent)
|
|
{
|
|
m_description = QStringLiteral("Arch User Repository");
|
|
m_maxPackageAge = TimeSpan::fromDays(1.0);
|
|
}
|
|
|
|
RepositoryType UserRepository::type() const
|
|
{
|
|
return RepositoryType::UserRepository;
|
|
}
|
|
|
|
PackageDetailAvailability UserRepository::requestsRequired(PackageDetail packageDetail) const
|
|
{
|
|
switch(packageDetail) {
|
|
case PackageDetail::Basics:
|
|
return PackageDetailAvailability::Request;
|
|
case PackageDetail::Dependencies:
|
|
case PackageDetail::SourceInfo:
|
|
case PackageDetail::AllAvailable:
|
|
return PackageDetailAvailability::FullRequest;
|
|
default:
|
|
return PackageDetailAvailability::Never;
|
|
}
|
|
}
|
|
|
|
AurSuggestionsReply *UserRepository::requestSuggestions(const QString &term)
|
|
{
|
|
size_t remainingSuggestions = 20;
|
|
//for(auto i = packages().lower_bound(term), end = packages().cend(); i != end && remainingSuggestions && i->first.startsWith(term, Qt::CaseInsensitive); ++i, --remainingSuggestions);
|
|
for(const auto &pkgEntry : packages()) {
|
|
if(pkgEntry.first.contains(term, Qt::CaseInsensitive)) {
|
|
if(!(--remainingSuggestions)) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if(remainingSuggestions && !m_requestedSuggestions.contains(term)) {
|
|
m_requestedSuggestions << term;
|
|
auto url = m_aurRpcUrl;
|
|
QUrlQuery query;
|
|
query.addQueryItem(rpcRequestTypeKey, term.size() < 3 ? rpcRequestTypeSuggest : rpcRequestTypeSearch);
|
|
query.addQueryItem(rpcArgKey, term);
|
|
url.setQuery(query);
|
|
return new AurSuggestionsReply(networkAccessManager().get(QNetworkRequest(url)), term, this);
|
|
} else {
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
AurPackageReply *UserRepository::requestPackageInfo(const QStringList &packageNames, bool forceUpdate)
|
|
{
|
|
if(packageNames.isEmpty()) {
|
|
return nullptr;
|
|
}
|
|
|
|
QList<QNetworkReply *> replies;
|
|
replies.reserve(1 + packageNames.size() / 200);
|
|
|
|
QUrlQuery query;
|
|
size_t queryItems = 0;
|
|
for(const auto &packageName : packageNames) {
|
|
try {
|
|
const auto &pkg = m_packages.at(packageName);
|
|
if(!pkg->hasGeneralInfo() || forceUpdate) {
|
|
query.addQueryItem(rpcArgArray, packageName);
|
|
}
|
|
} catch(const out_of_range &) {
|
|
query.addQueryItem(rpcArgArray, packageName);
|
|
}
|
|
|
|
if(++queryItems > 200) {
|
|
auto url = m_aurRpcUrl;
|
|
query.addQueryItem(rpcRequestTypeKey, rpcRequestTypeMultiInfo);
|
|
url.setQuery(query);
|
|
replies << networkAccessManager().get(QNetworkRequest(url));
|
|
|
|
queryItems = 0;
|
|
query.clear();
|
|
}
|
|
}
|
|
|
|
if(!query.isEmpty()) {
|
|
auto url = m_aurRpcUrl;
|
|
query.addQueryItem(rpcRequestTypeKey, rpcRequestTypeMultiInfo);
|
|
url.setQuery(query);
|
|
replies << networkAccessManager().get(QNetworkRequest(url));
|
|
}
|
|
|
|
if(replies.isEmpty()) {
|
|
return nullptr;
|
|
} else {
|
|
return new AurPackageReply(replies, packageNames, this);
|
|
}
|
|
}
|
|
|
|
AurFullPackageReply *UserRepository::requestFullPackageInfo(const QStringList &packageNames, bool forceUpdate)
|
|
{
|
|
QList<QNetworkReply *> replies;
|
|
for(const auto &packageName : packageNames) {
|
|
try {
|
|
const auto &pkg = m_packages.at(packageName);
|
|
if(!pkg->hasAllGeneralInfo() || !pkg->hasSourceRelatedMetaData() || forceUpdate) {
|
|
if(pkg->tarUrl().isEmpty()) {
|
|
replies << networkAccessManager().get(QNetworkRequest(m_aurSnapshotPath.arg(pkg->name())));
|
|
} else {
|
|
replies << networkAccessManager().get(QNetworkRequest(m_aurBaseUrl + pkg->tarUrl()));
|
|
}
|
|
}
|
|
} catch(const out_of_range &) {
|
|
replies << networkAccessManager().get(QNetworkRequest(m_aurSnapshotPath.arg(packageName)));
|
|
}
|
|
}
|
|
if(replies.isEmpty()) {
|
|
return nullptr;
|
|
} else {
|
|
return new AurFullPackageReply(replies, packageNames, this);
|
|
}
|
|
}
|
|
|
|
unique_ptr<Package> UserRepository::emptyPackage()
|
|
{
|
|
return make_unique<AurPackage>(this);
|
|
}
|
|
|
|
} // namespace Alpm
|
|
|