2016-02-20 03:00:34 +01:00
# include "./userrepository.h"
# include "./networkaccessmanager.h"
2015-09-04 14:37:01 +02:00
2015-09-07 19:36:45 +02:00
# include "../alpm/aurpackage.h"
2016-02-12 01:05:08 +01:00
# include "../alpm/config.h"
2015-09-04 14:37:01 +02:00
2015-09-06 15:34:39 +02:00
# include <KTar>
2015-09-27 19:29:45 +02:00
# include <KArchiveFile>
2015-10-28 19:55:36 +01:00
# include <KFilterDev>
2015-09-06 15:34:39 +02:00
2015-09-04 14:37:01 +02:00
# include <QStringBuilder>
# include <QUrlQuery>
# include <QNetworkRequest>
# include <QNetworkReply>
# include <QNetworkAccessManager>
# include <QJsonDocument>
# include <QJsonObject>
# include <QJsonArray>
2015-10-28 19:55:36 +01:00
# include <QDir>
# include <QTemporaryFile>
# include <QStringBuilder>
2015-09-04 14:37:01 +02:00
2016-02-12 01:05:08 +01:00
# include <iostream>
2017-03-25 19:49:41 +01:00
# include <memory>
2016-02-12 01:05:08 +01:00
2015-09-04 14:37:01 +02:00
using namespace std ;
2019-06-10 22:51:09 +02:00
using namespace CppUtilities ;
2015-09-04 14:37:01 +02:00
2015-09-05 17:25:05 +02:00
namespace RepoIndex {
2015-09-04 14:37:01 +02:00
const char * requestTypeProp = " type " ;
const QString rpcRequestTypeKey ( QStringLiteral ( " type " ) ) ;
2015-09-27 19:29:45 +02:00
const QString rpcRequestTypeSearch ( QStringLiteral ( " search " ) ) ;
2015-09-04 14:37:01 +02:00
const QString rpcRequestTypeSuggest ( QStringLiteral ( " suggest " ) ) ;
const QString rpcRequestTypeMultiInfo ( QStringLiteral ( " multiinfo " ) ) ;
const QString rpcArgKey ( QStringLiteral ( " arg " ) ) ;
const QString rpcArgArray ( QStringLiteral ( " arg[] " ) ) ;
const QString pkgbuildRequestType ( QString ( " pkgbuild " ) ) ;
2015-10-28 19:55:36 +01:00
QString UserRepository : : m_aurBaseUrl = QStringLiteral ( " https://aur.archlinux.org " ) ;
2015-09-04 14:37:01 +02:00
QUrl UserRepository : : m_aurRpcUrl = QUrl ( QStringLiteral ( " https://aur.archlinux.org/rpc.php " ) ) ;
QUrl UserRepository : : m_aurPkgbuildUrl = QUrl ( QStringLiteral ( " https://aur.archlinux.org/cgit/aur.git/plain/PKGBUILD " ) ) ;
QUrl UserRepository : : m_aurSrcInfoUrl = QUrl ( QStringLiteral ( " https://aur.archlinux.org/cgit/aur.git/plain/.SRCINFO " ) ) ;
2015-09-05 17:25:05 +02:00
QString UserRepository : : m_aurSnapshotPath = QStringLiteral ( " https://aur.archlinux.org/cgit/aur.git/snapshot/%1.tar.gz " ) ;
2015-09-04 14:37:01 +02:00
2016-09-28 22:23:09 +02:00
AurPackageReply : : AurPackageReply ( const QList < QNetworkReply * > & networkReplies , const QStringList & requestedPackages , UserRepository * userRepo ) :
PackageReply ( networkReplies , requestedPackages , userRepo ) ,
2015-09-04 14:37:01 +02:00
m_userRepo ( userRepo )
{ }
2015-12-08 00:24:45 +01:00
void AurPackageReply : : processData ( QNetworkReply * reply )
2015-09-04 14:37:01 +02:00
{
2015-09-05 17:25:05 +02:00
if ( reply - > error ( ) = = QNetworkReply : : NoError ) {
2015-09-04 14:37:01 +02:00
QJsonParseError error ;
2015-09-29 21:52:30 +02:00
const auto doc = QJsonDocument : : fromJson ( reply - > readAll ( ) , & error ) ;
2015-09-04 14:37:01 +02:00
if ( error . error = = QJsonParseError : : NoError ) {
2016-02-27 21:00:58 +01:00
auto now = DateTime : : gmtNow ( ) ;
2015-11-03 18:19:24 +01:00
QWriteLocker locker ( m_repo - > lock ( ) ) ;
auto & packages = m_repo - > packages ( ) ;
2015-09-04 14:37:01 +02:00
for ( const auto & result : doc . object ( ) . value ( QStringLiteral ( " results " ) ) . toArray ( ) ) {
2015-09-27 19:29:45 +02:00
QJsonObject obj = result . toObject ( ) ;
QString packageName = obj . value ( QStringLiteral ( " Name " ) ) . toString ( ) ;
if ( ! packageName . isEmpty ( ) ) {
2015-09-29 21:52:30 +02:00
auto & package = packages [ packageName ] ;
2015-09-27 19:29:45 +02:00
if ( package ) {
static_cast < AurPackage * > ( package . get ( ) ) - > putJson ( obj ) ;
} else {
package = make_unique < AurPackage > ( obj , static_cast < UserRepository * > ( m_userRepo ) ) ;
}
2016-02-27 21:00:58 +01:00
package - > setTimeStamp ( now ) ;
2015-09-04 14:37:01 +02:00
}
}
} else {
2015-11-03 18:19:24 +01:00
m_error = QStringLiteral ( " Unable to parse JSON received from AUR: " ) % error . errorString ( ) % QStringLiteral ( " at character " ) % QString : : number ( error . offset ) ;
2015-09-04 14:37:01 +02:00
}
} else {
2015-11-03 18:19:24 +01:00
m_error = QStringLiteral ( " Unable to request data from AUR via AurJson: " ) + reply - > errorString ( ) ;
2015-09-04 14:37:01 +02:00
}
}
2015-09-29 21:52:30 +02:00
AurFullPackageReply : : AurFullPackageReply ( const QList < QNetworkReply * > & networkReplies , const QStringList & requestedPackages , UserRepository * userRepo ) :
PackageReply ( networkReplies , requestedPackages , userRepo ) ,
2015-09-05 17:25:05 +02:00
m_userRepo ( userRepo )
{ }
2015-12-08 00:24:45 +01:00
void AurFullPackageReply : : processData ( QNetworkReply * reply )
2015-09-05 17:25:05 +02:00
{
2015-09-27 19:29:45 +02:00
if ( reply - > error ( ) = = QNetworkReply : : NoError ) {
2015-10-28 19:55:36 +01:00
//QBuffer buffer;
//buffer.setData(reply->readAll());
QTemporaryFile tmpFile ;
2015-12-25 01:58:19 +01:00
tmpFile . setFileTemplate ( QDir : : tempPath ( ) % QChar ( ' / ' ) % QStringLiteral ( " repoindex-XXXXXX.tar.gz " ) ) ;
2015-10-28 19:55:36 +01:00
tmpFile . open ( ) ;
tmpFile . write ( reply - > readAll ( ) ) ;
tmpFile . seek ( 0 ) ;
//QFile testFile("/tmp/test.tar.gz");
//testFile.open(QFile::WriteOnly);
//testFile.write(buffer.data());
//testFile.close();
//testFile.open(QFile::ReadOnly);
//buffer.open(QBuffer::ReadOnly);
2015-10-29 20:43:30 +01:00
//KCompressionDevice dev(&tmpFile, false, KCompressionDevice::GZip);
2015-10-28 19:55:36 +01:00
KTar tar ( tmpFile . fileName ( ) ) ;
2015-09-27 19:29:45 +02:00
if ( tar . open ( QIODevice : : ReadOnly ) ) {
2015-10-28 19:55:36 +01:00
const KArchiveDirectory * baseDir ;
const KArchiveEntry * srcInfoEntry = nullptr ;
if ( ! tar . directory ( ) - > entries ( ) . isEmpty ( ) ) {
const auto * baseEntry = tar . directory ( ) - > entry ( tar . directory ( ) - > entries ( ) . front ( ) ) ;
if ( baseEntry & & baseEntry - > isDirectory ( ) ) {
baseDir = static_cast < const KArchiveDirectory * > ( baseEntry ) ;
srcInfoEntry = baseDir - > entry ( QStringLiteral ( " .SRCINFO " ) ) ;
}
}
2015-09-27 19:29:45 +02:00
if ( srcInfoEntry & & srcInfoEntry - > isFile ( ) ) {
2015-10-28 19:55:36 +01:00
const auto srcInfo = static_cast < const KArchiveFile * > ( srcInfoEntry ) - > data ( ) ;
2015-11-03 18:19:24 +01:00
QWriteLocker locker ( m_userRepo - > lock ( ) ) ;
2016-02-27 21:00:58 +01:00
const auto packages = m_userRepo - > addPackagesFromSrcInfo ( srcInfo , DateTime : : gmtNow ( ) ) ;
2015-09-27 19:29:45 +02:00
// TODO: error handling
for ( const auto & entryName : baseDir - > entries ( ) ) {
if ( entryName ! = QLatin1String ( " .SRCINFO " ) ) {
const auto * entry = baseDir - > entry ( entryName ) ;
if ( entry - > isFile ( ) ) {
2015-10-28 19:55:36 +01:00
const auto data = static_cast < const KArchiveFile * > ( entry ) - > data ( ) ;
for ( Package * package : packages ) {
package - > putSourceFile ( entry - > name ( ) , data ) ;
2015-09-27 19:29:45 +02:00
}
} else {
2016-02-12 01:05:08 +01:00
// don't think it is required to read sub directories recursively (TODO?)
2015-09-27 19:29:45 +02:00
}
}
}
} else {
2015-11-03 18:19:24 +01:00
m_error = QStringLiteral ( " Aur tarball does not contain \" .SRCINFO \" . " ) ;
2015-09-27 19:29:45 +02:00
}
} else {
2015-11-03 18:19:24 +01:00
m_error = QStringLiteral ( " Unable to open tarball reply. " ) ;
2015-09-27 19:29:45 +02:00
}
} else {
2015-11-03 18:19:24 +01:00
m_error = QStringLiteral ( " Unable to request tarball from AUR: " ) + reply - > errorString ( ) ;
2015-09-27 19:29:45 +02:00
}
2015-10-28 19:55:36 +01:00
if ( ! m_error . isEmpty ( ) ) {
2016-02-20 03:00:34 +01:00
cerr < < shchar < < m_error . toLocal8Bit ( ) . data ( ) < < endl ;
2015-10-28 19:55:36 +01:00
}
2015-09-05 17:25:05 +02:00
}
2015-09-21 22:16:19 +02:00
AurSuggestionsReply : : AurSuggestionsReply ( QNetworkReply * networkReply , const QString & term , UserRepository * repo ) :
SuggestionsReply ( networkReply , term , repo )
2015-09-04 14:37:01 +02:00
{ }
2015-12-08 00:24:45 +01:00
void AurSuggestionsReply : : processData ( QNetworkReply * reply )
2015-09-04 14:37:01 +02:00
{
2015-09-05 17:25:05 +02:00
if ( reply - > error ( ) = = QNetworkReply : : NoError ) {
2015-09-04 14:37:01 +02:00
QJsonParseError error ;
//QByteArray data = m_networkReply->readAll();
2015-09-05 17:25:05 +02:00
//cerr << shchar << "AUR reply: " << data.data() << endl;
2015-09-04 14:37:01 +02:00
//const QJsonDocument doc = QJsonDocument::fromJson(data, &error);
2015-09-29 21:52:30 +02:00
const auto doc = QJsonDocument : : fromJson ( reply - > readAll ( ) , & error ) ;
2015-09-04 14:37:01 +02:00
if ( error . error = = QJsonParseError : : NoError ) {
2015-11-03 18:19:24 +01:00
QWriteLocker locker ( m_repo - > lock ( ) ) ;
2015-09-21 22:16:19 +02:00
auto & packages = m_repo - > packages ( ) ;
2015-09-27 19:29:45 +02:00
if ( doc . isObject ( ) ) {
for ( const auto & result : doc . object ( ) . value ( QStringLiteral ( " results " ) ) . toArray ( ) ) {
QJsonObject obj = result . toObject ( ) ;
QString packageName = obj . value ( QStringLiteral ( " Name " ) ) . toString ( ) ;
if ( ! packageName . isEmpty ( ) ) {
auto & package = packages [ packageName ] ;
if ( package ) {
static_cast < AurPackage * > ( package . get ( ) ) - > putJson ( obj ) ;
} else {
package = make_unique < AurPackage > ( obj , static_cast < UserRepository * > ( m_repo ) ) ;
}
}
}
} else if ( doc . isArray ( ) ) {
for ( const auto & suggestion : doc . array ( ) ) {
const auto suggestionString = suggestion . toString ( ) ;
if ( ! suggestionString . isEmpty ( ) ) {
auto & package = packages [ suggestionString ] ;
if ( ! package ) {
package = make_unique < AurPackage > ( suggestionString , static_cast < UserRepository * > ( m_repo ) ) ;
}
2015-09-21 22:16:19 +02:00
}
}
}
2015-09-04 14:37:01 +02:00
} else {
2015-11-03 18:19:24 +01:00
m_error = QStringLiteral ( " Unable to parse JSON received from AUR: " ) % error . errorString ( ) % QStringLiteral ( " at character " ) % QString : : number ( error . offset ) ;
2015-09-04 14:37:01 +02:00
}
} else {
2015-11-03 18:19:24 +01:00
m_error = QStringLiteral ( " Unable to request data from AUR: " ) + reply - > errorString ( ) ;
2015-09-04 14:37:01 +02:00
}
}
2016-02-20 03:00:34 +01:00
UserRepository : : UserRepository ( QObject * parent ) :
Repository ( QStringLiteral ( " AUR " ) , invalidIndex , parent )
2015-09-04 14:37:01 +02:00
{
m_description = QStringLiteral ( " Arch User Repository " ) ;
2016-01-18 20:34:29 +01:00
m_maxPackageAge = TimeSpan : : fromDays ( 1.0 ) ;
2015-09-04 14:37:01 +02:00
}
RepositoryType UserRepository : : type ( ) const
{
return RepositoryType : : UserRepository ;
}
2015-09-05 17:25:05 +02:00
PackageDetailAvailability UserRepository : : requestsRequired ( PackageDetail packageDetail ) const
2015-09-04 14:37:01 +02:00
{
2015-09-05 17:25:05 +02:00
switch ( packageDetail ) {
case PackageDetail : : Basics :
return PackageDetailAvailability : : Request ;
case PackageDetail : : Dependencies :
case PackageDetail : : SourceInfo :
2015-09-11 21:59:47 +02:00
case PackageDetail : : AllAvailable :
2015-09-05 17:25:05 +02:00
return PackageDetailAvailability : : FullRequest ;
2015-09-11 21:59:47 +02:00
default :
2015-09-05 17:25:05 +02:00
return PackageDetailAvailability : : Never ;
}
2015-09-04 14:37:01 +02:00
}
2015-09-27 19:29:45 +02:00
AurSuggestionsReply * UserRepository : : requestSuggestions ( const QString & term )
2015-09-04 14:37:01 +02:00
{
2015-09-21 22:16:19 +02:00
size_t remainingSuggestions = 20 ;
2015-09-27 19:29:45 +02:00
//for(auto i = packages().lower_bound(term), end = packages().cend(); i != end && remainingSuggestions && i->first.startsWith(term, Qt::CaseInsensitive); ++i, --remainingSuggestions);
for ( const auto & pkgEntry : packages ( ) ) {
2015-12-08 00:24:45 +01:00
if ( pkgEntry . first . contains ( term , Qt : : CaseInsensitive ) ) {
2015-09-27 19:29:45 +02:00
if ( ! ( - - remainingSuggestions ) ) {
break ;
}
}
}
2015-12-08 00:24:45 +01:00
if ( remainingSuggestions & & ! m_requestedSuggestions . contains ( term ) ) {
m_requestedSuggestions < < term ;
2015-09-21 22:16:19 +02:00
auto url = m_aurRpcUrl ;
QUrlQuery query ;
2015-09-27 19:29:45 +02:00
query . addQueryItem ( rpcRequestTypeKey , term . size ( ) < 3 ? rpcRequestTypeSuggest : rpcRequestTypeSearch ) ;
2015-09-21 22:16:19 +02:00
query . addQueryItem ( rpcArgKey , term ) ;
url . setQuery ( query ) ;
2016-02-20 03:00:34 +01:00
return new AurSuggestionsReply ( networkAccessManager ( ) . get ( QNetworkRequest ( url ) ) , term , this ) ;
2015-09-21 22:16:19 +02:00
} else {
return nullptr ;
}
2015-09-04 14:37:01 +02:00
}
2015-09-27 19:29:45 +02:00
AurPackageReply * UserRepository : : requestPackageInfo ( const QStringList & packageNames , bool forceUpdate )
2015-09-04 14:37:01 +02:00
{
2016-09-28 22:23:09 +02:00
if ( packageNames . isEmpty ( ) ) {
return nullptr ;
}
QList < QNetworkReply * > replies ;
replies . reserve ( 1 + packageNames . size ( ) / 200 ) ;
2015-09-04 14:37:01 +02:00
QUrlQuery query ;
2016-09-28 22:23:09 +02:00
size_t queryItems = 0 ;
2015-09-04 14:37:01 +02:00
for ( const auto & packageName : packageNames ) {
2015-09-27 19:29:45 +02:00
try {
const auto & pkg = m_packages . at ( packageName ) ;
if ( ! pkg - > hasGeneralInfo ( ) | | forceUpdate ) {
query . addQueryItem ( rpcArgArray , packageName ) ;
}
} catch ( const out_of_range & ) {
2015-09-04 14:37:01 +02:00
query . addQueryItem ( rpcArgArray , packageName ) ;
}
2016-09-28 22:23:09 +02:00
if ( + + queryItems > 200 ) {
auto url = m_aurRpcUrl ;
query . addQueryItem ( rpcRequestTypeKey , rpcRequestTypeMultiInfo ) ;
url . setQuery ( query ) ;
replies < < networkAccessManager ( ) . get ( QNetworkRequest ( url ) ) ;
queryItems = 0 ;
query . clear ( ) ;
}
2015-09-04 14:37:01 +02:00
}
2016-09-28 22:23:09 +02:00
if ( ! query . isEmpty ( ) ) {
2015-09-04 14:37:01 +02:00
auto url = m_aurRpcUrl ;
query . addQueryItem ( rpcRequestTypeKey , rpcRequestTypeMultiInfo ) ;
url . setQuery ( query ) ;
2016-09-28 22:23:09 +02:00
replies < < networkAccessManager ( ) . get ( QNetworkRequest ( url ) ) ;
}
if ( replies . isEmpty ( ) ) {
return nullptr ;
} else {
return new AurPackageReply ( replies , packageNames , this ) ;
2015-09-04 14:37:01 +02:00
}
}
2015-09-27 19:29:45 +02:00
AurFullPackageReply * UserRepository : : requestFullPackageInfo ( const QStringList & packageNames , bool forceUpdate )
2015-09-04 14:37:01 +02:00
{
2015-09-05 17:25:05 +02:00
QList < QNetworkReply * > replies ;
for ( const auto & packageName : packageNames ) {
try {
const auto & pkg = m_packages . at ( packageName ) ;
2015-12-08 00:24:45 +01:00
if ( ! pkg - > hasAllGeneralInfo ( ) | | ! pkg - > hasSourceRelatedMetaData ( ) | | forceUpdate ) {
2015-09-05 17:25:05 +02:00
if ( pkg - > tarUrl ( ) . isEmpty ( ) ) {
2016-02-20 03:00:34 +01:00
replies < < networkAccessManager ( ) . get ( QNetworkRequest ( m_aurSnapshotPath . arg ( pkg - > name ( ) ) ) ) ;
2015-09-05 17:25:05 +02:00
} else {
2016-02-20 03:00:34 +01:00
replies < < networkAccessManager ( ) . get ( QNetworkRequest ( m_aurBaseUrl + pkg - > tarUrl ( ) ) ) ;
2015-09-05 17:25:05 +02:00
}
}
} catch ( const out_of_range & ) {
2016-02-20 03:00:34 +01:00
replies < < networkAccessManager ( ) . get ( QNetworkRequest ( m_aurSnapshotPath . arg ( packageName ) ) ) ;
2015-09-04 14:37:01 +02:00
}
}
2015-09-11 21:59:47 +02:00
if ( replies . isEmpty ( ) ) {
return nullptr ;
} else {
2015-09-29 21:52:30 +02:00
return new AurFullPackageReply ( replies , packageNames , this ) ;
2015-09-11 21:59:47 +02:00
}
}
unique_ptr < Package > UserRepository : : emptyPackage ( )
{
return make_unique < AurPackage > ( this ) ;
2015-09-04 14:37:01 +02:00
}
} // namespace Alpm