pkgrepo: add option to clone all existing packages

This can be very handy for cache warming on the repo server or
to perform mass operations on all PKGBUILDs.

Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

contrib/completion/zsh/_devtools.in

@@ -24,6 +24,7 @@ _pkgrepo_cmds=(
 _pkgrepo_clone_args=(
 	'(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:'
 	'(-u --unprivileged)'{-u,--unprivileged}'[Read-only access without packager info as Git author]'
+	'--universe[Clone all existing packages, useful for cache warming]'
 	'(-h --help)'{-h,--help}'[Display usage]'
 	'*:packages:_devtools_completions_all_packages'
 )

src/pkgrepo.in

@@ -49,6 +49,7 @@ usage_clone() {
 		    -m, --maintainer=NAME  Clone all packages of the named maintainer
 		    -u, --unprivileged     Clone package with read-only access and without
 		                           packager info as Git author.
+		    --universe             Clone all existing packages, useful for cache warming
 		    -h, --help             Show this help text
 _EOF_
 }
@@ -83,6 +84,7 @@ CONFIGURE=0
 # options
 GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_SSH}
 UNPRIVILEGED=0
+CLONE_ALL=0
 MAINTAINER=
 PACKAGER_NAME=
 PACKAGER_EMAIL=
@@ -137,6 +139,10 @@ if (( CLONE )); then
 				MAINTAINER="${1#*=}"
 				shift
 				;;
+			--all)
+				CLONE_ALL=1
+				shift
+				;;
 			--)
 				shift
 				break
@@ -214,6 +220,17 @@ if [[ -n ${MAINTAINER} ]]; then
 	stat_done
 fi
 
+# Query all released packages
+if (( CLONE_ALL )); then
+	stat_busy "Query all released packages"
+	max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name" | jq -r '.num_pages')
+	mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
+		curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&page=${page}" | jq -r '.results[].pkgbase'
+		stat_progress
+	done | sort --unique)
+	stat_done
+fi
+
 for pkgbase in "${pkgbases[@]}"; do
 	if (( CLONE )); then
 		if [[ ! -d ${pkgbase} ]]; then