feat(clone): speedup maintainer and universe clone query

There is a single endpoint now to list all pkgbases and their current
maintainers. Use this endpoint for speeding up the clone of all packages
of a maintainer by only issuing a single API call.

Component: pkgctl repo clone
Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

src/lib/api/archweb.sh

@@ -0,0 +1,57 @@
+#!/hint/bash
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+[[ -z ${DEVTOOLS_INCLUDE_API_ARCHWEB_SH:-} ]] || return 0
+DEVTOOLS_INCLUDE_API_ARCHWEB_SH=1
+
+_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
+# shellcheck source=src/lib/common.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+
+set -e
+set -o pipefail
+
+
+archweb_query_all_packages() {
+	[[ -z ${WORKDIR:-} ]] && setup_workdir
+
+	stat_busy "Query all released packages"
+	mapfile -t pkgbases < <(
+		curl --location --show-error --no-progress-meter --fail --retry 3 --retry-delay 3 \
+			"${PKGBASE_MAINTAINER_URL}" 2> "${WORKDIR}/error" \
+			| jq --raw-output --exit-status 'keys[]' 2> "${WORKDIR}/error"
+	)
+	if ! wait $!; then
+		stat_failed
+		print_workdir_error
+		return 1
+	fi
+	stat_done
+
+	printf "%s\n" "${pkgbases[@]}"
+	return 0
+}
+
+
+archweb_query_maintainer_packages() {
+	local maintainer=$1
+
+	[[ -z ${WORKDIR:-} ]] && setup_workdir
+
+	stat_busy "Query maintainer packages"
+	mapfile -t pkgbases < <(
+		curl --location --show-error --no-progress-meter --fail --retry 3 --retry-delay 3 \
+			"${PKGBASE_MAINTAINER_URL}" 2> "${WORKDIR}/error" \
+			| jq --raw-output --exit-status '. as $parent | keys[] | select(. as $key | $parent[$key] | index("'"${maintainer}"'"))' 2> "${WORKDIR}/error"
+	)
+	if ! wait $!; then
+		stat_failed
+		print_workdir_error
+		return 1
+	fi
+	stat_done
+
+	printf "%s\n" "${pkgbases[@]}"
+	return 0
+}

src/lib/common.sh

@@ -28,6 +28,7 @@ export GIT_PACKAGING_NAMESPACE_ID=11323
 export GIT_PACKAGING_URL_SSH="git@${GITLAB_HOST}:${GIT_PACKAGING_NAMESPACE}"
 export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE}"
 export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
+export PKGBASE_MAINTAINER_URL=https://archlinux.org/packages/pkgbase-maintainer
 
 # check if messages are to be printed using color
 if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
@@ -53,6 +54,11 @@ stat_done() {
 	printf "${BOLD}done${ALL_OFF}\n" >&2
 }
 
+stat_failed() {
+	# shellcheck disable=2059
+	printf "${BOLD}${RED}failed${ALL_OFF}\n" >&2
+}
+
 msg_success() {
 	local msg=$1
 	local padding
@@ -77,6 +83,15 @@ msg_warn() {
 	printf "%s %s\n" "${padding}${YELLOW}!${ALL_OFF}" "${msg}" >&2
 }
 
+print_workdir_error() {
+	if [[ ! -f "${WORKDIR}"/error ]]; then
+		return
+	fi
+	while read -r LINE; do
+		error '%s' "${LINE}"
+	done < "${WORKDIR}/error"
+}
+
 _setup_workdir=false
 setup_workdir() {
 	[[ -z ${WORKDIR:-} ]] && WORKDIR=$(mktemp -d --tmpdir "${0##*/}.XXXXXXXXXX")

src/lib/repo/clone.sh

@@ -8,6 +8,8 @@ DEVTOOLS_INCLUDE_REPO_CLONE_SH=1
 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
 # shellcheck source=src/lib/common.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/api/archweb.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/archweb.sh
 # shellcheck source=src/lib/api/gitlab.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh
 # shellcheck source=src/lib/repo/configure.sh
@@ -18,6 +20,7 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh
 source /usr/share/makepkg/util/message.sh
 
 set -e
+set -o pipefail
 
 
 pkgctl_repo_clone_usage() {
@@ -137,33 +140,18 @@ pkgctl_repo_clone() {
 
 	# Query packages of a maintainer
 	if [[ -n ${MAINTAINER} ]]; then
-		stat_busy "Query packages"
-		max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}" | jq -r '.num_pages')
-		if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then
-			stat_done
-			warning "found no packages for maintainer ${MAINTAINER}"
-			exit 0
+		mapfile -t pkgbases < <(archweb_query_maintainer_packages "${MAINTAINER}")
+		if ! wait $!; then
+			die "Failed to query maintainer packages"
 		fi
-		mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
-			curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}&page=${page}" | jq -r '.results[].pkgbase'
-			stat_progress
-		done | sort --unique)
-		stat_done
 	fi
 
 	# Query all released packages
 	if (( CLONE_ALL )); then
-		stat_busy "Query all released packages"
-		max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name" | jq -r '.num_pages')
-		if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then
-			stat_done
-			die "failed to query packages"
+		mapfile -t pkgbases < <(archweb_query_all_packages)
+		if ! wait $!; then
+			die "Failed to query all packages"
 		fi
-		mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
-			curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&page=${page}" | jq -r '.results[].pkgbase'
-			stat_progress
-		done | sort --unique)
-		stat_done
 	fi
 
 	# parallelization

src/pkgctl.in

@@ -39,6 +39,8 @@ fi
 
 export _DEVTOOLS_COMMAND='pkgctl'
 
+setup_workdir
+
 load_devtools_config
 
 # command checking