feature(parallel): run up to N jobs in parallel for repo clone/configure
Run up to N jobs in parallel. By default the number of jobs is equal to the number of available processing units. For sequential processing this option needs to be passed with 1. Signed-off-by: Levente Polyak <anthraxx@archlinux.org>
This commit is contained in:
Levente Polyak
parent
bf61b8472a
commit
6ce666a166
|
|
@ -268,17 +268,23 @@ _pkgctl_repo_clone_args=(
|
|||
--switch
|
||||
-u --unprivileged
|
||||
--universe
|
||||
-j --jobs
|
||||
-h --help
|
||||
)
|
||||
_pkgctl_repo_clone_args__maintainer_opts() { :; }
|
||||
_pkgctl_repo_clone_args_m_opts() { _pkgctl_repo_clone_args__maintainer_opts; }
|
||||
_pkgctl_repo_clone_args__switch_opts() { :; }
|
||||
_pkgctl_repo_clone_args__jobs_opts() { :; }
|
||||
_pkgctl_repo_clone_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; }
|
||||
_pkgctl_repo_clone_opts() { _devtools_completions_all_packages; }
|
||||
|
||||
|
||||
_pkgctl_repo_configure_args=(
|
||||
-j --jobs
|
||||
-h --help
|
||||
)
|
||||
_pkgctl_repo_configure_args__jobs_opts() { :; }
|
||||
_pkgctl_repo_configure_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; }
|
||||
_pkgctl_repo_configure_opts() { _filedir -d; }
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -110,11 +110,13 @@ _pkgctl_repo_clone_args=(
|
|||
'(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:'
|
||||
'--switch=[Switch the current working tree to a specified version]'
|
||||
'--universe[Clone all existing packages, useful for cache warming]'
|
||||
'(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
|
||||
'(-h --help)'{-h,--help}'[Display usage]'
|
||||
'*:packages:_devtools_completions_all_packages'
|
||||
)
|
||||
|
||||
_pkgctl_repo_configure_args=(
|
||||
'(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
|
||||
'(-h --help)'{-h,--help}'[Display usage]'
|
||||
'*:git_dir:_files -/'
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,11 @@ Options
|
|||
Switch to a specified version. The working tree and the index are updated to
|
||||
match the version.
|
||||
|
||||
*-j, --jobs* 'N'::
|
||||
Run up to N jobs in parallel. By default the number of jobs is equal to the
|
||||
number of available processing units. For sequential processing this option
|
||||
needs to be passed with 1.
|
||||
|
||||
*-h, --help*::
|
||||
Show a help text
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,11 @@ read-only HTTPS otherwise.
|
|||
Options
|
||||
-------
|
||||
|
||||
*-j, --jobs* 'N'::
|
||||
Run up to N jobs in parallel. By default the number of jobs is equal to the
|
||||
number of available processing units. For sequential processing this option
|
||||
needs to be passed with 1.
|
||||
|
||||
*-h, --help*::
|
||||
Show a help text
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE
|
|||
export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
|
||||
|
||||
# check if messages are to be printed using color
|
||||
if [[ -t 2 && "$TERM" != dumb ]]; then
|
||||
if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
|
||||
colorize
|
||||
else
|
||||
# shellcheck disable=2034
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ pkgctl_repo_clone_usage() {
|
|||
-m, --maintainer=NAME Clone all packages of the named maintainer
|
||||
--switch VERSION Switch the current working tree to a specified version
|
||||
--universe Clone all existing packages, useful for cache warming
|
||||
-j, --jobs N Run up to N jobs in parallel (default: $(nproc))
|
||||
-h, --help Show this help text
|
||||
|
||||
EXAMPLES
|
||||
|
|
@ -55,9 +56,11 @@ pkgctl_repo_clone() {
|
|||
local MAINTAINER=
|
||||
local VERSION=
|
||||
local CONFIGURE_OPTIONS=()
|
||||
local pkgbases
|
||||
local jobs=
|
||||
jobs=$(nproc)
|
||||
|
||||
# variables
|
||||
local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
|
||||
local project_path
|
||||
|
||||
while (( $# )); do
|
||||
|
|
@ -97,6 +100,11 @@ pkgctl_repo_clone() {
|
|||
CLONE_ALL=1
|
||||
shift
|
||||
;;
|
||||
-j|--jobs)
|
||||
(( $# <= 1 )) && die "missing argument for %s" "$1"
|
||||
jobs=$2
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
|
|
@ -142,12 +150,31 @@ pkgctl_repo_clone() {
|
|||
stat_done
|
||||
fi
|
||||
|
||||
# parallelization
|
||||
if [[ ${jobs} != 1 ]] && (( ${#pkgbases[@]} > 1 )); then
|
||||
# force colors in parallel if parent process is colorized
|
||||
if [[ -n ${BOLD} ]]; then
|
||||
export DEVTOOLS_COLOR=always
|
||||
fi
|
||||
# assign command options
|
||||
if [[ -n "${VERSION}" ]]; then
|
||||
command+=" --switch '${VERSION}'"
|
||||
fi
|
||||
if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${pkgbases[@]}"; then
|
||||
die 'Failed to clone some packages, please check the output'
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for pkgbase in "${pkgbases[@]}"; do
|
||||
if [[ ! -d ${pkgbase} ]]; then
|
||||
msg "Cloning ${pkgbase} ..."
|
||||
project_path=$(gitlab_project_name_to_path "${pkgbase}")
|
||||
remote_url="${GIT_REPO_BASE_URL}/${project_path}.git"
|
||||
git clone --origin origin "${remote_url}" "${pkgbase}"
|
||||
if ! git clone --origin origin "${remote_url}" "${pkgbase}"; then
|
||||
die 'failed to clone %s' "${pkgbase}"
|
||||
fi
|
||||
else
|
||||
warning "Skip cloning ${pkgbase}: Directory exists"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ pkgctl_repo_configure_usage() {
|
|||
read-only HTTPS otherwise.
|
||||
|
||||
OPTIONS
|
||||
-j, --jobs N Run up to N jobs in parallel (default: $(nproc))
|
||||
-h, --help Show this help text
|
||||
|
||||
EXAMPLES
|
||||
|
|
@ -93,9 +94,12 @@ pkgctl_repo_configure() {
|
|||
local GIT_REPO_BASE_URL=${GIT_PACKAGING_URL_HTTPS}
|
||||
local official=0
|
||||
local proto=https
|
||||
local jobs=
|
||||
jobs=$(nproc)
|
||||
local paths=()
|
||||
|
||||
# variables
|
||||
local -r command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
|
||||
local path realpath pkgbase remote_url project_path
|
||||
local PACKAGER GPGKEY packager_name packager_email
|
||||
|
||||
|
|
@ -105,6 +109,11 @@ pkgctl_repo_configure() {
|
|||
pkgctl_repo_configure_usage
|
||||
exit 0
|
||||
;;
|
||||
-j|--jobs)
|
||||
(( $# <= 1 )) && die "missing argument for %s" "$1"
|
||||
jobs=$2
|
||||
shift 2
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
|
|
@ -157,10 +166,21 @@ pkgctl_repo_configure() {
|
|||
msg2 "protocol: ${YELLOW}${proto}${ALL_OFF}"
|
||||
fi
|
||||
|
||||
# parallelization
|
||||
if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then
|
||||
if [[ -n ${BOLD} ]]; then
|
||||
export DEVTOOLS_COLOR=always
|
||||
fi
|
||||
if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then
|
||||
die 'Failed to configure some packages, please check the output'
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for path in "${paths[@]}"; do
|
||||
if ! realpath=$(realpath -e "${path}"); then
|
||||
error "No such directory: ${path}"
|
||||
continue
|
||||
die "No such directory: ${path}"
|
||||
fi
|
||||
|
||||
pkgbase=$(basename "${realpath}")
|
||||
|
|
@ -168,8 +188,7 @@ pkgctl_repo_configure() {
|
|||
msg "Configuring ${pkgbase}"
|
||||
|
||||
if [[ ! -d "${path}/.git" ]]; then
|
||||
error "Not a Git repository: ${path}"
|
||||
continue
|
||||
die "Not a Git repository: ${path}"
|
||||
fi
|
||||
|
||||
pushd "${path}" >/dev/null
|
||||
|
|
|
|||
Loading…
Reference in New Issue