feat(util): parallelize srcinfo generation

Heavily improve the runtime of huge split packages, by creating an own
parallelized high level implementation of the makepkg low level building
blocks for srcinfo generation.

This reduces the runtine to generate the srcinfo file for thunderbird
from 24 seconds down to 1 second.
This commit is contained in:
Levente Polyak 2023-09-10 00:00:27 +02:00
parent 2b8033b911
commit b264c7f1c7
No known key found for this signature in database
GPG Key ID: FC1B547C8D8172C8
3 changed files with 78 additions and 6 deletions

View File

@ -5,9 +5,12 @@
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
# shellcheck source=src/lib/common.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
# shellcheck source=src/lib/util/srcinfo.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh
source /usr/share/makepkg/util/util.sh
source /usr/share/makepkg/srcinfo.sh
set -eo pipefail
check_pkgbuild_validity() {
@ -185,10 +188,9 @@ done
check_pkgbuild_validity
# auto generate .SRCINFO
stat_busy 'Generating .SRCINFO'
write_srcinfo_content > .SRCINFO
# shellcheck disable=SC2119
write_srcinfo_file
git add --force .SRCINFO
stat_done
if [[ -n $(git status --porcelain --untracked-files=no) ]]; then
stat_busy 'Staging files'

View File

@ -14,6 +14,8 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/release.sh
# shellcheck source=src/lib/util/git.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh
# shellcheck source=src/lib/util/srcinfo.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh
# shellcheck source=src/lib/util/pacman.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh
# shellcheck source=src/lib/valid-repos.sh
@ -26,8 +28,7 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-inspect.sh
source /usr/share/makepkg/util/config.sh
source /usr/share/makepkg/util/message.sh
set -e
set -o pipefail
set -eo pipefail
pkgctl_build_usage() {

69
src/lib/util/srcinfo.sh Normal file
View File

@ -0,0 +1,69 @@
#!/hint/bash
#
# SPDX-License-Identifier: GPL-3.0-or-later
[[ -z ${DEVTOOLS_INCLUDE_UTIL_SRCINFO_SH:-} ]] || return 0
DEVTOOLS_INCLUDE_UTIL_SRCINFO_SH=1
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
# shellcheck source=src/lib/common.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
source /usr/share/makepkg/util/util.sh
source /usr/share/makepkg/srcinfo.sh
set -eo pipefail
print_srcinfo() {
local pkgpath=${1:-.}
local outdir pkg pid
local pids=()
# source the PKGBUILD
# shellcheck source=contrib/makepkg/PKGBUILD.proto
. "${pkgpath}"/PKGBUILD
# run without parallelization for single packages
if (( ${#pkgname[@]} == 1 )); then
write_srcinfo_content
return 0
fi
[[ -z ${WORKDIR:-} ]] && setup_workdir
outdir=$(mktemp --directory --tmpdir="${WORKDIR}" pkgctl-srcinfo.XXXXXXXXXX)
# fork workload for each split pkgname
for pkg in "${pkgname[@]}"; do
(
# deactivate errexit to avoid makepkg abort on grep_function
set +e
srcinfo_write_package "$pkg" > "${outdir}/${pkg}"
)&
pids+=($!)
done
# join workload
for pid in "${pids[@]}"; do
if ! wait "${pid}"; then
return 1
fi
done
# collect output
srcinfo_write_global
for pkg in "${pkgname[@]}"; do
srcinfo_separate_section
cat "${outdir}/${pkg}"
done
}
write_srcinfo_file() {
local pkgpath=${1:-.}
stat_busy 'Generating .SRCINFO'
if ! print_srcinfo "${pkgpath}" > "${pkgpath}"/.SRCINFO; then
error 'Failed to write .SRCINFO file'
return 1
fi
stat_done
}