Version 20210202

After further followups always moving all products isn't actually
desired as they can theoretically be broken in various ways if
arch-nspawn exists non successful.

However, as we would like to always preserve the produced log files we
instead split out the logfiles into an own function and call that for
unsuccessful buils.

Fixes 4f305aa3

.gitignore

@@ -11,6 +11,7 @@ lddd
 makechrootpkg
 makerepropkg
 mkarchroot
+offload-build
 rebuildpkgs
 zsh_completion
 find-libdeps
@@ -18,3 +19,4 @@ crossrepomove
 arch-nspawn
 sogrep
 doc/*.1
+doc/*.7

Makefile

@@ -1,4 +1,4 @@
-V=20191227
+V=20210202
 
 PREFIX = /usr/local
 MANDIR = $(PREFIX)/share/man
@@ -17,12 +17,12 @@ IN_PROGS = \
 	makerepropkg \
 	mkarchroot \
 	makechrootpkg \
+	offload-build \
 	rebuildpkgs \
 	sogrep
 
 BINPROGS = \
-	$(IN_PROGS) \
-	offload-build \
+	$(IN_PROGS)
 
 CONFIGFILES = \
 	makepkg-x86_64.conf \
@@ -75,7 +75,8 @@ MANS = \
 	doc/makerepropkg.1 \
 	doc/mkarchroot.1 \
 	doc/find-libdeps.1 \
-	doc/find-libprovides.1
+	doc/find-libprovides.1 \
+	doc/devtools.7
 
 
 all: $(BINPROGS) bash_completion zsh_completion man

arch-nspawn.in

@@ -69,7 +69,9 @@ host_mirrors=($($pacconf_cmd --repo extra Server 2> /dev/null | sed -r 's#(.*/)e
 for host_mirror in "${host_mirrors[@]}"; do
 	if [[ $host_mirror == *file://* ]]; then
 		host_mirror=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo/os/\$arch#\1#g')
-		in_array "$host_mirror" "${cache_dirs[@]}" || cache_dirs+=("$host_mirror")
+		for m in "$host_mirror"/pool/*/; do
+			in_array "$m" "${cache_dirs[@]}" || cache_dirs+=("$m")
+		done
 	fi
 done
 

checkpkg.in

@@ -95,8 +95,10 @@ for _pkgname in "${pkgname[@]}"; do
 
 	if (( $# )); then
 		case $1 in
+			*://*)
+				pkgurl=$1 ;;
 			/*|*/*)
-				pkgurl=file://$(readlink -m "$1") ;;
+				pkgurl=$(readlink -m "$1") ;;
 			*.pkg.tar*)
 				pkgurl=$1 ;;
 			'')

doc/devtools.7.asciidoc

@@ -0,0 +1,46 @@
+devtools(7)
+===========
+
+Name
+----
+devtools - Developer tools for the Arch Linux distribution
+
+Description
+-----------
+
+Devtools contains tools for package maintenance in Arch Linux. The toolset
+varies from tools for building packages in a clean chroot ('mkarchroot',...),
+packaging related tools for sonames ('sogrep', 'lddd') and tools for
+repository management such as ('archco', 'extra2community')
+
+Programs
+--------
+The list below gives a short overview; see the respective documentation
+for details.
+
+linkman:checkpkg[1]
+	Compare the current build pakcage with the repository version
+
+linkman:find-libdeps[1]
+	Find soname dependencies for a package
+
+linkman:find-libprovides[1]
+	Find soname's which are provided by a package
+
+linkman:lddd[1]
+	Find broken library links on your system
+
+linkman:makerepropkg[1]
+	Rebuild a package to see if it is reproducible
+
+linkman:mkarchroot[1]
+	Creates an arch chroot in a specified location with a specified set of
+	packages
+
+linkman:offload-build[1]
+	Build a PKGBUILD on a remote server using makechrootpkg
+
+linkman:sogrep[1]
+	Find packages using a linked to a given shared library
+
+include::footer.asciidoc[]

doc/makerepropkg.1.asciidoc

@@ -7,12 +7,12 @@ makerepropkg - Rebuild a package to see if it is reproducible
 
 Synopsis
 --------
-makerepropkg [OPTIONS] <package_file>
+makerepropkg [OPTIONS] [<package_file|pkgname>...]
 
 Description
 -----------
 
-Given the path to a built pacman package, attempt to rebuild it using the
+Given the path to a built pacman package(s), attempt to rebuild it using the
 PKGBUILD in the current directory. The package will be built in an environment
 as closely matching the environment of the initial package as possible, by
 building up a chroot to match the information exposed in the package's
@@ -20,12 +20,28 @@ linkman:BUILDINFO[5] manifest. On success, the resulting package will be
 compared to the input package, and makerepropkg will report whether the
 artifacts are identical.
 
+When given multiple packages, additional package files are assumed to be split
+packages and will be treated as additional artifacts to compare during the
+verification step.
+
+A valid target(s) for pacman -S can be specified instead, and makerepropkg will
+download it to the cache if needed. This is mostly useful to specify which
+repository to retrieve from. If no positional arguments are specified, the
+targets will be sourced from the PKGBUILD.
+
+In either case, the package name will be converted to a filename from the
+cache, and makerepropkg will proceed as though this filename was initially
+specified.
+
 This implements a verifier for pacman/libalpm packages in accordance with the
 link:https://reproducible-builds.org/[Reproducible Builds] project.
 
 Options
 -------
 
+*-d*::
+	If packages are not reproducible, compare them using diffoscope.
+
 *-c*::
 	Set the pacman cache directory.
 

doc/offload-build.1.asciidoc

@@ -28,7 +28,7 @@ Options
 	architecture officially supported by Arch Linux.
 
 *-s, --server* <hostname>::
-	Offload to a specific build server. The default is dragon.archlinux.org
+	Offload to a specific build server. The default is build.archlinux.org
 	which is used as part of the build toolchain for the official Arch Linux
 	repos.
 

makechrootpkg.in

@@ -242,6 +242,15 @@ download_sources() {
 		die "Could not download sources."
 }
 
+move_logfiles() {
+	local l
+	for l in "$copydir"/logdest/*; do
+		[[ $l == */logpipe.* ]] && continue
+		chown "$src_owner" "$l"
+		mv "$l" "$LOGDEST"
+	done
+}
+
 move_products() {
 	local pkgfile
 	for pkgfile in "$copydir"/pkgdest/*; do
@@ -254,12 +263,7 @@ move_products() {
 		fi
 	done
 
-	local l
-	for l in "$copydir"/logdest/*; do
-		[[ $l == */logpipe.* ]] && continue
-		chown "$src_owner" "$l"
-		mv "$l" "$LOGDEST"
-	done
+	move_logfiles
 
 	for s in "$copydir"/srcpkgdest/*; do
 		chown "$src_owner" "$s"
@@ -366,14 +370,11 @@ if arch-nspawn "$copydir" \
 	"${bindmounts_ro[@]}" "${bindmounts_rw[@]}" \
 	/chrootbuild "${makepkg_args[@]}"
 then
-	pkgnames=()
-	for pkgfile in "$copydir"/pkgdest/*; do
-		pkgfile=${pkgfile##*/};
-		pkgnames+=("${pkgfile%-*-*-*}");
-	done
+	mapfile -t pkgnames < <(sudo -u "$makepkg_user" bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"')
 	move_products
 else
 	(( ret += 1 ))
+	move_logfiles
 fi
 
 (( temp_chroot )) && delete_chroot "$copydir" "$copy"
@@ -388,29 +389,29 @@ else
 	if (( run_checkpkg )); then
 		msg "Running checkpkg"
 
-		# sync off-site databases for up-to-date queries
-		trap 'rm -rf $dbpath; cleanup' EXIT INT TERM QUIT
-		dbpath=$(mktemp -d --tmpdir makechrootpkg-database.XXXXXXXXXX)
-		mkdir -p "$dbpath"
-		pacman -Sy --dbpath "$dbpath" --logfile /dev/null
+		mapfile -t remotepkgs < <(pacman --config "$copydir"/etc/pacman.conf \
+			--dbpath "$copydir"/var/lib/pacman \
+			-Sddp "${pkgnames[@]}")
 
-		# query current package locations
-		remotepkgs=($(pacman -Sddp --dbpath "$dbpath" --logfile /dev/null "${pkgnames[@]}"))
-		if (( $? )); then
+		if ! wait $!; then
 			warning "Skipped checkpkg due to missing repo packages"
 			exit 0
 		fi
 
 		# download package files if any non-local location exists
 		for remotepkg in "${remotepkgs[@]}"; do
-			[[ $remotepkg == file://* ]] && continue
-			msg2 "Downloading current versions"
-			pacman --noconfirm -Swdd --dbpath "$dbpath" --logfile /dev/null "${pkgnames[@]}"
-			break
+			if [[ $remotepkg != file://* ]]; then
+				msg2 "Downloading current versions"
+				arch-nspawn "$copydir" pacman --noconfirm -Swdd "${pkgnames[@]}"
+				mapfile -t remotepkgs < <(pacman --config "$copydir"/etc/pacman.conf \
+					--dbpath "$copydir"/var/lib/pacman \
+					-Sddp "${pkgnames[@]}")
+				break
+			fi
 		done
 
 		msg2 "Checking packages"
-		sudo -u "$makepkg_user" checkpkg --rmdir --warn
+		sudo -u "$makepkg_user" checkpkg --rmdir --warn "${remotepkgs[@]/#file:\/\//}"
 	fi
 	true
 fi

makepkg-x86_64.conf

@@ -15,7 +15,7 @@ DLAGENTS=('file::/usr/bin/curl -gqC - -o %o %u'
           'ftp::/usr/bin/curl -gqfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u'
           'http::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u'
           'https::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u'
-          'rsync::/usr/bin/rsync --no-motd -z %u %o'
+          'rsync::/usr/bin/rsync --no-motd -zz %u %o'
           'scp::/usr/bin/scp -C %u %o')
 
 # Other common tools:

makerepropkg.in

@@ -29,6 +29,7 @@ declare -a buildenv buildopts installed installpkgs
 archiveurl='https://archive.archlinux.org/packages'
 buildroot=/var/lib/archbuild/reproducible
 chroot=testenv
+diffoscope=0
 
 parse_buildinfo() {
     local line var val
@@ -56,10 +57,16 @@ parse_buildinfo() {
 get_pkgfile() {
     local cdir=${cache_dirs[0]}
     local pkgfilebase=${1}
+    local mode=${2}
     local pkgname=${pkgfilebase%-*-*-*}
     local pkgfile ext
 
-    for ext in .xz .zst ''; do
+    # try without downloading
+    if [[ ${mode} != localonly ]] && get_pkgfile "${pkgfilebase}" localonly; then
+        return 0
+    fi
+
+    for ext in .zst .xz ''; do
         pkgfile=${pkgfilebase}.pkg.tar${ext}
 
         for c in "${cache_dirs[@]}"; do
@@ -71,6 +78,9 @@ get_pkgfile() {
 
         for f in "${pkgfile}" "${pkgfile}.sig"; do
             if [[ ! -f "${cdir}/${f}" ]]; then
+                if [[ ${mode} = localonly ]]; then
+                    continue 2
+                fi
                 msg2 "retrieving '%s'..." "${f}" >&2
                 curl -Llf -# -o "${cdir}/${f}" "${archiveurl}/${pkgname:0:1}/${pkgname}/${f}" || continue 2
             fi
@@ -94,34 +104,65 @@ package, including the .PKGINFO as well as the buildinfo.
 For more details see https://reproducible-builds.org/
 
 OPTIONS
+    -d            Run diffoscope if the package is unreproducible
     -c <dir>      Set pacman cache
     -M <file>     Location of a makepkg config file
     -h            Show this usage message
 __EOF__
 }
 
-while getopts 'M:c:h' arg; do
-        case "$arg" in
-                M) archroot_args+=(-M "$OPTARG") ;;
-                c) cache_dirs+=("$OPTARG") ;;
-                h) usage; exit 0 ;;
-                *|?) usage; exit 1 ;;
-        esac
+while getopts 'dM:c:h' arg; do
+    case "$arg" in
+        d) diffoscope=1 ;;
+        M) archroot_args+=(-M "$OPTARG") ;;
+        c) cache_dirs+=("$OPTARG") ;;
+        h) usage; exit 0 ;;
+        *|?) usage; exit 1 ;;
+    esac
 done
 shift $((OPTIND - 1))
 
 check_root
 
-if [[ -n $1 ]]; then
-    pkgfile="$1"
-    if ! bsdtar -tqf "${pkgfile}" .BUILDINFO >/dev/null 2>&1; then
-        error "file is not a valid pacman package: '%s'" "${pkgfile}"
+[[ -f PKGBUILD ]] || { error "No PKGBUILD in current directory."; exit 1; }
+
+# without arguments, get list of packages from PKGBUILD
+if [[ -z $1 ]]; then
+    mapfile -t pkgnames < <(source PKGBUILD; pacman -Sddp --print-format '%r/%n' "${pkgname[@]}")
+    wait $! || {
+        error "No package file specified and failed to retrieve package names from './PKGBUILD'."
+        plain "Try '${BASH_SOURCE[0]##*/} -h' for more information." >&2
+        exit 1
+    }
+    msg "Reproducing all pkgnames listed in ./PKGBUILD"
+    set -- "${pkgnames[@]}"
+fi
+
+# check each package to see if it's a file, and if not, try to download it
+# using pacman -Sw, and get the filename from there
+splitpkgs=()
+for p in "$@"; do
+    if [[ -f ${p} ]]; then
+        splitpkgs+=("${p}")
+    else
+        pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null) || { error "package name '%s' not in repos" "${p}"; exit 1; }
+        pkgfile=${pkgfile_remote#file://}
+        if [[ ! -f ${pkgfile} ]]; then
+            msg "Downloading package '%s' into pacman's cache" "${pkgfile}"
+            sudo pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1
+            pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null)
+            pkgfile="${pkgfile_remote#file://}"
+        fi
+        splitpkgs+=("${pkgfile}")
+    fi
+done
+
+for f in "${splitpkgs[@]}"; do
+    if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then
+        error "file is not a valid pacman package: '%s'" "${f}"
         exit 1
     fi
-else
-    error "no package file specified. Try '${BASH_SOURCE[0]##*/} -h' for more information. "
-    exit 1
-fi
+done
 
 if (( ${#cache_dirs[@]} == 0 )); then
 	mapfile -t cache_dirs < <(pacman-conf CacheDir)
@@ -133,11 +174,11 @@ load_makepkg_config
 HOME=${ORIG_HOME}
 [[ -d ${SRCDEST} ]] || SRCDEST=${PWD}
 
-parse_buildinfo < <(bsdtar -xOqf "${pkgfile}" .BUILDINFO)
+parse_buildinfo < <(bsdtar -xOqf "${splitpkgs[0]}" .BUILDINFO)
 export SOURCE_DATE_EPOCH="${buildinfo[builddate]}"
 PACKAGER="${buildinfo[packager]}"
 BUILDDIR="${buildinfo[builddir]}"
-PKGEXT=${pkgfile#${pkgfile%.pkg.tar*}}
+PKGEXT=${splitpkgs[0]#${splitpkgs[0]%.pkg.tar*}}
 
 # nuke and restore reproducible testenv
 for copy in "${buildroot}"/*/; do
@@ -153,8 +194,7 @@ for fname in "${installed[@]}"; do
         exit 1
     fi
 done
-printf '%s\n' "${allpkgfiles[@]}" | mkarchroot -U "${archroot_args[@]}" "${buildroot}"/root - || exit 1
-
+printf '%s\n' "${allpkgfiles[@]}" | mkarchroot -M @pkgdatadir@/makepkg-x86_64.conf -U "${archroot_args[@]}" "${buildroot}"/root - || exit 1
 
 # use makechrootpkg to prep the build directory
 makechrootpkg -r "${buildroot}" -l "${chroot}" -- --packagelist || exit 1
@@ -174,18 +214,26 @@ arch-nspawn "${buildroot}/${chroot}" \
     --bind="${PWD}:/startdir" \
     --bind="${SRCDEST}:/srcdest" \
     /chrootbuild -C --noconfirm --log --holdver --skipinteg
+ret=$?
 
-if (( $? == 0 )); then
+if (( ${ret} == 0 )); then
     msg2 "built succeeded! built packages can be found in ${buildroot}/${chroot}/pkgdest"
     msg "comparing artifacts..."
-    if cmp -s "${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}"; then
-        msg2 "Package successfully reproduced!"
-        exit 0
-    else
-        warning "Package is not reproducible. :("
-        sha256sum "${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}"
-    fi
+
+    for pkgfile in "${splitpkgs[@]}"; do
+        comparefiles=("${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}")
+        if cmp -s "${comparefiles[@]}"; then
+            msg2 "Package '%s' successfully reproduced!" "${pkgfile}"
+        else
+            ret=1
+            warning "Package '%s' is not reproducible. :(" "${pkgfile}"
+            sha256sum "${comparefiles[@]}"
+            if (( diffoscope )); then
+                diffoscope "${comparefiles[@]}"
+            fi
+        fi
+    done
 fi
 
-# the package either failed to build, or was unreproducible
-exit 1
+# return failure from chrootbuild, or the reproducibility status
+exit ${ret}

offload-build.in

@@ -24,7 +24,7 @@ source /usr/share/makepkg/util/config.sh
 # global defaults suitable for use by Arch staff
 repo=extra
 arch=x86_64
-server=dragon.archlinux.org
+server=build.archlinux.org
 
 die() { printf "error: $1\n" "${@:2}"; exit 1; }
 
@@ -74,18 +74,23 @@ while (( $# )); do
 done
 
 # multilib must be handled specially
+archbuild_arch="${arch}"
 if [[ $repo = multilib* ]]; then
-    arch=
+    archbuild_arch=
 fi
 
-archbuild_cmd=("${repo}${arch:+-$arch}-build" "$@")
+archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
 
-trap 'rm -rf $SRCPKGDEST' EXIT INT TERM QUIT
+trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT
+
+# Load makepkg.conf variables to be available
+load_makepkg_config
 
 # Use a source-only tarball as an intermediate to transfer files. This
 # guarantees the checksums are okay, and guarantees that all needed files are
 # transferred, including local sources, install scripts, and changelogs.
-export SRCPKGDEST=$(mktemp -d)
+export TEMPDIR=$(mktemp -d --tmpdir offload-build.XXXXXXXXXX)
+export SRCPKGDEST=${TEMPDIR}
 makepkg --source || die "unable to make source package"
 
 # Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
@@ -96,7 +101,7 @@ mapfile -t files < <(
     # This is sort of bash golfing but it allows running a mildly complex
     # command over ssh with a single connection.
     # shellcheck disable=SC2145
-    cat "$SRCPKGDEST"/*.src.tar.gz |
+    cat "$SRCPKGDEST"/*"$SRCEXT" |
         ssh $server '
             temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" &&
             mkdir -p "$temp" &&
@@ -108,14 +113,21 @@ mapfile -t files < <(
                 printf "%s\n" "" "-> build complete" &&
                 printf "\t%s\n" "$temp"/*
             } >&2 &&
-            makepkg --packagelist
+            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
+            makepkg_config="/usr/share/devtools/makepkg-'"${arch}"'.conf" &&
+            if [[ -f /usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf ]]; then
+                makepkg_config="/usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf"
+            fi &&
+            makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist &&
+            printf "%s\n" "${temp}/PKGBUILD"
 ')
 
 
 if (( ${#files[@]} )); then
     printf '%s\n' '' '-> copying files...'
-    load_makepkg_config
-    scp "${files[@]/#/$server:}" "${PKGDEST:-${PWD}}/"
+    scp "${files[@]/#/$server:}" "${TEMPDIR}/"
+    mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
 else
     exit 1
 fi

zsh_completion.in

@@ -94,10 +94,11 @@ _offload_build_args=(
 )
 
 _makerepropkg_args=(
+	'-d[Run diffoscope if the package is unreproducible]'
 	'-c[Set pacman cache]:pacman_cache:_files -/'
 	'-M[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
 	'-h[Display usage]'
-	'1:working_dir:_files -g "*.pkg.tar.*(.)"'
+	'*:working_dir:_files -g "*.pkg.tar.*(.)"'
 )
 
 _devtools_completions_all_packages() {