[pacman-dev] [PATCH] makepkg: parallelize integrity checks

[Dan McGee]

This enables parallel integrity checks in makepkg within a given family
of integrity sums. Subshell jobs for each source file are kicked off and
run in parallel, and then we wait for each of them in turn to complete
and print the same information as before.

Note that programming sense says this loop should be done differently
for filesystem access reasons; doing all checks for a given file would
make more sense rather than running through the filelist multiple times.
However, that would be a very different patch than what this is trying
to accomplish.

On a completely suited for the task PKGBUILD containing md5sums ans
sha256sums of several large data files, as well as a failing integrity
check so, this brings execution time way down:

     $ time makepkg -f 2>/dev/null
     real	0m7.924s
     user	0m7.293s
     sys	0m0.480s

     $ time ~/projects/pacman/scripts/makepkg -f 2>/dev/null
     real	0m2.447s
     user	0m7.470s
     sys	0m0.537s

Signed-off-by: Dan McGee <dan at archlinux.org>
---

Said PKGBUILD from above:

pkgname=integspeed
pkgver=1.0
pkgrel=1
arch=('any')

source=(
eclipse-3.6.1-1-x86_64.pkg.tar.xz
eclipse-3.6.2-1-x86_64.pkg.tar.xz
nltk-data-2.0-5-any.pkg.tar.xz
openoffice-base-3.2.1-5-x86_64.pkg.tar.xz
smc-1.9-8-x86_64.pkg.tar.xz
texlive-core-2010.20954-2-any.pkg.tar.xz
tremulous-data-1.1.0-1-any.pkg.tar.gz
)

md5sums=('6ab0488636b4b52957a8d069c4330d3d'
         'e889ada205ab8b9b17c64de6c7b62956'
         '968bebb2a8e77b2cd2ee2bbb3d9122d4'
         '17d59366ef890dab62bebfe786a0cccb'
         '129fa6ed2208b355e8e55bdd38ae6677'
         '36437e05082be8c4e6764fc2bcfe4f4b'
         '107d367e2e0245b38402f457c1d735f7')
sha256sums=('d0fdac48f982f4f7794187c6652fb55ac6c63c439a62476bcf90addad25f9274'
            'fc30a73f6313ba202a6fb4fd112de93c88c28b3f7af68dd0789d3317afe89741'
            'aec68224fd31713ff0cee49fb3c27cf5be82ada03657b9d592ab029c5df1ad92'
            '1c9404f87a8b00bd07d226ca93c7254985af3195719a5ddb69be40faa78914c4'
            'bbde245ff84a3ec93f76a53b97d2d261193699a55881209ffae9310de00c6ad4'
            '10a0ba732253d89ef3b6d72486474a356179e4665efc6482ac3c47c891f7f3a2'
            '53d990b7123d12409290c3a3c8b6be32e2887c88a0650cd338a7f58cc951f4d9')

 scripts/makepkg.sh.in |   47 +++++++++++++++++++++++++++++++++--------------
 1 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in
index 3d5184a..92b7597 100644
--- a/scripts/makepkg.sh.in
+++ b/scripts/makepkg.sh.in
@@ -627,33 +627,52 @@ check_checksums() {
 			correlation=1
 			local errors=0
 			local idx=0
+			local -a jobs
+			local job
 			local file
 			for file in "${source[@]}"; do
-				local found=1
-				file="$(get_filename "$file")"
-				echo -n "    $file ... " >&2
+				(
+					file="$(get_filename "$file")"
 
-				if ! file="$(get_filepath "$file")"; then
-					echo "$(gettext "NOT FOUND")" >&2
-					errors=1
-					found=0
-				fi
+					if ! file="$(get_filepath "$file")"; then
+						exit 2
+					fi
 
-				if (( $found )) ; then
 					local expectedsum=$(tr '[:upper:]' '[:lower:]' <<< "${integrity_sums[$idx]}")
 					local realsum="$(openssl dgst -${integ} "$file")"
 					realsum="${realsum##* }"
 					if [[ $expectedsum = $realsum ]]; then
-						echo "$(gettext "Passed")" >&2
-					else
-						echo "$(gettext "FAILED")" >&2
-						errors=1
+						exit 0
 					fi
-				fi
+					exit 1
+				) &
+				jobs[$idx]=$!
 
 				idx=$((idx + 1))
 			done
 
+			idx=0
+			while [[ $idx -lt ${#source[@]} ]]; do
+				file="$(get_filename "${source[$idx]}")"
+				job=${jobs[$idx]}
+				status=0
+				wait $job || status=$?
+				echo -n "    $file ... " >&2
+				if [[ $status -eq 0 ]]; then
+					echo "$(gettext "Passed")" >&2
+				elif [[ $status -eq 1 ]]; then
+					echo "$(gettext "FAILED")" >&2
+					errors=1
+				elif [[ $status -eq 2 ]]; then
+					echo "$(gettext "NOT FOUND")" >&2
+					errors=1
+				else
+					echo "$(gettext "ERROR")" >&2
+					errors=1
+				fi
+				idx=$((idx + 1))
+			done
+
 			if (( errors )); then
 				error "$(gettext "One or more files did not pass the validity check!")"
 				exit 1 # TODO: error code
-- 
1.7.4.4