[arch-projects] [dbscripts] [PATCH v2 5/8] config: let PKGEXT_glob be an extglob; have its value match makepkg

[Luke Shumaker]

From: Luke Shumaker <lukeshu at parabola.nu>

 1. In order to support PKGEXT_glob being an extglob, we must switch
    several uses of `find` (in db-update and ftpdir-cleanup) to use
    bash globbing instead, as `find` can't be made to use extended
    globbing.  In general, this technique requires us to also enable
    nullglob (which we do globally in db-functions).

    Of the 4 searches that we have to rewrite:

    - (ftpdir-cleanup) Two were just being used to strip leading
      directory paths, and can be replaced by ${filepath##*/} in a for
      loop.

    - (ftpdir-cleanup) One was checking the modification time of the
      files, and can be replaced with touch(1) and [[ -nt ]].
      Although this introduces an additional temporary file, this is
      not such a big deal.

    - (db-update) One was getting leading directory paths, this can be
      replaced with globstar (which we also enable globally in
      db-functions) and ${filepath%/*} in a for loop.  We can drop the
      error handling that was on that search, as the old code never
      aborted on errors anyway, as without `set -o pipefail` the sort
      command swallowed the return code.

 2. In addition to enabling exglob in db-functions for the main code,
    we also need to do it in test/lib/common.bash for the tests.  Go
    ahead and also enable globstar and nullglob there too, for
    consistency with db-functions.

    Turning on nullglob there in turn forces us to fix up a couple of
    [ -f ] glob checks in sourceballs.bats.

 3. Use this new extended globbing capability to set the default value
    of PKGEXT_glob to reflect the behavior of makepkg (v5.0.2); the
    old value of PKGEXT_glob both accepted things that makepkg would
    reject, and rejected things that makepkg would accept.

This is based on patches by Eli Schwartz <eschwartz at archlinux.org>
---
 config                      |  4 ++--
 cron-jobs/ftpdir-cleanup    | 18 +++++++++++++++---
 db-functions                |  3 +++
 db-update                   | 10 +++++++---
 test/cases/sourceballs.bats |  4 ++--
 test/lib/common.bash        |  3 +++
 6 files changed, 32 insertions(+), 10 deletions(-)

diff --git a/config b/config
index 13fe202..d2d92ba 100644
--- a/config
+++ b/config
@@ -27,8 +27,8 @@ ARCHES=(x86_64)
 DBEXT=".db.tar.gz"
 FILESEXT=".files.tar.gz"
 SRCEXT=".src.tar.gz"
-# descriptive; bash glob listing allowed extensions.
-PKGEXT_glob=".pkg.tar.?z"
+# descriptive; bash glob listing allowed extensions. Note that db-functions turns on extglob.
+PKGEXT_glob=".pkg.tar?(.gz|.bz2|.xz|.lrz|.lzo|.Z)"
 
 # Allowed licenses: get sourceballs only for licenses in this array
 ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'GPL3' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1' 'LGPL3')
diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup
index 4dc02a0..20d579f 100755
--- a/cron-jobs/ftpdir-cleanup
+++ b/cron-jobs/ftpdir-cleanup
@@ -38,7 +38,11 @@ for repo in ${PKGREPOS[@]}; do
 			continue
 		fi
 		# get a list of actual available package files
-		find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT_glob}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}"
+		for f in "${FTP_BASE}"/${repo}/os/${arch}/*${PKGEXT_glob}; do
+			if [[ -f $f ]]; then
+				printf '%s\n' "${f##*/}"
+			fi
+		done | sort > "${WORKDIR}/repo-${repo}-${arch}"
 		# get a list of package files defined in the repo db
 		bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
 
@@ -62,7 +66,9 @@ for repo in ${PKGREPOS[@]}; do
 done
 
 # get a list of all available packages in the pacakge pool
-find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT_glob}" -printf '%f\n' | sort > "${WORKDIR}/pool"
+for f in "$FTP_BASE/${PKGPOOL}"/*${PKGEXT_glob}; do
+	printf '%s\n' "${f##*/}"
+done | sort > "${WORKDIR}/pool"
 # create a list of packages in our db
 find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db"
 
@@ -75,7 +81,13 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then
 	done
 fi
 
-old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT_glob}" -mtime +${CLEANUP_KEEP} -printf '%f\n'))
+old_pkgs=()
+touch -d "${CLEANUP_KEEP} days ago"  "${WORKDIR}/cleanup_timestamp"
+for f in "${CLEANUP_DESTDIR}"/**/*${PKGEXT_glob}; do
+	if [[ ${WORKDIR}/cleanup_timestamp -nt $f ]]; then
+		old_pkgs+=("${f##*/}")
+	fi
+done
 if [ ${#old_pkgs[@]} -ge 1 ]; then
 	msg "Removing old packages from the cleanup directory..."
 	for old_pkg in ${old_pkgs[@]}; do
diff --git a/db-functions b/db-functions
index 769d7ef..8ca9205 100644
--- a/db-functions
+++ b/db-functions
@@ -2,6 +2,9 @@
 
 . /usr/share/makepkg/util.sh
 
+# global shell options for enhanced bash scripting
+shopt -s extglob globstar nullglob
+
 # Some PKGBUILDs need CARCH to be set
 CARCH="x86_64"
 
diff --git a/db-update b/db-update
index 4afeb6e..37cffbf 100755
--- a/db-update
+++ b/db-update
@@ -9,9 +9,13 @@ if (( $# >= 1 )); then
 fi
 
 # Find repos with packages to release
-if ! staging_repos=($(find  "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT_glob}" -printf '%h\n' | sort -u)); then
-	die "Could not read %s" "$STAGING"
-fi
+readarray -t staging_repos < <(
+	for f in "${STAGING}"/**/*${PKGEXT}; do
+		if [[ -f $f ]]; then
+			printf '%s\n' "${f%/*}"
+		fi
+	done | sort -u
+)
 
 repos=()
 for staging_repo in ${staging_repos[@]##*/}; do
diff --git a/test/cases/sourceballs.bats b/test/cases/sourceballs.bats
index a0a2999..df7ddd4 100644
--- a/test/cases/sourceballs.bats
+++ b/test/cases/sourceballs.bats
@@ -2,12 +2,12 @@ load ../lib/common
 
 __checkSourcePackage() {
 	local pkgbase=$1
-	[ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ]
+	__isGlobfile "${FTP_BASE}/${SRCPOOL}/${pkgbase}"-*"${SRCEXT}"
 }
 
 __checkRemovedSourcePackage() {
 	local pkgbase=$1
-	[ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ]
+	! __isGlobfile "${FTP_BASE}/${SRCPOOL}/${pkgbase}"-*"${SRCEXT}"
 }
 
 @test "create simple package sourceballs" {
diff --git a/test/lib/common.bash b/test/lib/common.bash
index 94fedfe..9d6edeb 100644
--- a/test/lib/common.bash
+++ b/test/lib/common.bash
@@ -1,5 +1,8 @@
 . /usr/share/makepkg/util.sh
 
+# global shell options for enhanced bash scripting
+shopt -s extglob globstar nullglob
+
 __updatePKGBUILD() {
 	local pkgrel
 
-- 
2.16.1