[arch-projects] [dbscripts] [PATCH v2] Don't parse .db files ourselves; use expac instead

[Luke Shumaker]

From: Luke Shumaker <lukeshu at parabola.nu>

In a patchset that I recently submitted, Eli was concerned that I was
parsing .db files with bsdtar+awk, when the format of .db files isn't
"public"; the only guarantees made about it are that libalpm can parse it.

https://lists.archlinux.org/pipermail/arch-projects/2018-June/004932.html

I wasn't too concerned, because `ftpdir-cleanup` and `sourceballs` already
parse the .db files in the same way.  Nonetheless, I think Eli is right: we
shouldn't be parsing these files ourselves.

So, add an `arch_expac` function that uses `expac` to parse the .db files.
`expac` is particularly palatable as a new dependency, as it seems likely
that it will be included in future releases of pacman.  `expac` (like the
underlying libalpm) doesn't offer an easy way to say "parse this DB file
for me"; instead, we must construct a pacman.conf that has a repo pointing
to that file with a `Server=file://...` line.
---
v2:
 - use expac, not pyalpm

 cron-jobs/ftpdir-cleanup |  2 +-
 cron-jobs/sourceballs    | 21 ++++++++-------------
 db-functions             | 24 ++++++++++++++++++++++++
 test/Dockerfile          |  2 +-
 4 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup
index 9df5f99..e64b7e4 100755
--- a/cron-jobs/ftpdir-cleanup
+++ b/cron-jobs/ftpdir-cleanup
@@ -44,7 +44,7 @@ for repo in "${PKGREPOS[@]}"; do
 			fi
 		done | sort > "${WORKDIR}/repo-${repo}-${arch}"
 		# get a list of package files defined in the repo db
-		bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
+		arch_expac "$repo" "$arch" '%f' | sort > "${WORKDIR}/db-${repo}-${arch}"
 
 		missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}"))
 		if (( ${#missing_pkgs[@]} >= 1 )); then
diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index 6be28ab..e24102b 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -19,24 +19,19 @@ renice +10 -p $$ > /dev/null
 # Create a readable file for each repo with the following format
 # <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>]
 for repo in "${PKGREPOS[@]}"; do
+	# The %n+awk+cut bit is because we can't trust %e, as makepkg
+	# <5.1 didn't set pkgbase in .PKGINFO for non-split packages
+	# when when pkgbase==pkgname; so until all packages have been
+	# rebuilt with makepkg 5.1, we need to look at both pkgbase
+	# (%e) and pkgname (%n) to reliably get pkgbase.
 	for arch in "${ARCHES[@]}"; do
 		# Repo does not exist; skip it
 		if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then
 			continue
 		fi
-		bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \
-			| awk '/^%NAME%/ { getline b };
-				/^%BASE%/ { getline b };
-				/^%VERSION%/ { getline v };
-				/^%LICENSE%/,/^$/ {
-					if ( !/^%LICENSE%/ ) { l=l" "$0 }
-					};
-				/^%ARCH%/ {
-					getline a;
-					printf "%s %s %s %s\n", b, v, a, l;
-					l="";
-				}'
-	done | sort -u > "${WORKDIR}/db-${repo}"
+		arch_expac "$repo" "$arch" '%n %e %v %a %L'
+	done | awk -F' ' '$2 == "(null)" { $2 = $1 } { print }' | cut -d' ' -f2- |
+		sort -u > "${WORKDIR}/db-${repo}"
 done
 
 for repo in "${PKGREPOS[@]}"; do
diff --git a/db-functions b/db-functions
index 0491c22..cb55f87 100644
--- a/db-functions
+++ b/db-functions
@@ -436,6 +436,30 @@ set_repo_permission() {
 	fi
 }
 
+# usage: arch_expac repo arch expac_args...
+arch_expac() {
+	local repo=$1
+	local arch=$2
+	local args=("${@:3}")
+	local dir="${WORKDIR}/expac-${repo}-${arch}"
+
+	if ! [[ -d $dir ]]; then
+		mkdir -p -- "${dir}/root"
+		cat >"${dir}/pacman.conf" <<-EOT
+		[options]
+		RootDir = ${dir}/root
+		DBPath = ${dir}/root
+		Architecture = ${arch}
+
+		[$repo]
+		Server = file://${FTP_BASE}/\$repo/os/\$arch
+		EOT
+	fi
+
+	fakeroot pacman --config="${dir}/pacman.conf" -Sy >/dev/null
+	expac --config="${dir}/pacman.conf" --sync "${args[@]}"
+}
+
 arch_repo_modify() {
 	local action=$1
 	local repo=$2
diff --git a/test/Dockerfile b/test/Dockerfile
index 83c8449..cb40570 100644
--- a/test/Dockerfile
+++ b/test/Dockerfile
@@ -1,5 +1,5 @@
 FROM archlinux/base
-RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep
+RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep expac
 RUN pacman-key --init
 RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel
 RUN useradd -N -g users -G wheel -d /build -m tester
-- 
2.17.1