From: Luke Shumaker
In a patchset that I recently submitted, Eli was concerned that I was
parsing .db files with bsdtar+awk, when the format of .db files isn't
"public"; the only guarantees made about it are that libalpm can parse it.
https://lists.archlinux.org/pipermail/arch-projects/2018-June/004932.html
I wasn't too concerned, because `ftpdir-cleanup` and `sourceballs` already
parse the .db files in the same way. Nonetheless, I think Eli is right: we
shouldn't be parsing these files ourselves.
So, add an `arch_expac` function that uses `expac` to parse the .db files.
`expac` is particularly palatable as a new dependency, as it seems likely
that it will be included in future releases of pacman. `expac` (like the
underlying libalpm) doesn't offer an easy way to say "parse this DB file
for me"; instead, we must construct a pacman.conf that has a repo pointing
to that file with a `Server=file://...` line.
---
v2:
- use expac, not pyalpm
cron-jobs/ftpdir-cleanup | 2 +-
cron-jobs/sourceballs | 21 ++++++++-------------
db-functions | 24 ++++++++++++++++++++++++
test/Dockerfile | 2 +-
4 files changed, 34 insertions(+), 15 deletions(-)
diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup
index 9df5f99..e64b7e4 100755
--- a/cron-jobs/ftpdir-cleanup
+++ b/cron-jobs/ftpdir-cleanup
@@ -44,7 +44,7 @@ for repo in "${PKGREPOS[@]}"; do
fi
done | sort > "${WORKDIR}/repo-${repo}-${arch}"
# get a list of package files defined in the repo db
- bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}"
+ arch_expac "$repo" "$arch" '%f' | sort > "${WORKDIR}/db-${repo}-${arch}"
missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}"))
if (( ${#missing_pkgs[@]} >= 1 )); then
diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs
index 6be28ab..e24102b 100755
--- a/cron-jobs/sourceballs
+++ b/cron-jobs/sourceballs
@@ -19,24 +19,19 @@ renice +10 -p $$ > /dev/null
# Create a readable file for each repo with the following format
# <pkgver>-<pkgrel> <arch> <license>[ <license>]
for repo in "${PKGREPOS[@]}"; do
+ # The %n+awk+cut bit is because we can't trust %e, as makepkg
+ # <5.1 didn't set pkgbase in .PKGINFO for non-split packages
+ # when when pkgbase==pkgname; so until all packages have been
+ # rebuilt with makepkg 5.1, we need to look at both pkgbase
+ # (%e) and pkgname (%n) to reliably get pkgbase.
for arch in "${ARCHES[@]}"; do
# Repo does not exist; skip it
if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then
continue
fi
- bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \
- | awk '/^%NAME%/ { getline b };
- /^%BASE%/ { getline b };
- /^%VERSION%/ { getline v };
- /^%LICENSE%/,/^$/ {
- if ( !/^%LICENSE%/ ) { l=l" "$0 }
- };
- /^%ARCH%/ {
- getline a;
- printf "%s %s %s %s\n", b, v, a, l;
- l="";
- }'
- done | sort -u > "${WORKDIR}/db-${repo}"
+ arch_expac "$repo" "$arch" '%n %e %v %a %L'
+ done | awk -F' ' '$2 == "(null)" { $2 = $1 } { print }' | cut -d' ' -f2- |
+ sort -u > "${WORKDIR}/db-${repo}"
done
for repo in "${PKGREPOS[@]}"; do
diff --git a/db-functions b/db-functions
index 0491c22..cb55f87 100644
--- a/db-functions
+++ b/db-functions
@@ -436,6 +436,30 @@ set_repo_permission() {
fi
}
+# usage: arch_expac repo arch expac_args...
+arch_expac() {
+ local repo=$1
+ local arch=$2
+ local args=("${@:3}")
+ local dir="${WORKDIR}/expac-${repo}-${arch}"
+
+ if ! [[ -d $dir ]]; then
+ mkdir -p -- "${dir}/root"
+ cat >"${dir}/pacman.conf" <<-EOT
+ [options]
+ RootDir = ${dir}/root
+ DBPath = ${dir}/root
+ Architecture = ${arch}
+
+ [$repo]
+ Server = file://${FTP_BASE}/\$repo/os/\$arch
+ EOT
+ fi
+
+ fakeroot pacman --config="${dir}/pacman.conf" -Sy >/dev/null
+ expac --config="${dir}/pacman.conf" --sync "${args[@]}"
+}
+
arch_repo_modify() {
local action=$1
local repo=$2
diff --git a/test/Dockerfile b/test/Dockerfile
index 83c8449..cb40570 100644
--- a/test/Dockerfile
+++ b/test/Dockerfile
@@ -1,5 +1,5 @@
FROM archlinux/base
-RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep
+RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep expac
RUN pacman-key --init
RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel
RUN useradd -N -g users -G wheel -d /build -m tester
--
2.17.1