From: Luke Shumaker <lukeshu@parabola.nu> In a patchset that I recently submitted, Eli was concerned that I was parsing .db files with bsdtar+awk, when the format of .db files isn't "public"; the only guarantees made about it are that libalpm can parse it. https://lists.archlinux.org/pipermail/arch-projects/2018-June/004932.html I wasn't too concerned, because `ftpdir-cleanup` and `sourceballs` already parse the .db files in the same way. Nonetheless, I think Eli is right: we shouldn't be parsing these files ourselves. So, add an `arch_expac` function that uses `expac` to parse the .db files. `expac` is particularly palatable as a new dependency, as it seems likely that it will be included in future releases of pacman. `expac` (like the underlying libalpm) doesn't offer an easy way to say "parse this DB file for me"; instead, we must construct a pacman.conf that has a repo pointing to that file with a `Server=file://...` line. --- v2: - use expac, not pyalpm cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/sourceballs | 21 ++++++++------------- db-functions | 24 ++++++++++++++++++++++++ test/Dockerfile | 2 +- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 9df5f99..e64b7e4 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -44,7 +44,7 @@ for repo in "${PKGREPOS[@]}"; do fi done | sort > "${WORKDIR}/repo-${repo}-${arch}" # get a list of package files defined in the repo db - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" + arch_expac "$repo" "$arch" '%f' | sort > "${WORKDIR}/db-${repo}-${arch}" missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if (( ${#missing_pkgs[@]} >= 1 )); then diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6be28ab..e24102b 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -19,24 +19,19 @@ renice +10 -p $$ > /dev/null # Create a readable file for each repo with the following format # <pkgbase|pkgname> <pkgver>-<pkgrel> <arch> <license>[ <license>] for repo in "${PKGREPOS[@]}"; do + # The %n+awk+cut bit is because we can't trust %e, as makepkg + # <5.1 didn't set pkgbase in .PKGINFO for non-split packages + # when when pkgbase==pkgname; so until all packages have been + # rebuilt with makepkg 5.1, we need to look at both pkgbase + # (%e) and pkgname (%n) to reliably get pkgbase. for arch in "${ARCHES[@]}"; do # Repo does not exist; skip it if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ - | awk '/^%NAME%/ { getline b }; - /^%BASE%/ { getline b }; - /^%VERSION%/ { getline v }; - /^%LICENSE%/,/^$/ { - if ( !/^%LICENSE%/ ) { l=l" "$0 } - }; - /^%ARCH%/ { - getline a; - printf "%s %s %s %s\n", b, v, a, l; - l=""; - }' - done | sort -u > "${WORKDIR}/db-${repo}" + arch_expac "$repo" "$arch" '%n %e %v %a %L' + done | awk -F' ' '$2 == "(null)" { $2 = $1 } { print }' | cut -d' ' -f2- | + sort -u > "${WORKDIR}/db-${repo}" done for repo in "${PKGREPOS[@]}"; do diff --git a/db-functions b/db-functions index 0491c22..cb55f87 100644 --- a/db-functions +++ b/db-functions @@ -436,6 +436,30 @@ set_repo_permission() { fi } +# usage: arch_expac repo arch expac_args... +arch_expac() { + local repo=$1 + local arch=$2 + local args=("${@:3}") + local dir="${WORKDIR}/expac-${repo}-${arch}" + + if ! [[ -d $dir ]]; then + mkdir -p -- "${dir}/root" + cat >"${dir}/pacman.conf" <<-EOT + [options] + RootDir = ${dir}/root + DBPath = ${dir}/root + Architecture = ${arch} + + [$repo] + Server = file://${FTP_BASE}/\$repo/os/\$arch + EOT + fi + + fakeroot pacman --config="${dir}/pacman.conf" -Sy >/dev/null + expac --config="${dir}/pacman.conf" --sync "${args[@]}" +} + arch_repo_modify() { local action=$1 local repo=$2 diff --git a/test/Dockerfile b/test/Dockerfile index 83c8449..cb40570 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM archlinux/base -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep expac RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -d /build -m tester -- 2.17.1