From: Luke Shumaker lukeshu@parabola.nu

Don't try to parse PKGBUILD files from SVN; all of the information we need is already in the DBEXT files. Several programs use [[ -f PKGBUILD ]] or [[ -r PKGBUILD ]] on files from SVN; those checks can stay, just remove all instances of actually trying to *parse* those files.

As an exception, don't modify parse_pkgbuilds.sh (which is called by check_packages.py, which is called by cron-jobs/integrity-check).

Most of the attributes we need have been present in the DBEXT files for as long as `repo-add` has been part of pacman:

attribute | git | git date | ver | ver date ----------+----------+------------+-------+----------- FILENAME | aa1c0ba9 | 2006-11-20 | 3.0.0 | 2007-03-25 NAME | aa1c0ba9 | 2006-11-20 | 3.0.0 | 2007-03-25 VERSION | aa1c0ba9 | 2006-11-20 | 3.0.0 | 2007-03-25 ARCH | aa1c0ba9 | 2006-11-20 | 3.0.0 | 2007-03-25 BASE | 4b21504f | 2009-07-22 | 3.3.0 | 2009-08-02

However, `BASE` for split packages is slightly newer, and its presence relies not only on the `repo-add` version, but also the `makepkg` version used to build the package.

As of 2018-06-15, the oldest package in any of the Arch Linux repos is community/gimp-refocus, built on 2013-07-22; I don't believe it is necessary to handle packages from before that change was made (before `BASE` was set).

"Ignore space change" might be useful when viewing this diff. --- db-functions | 38 ++++++++++++++++++-- db-move | 99 ++++++++++++++++++---------------------------------- db-remove | 11 +++--- 3 files changed, 74 insertions(+), 74 deletions(-)

diff --git a/db-functions b/db-functions index 0491c22..69f35b4 100644 --- a/db-functions +++ b/db-functions @@ -6,9 +6,6 @@ shopt -s extglob globstar nullglob

-# Some PKGBUILDs need CARCH to be set -CARCH="x86_64" - # Useful functions UMASK="" set_umask () { @@ -294,6 +291,41 @@ getpkgfiles() { echo "${files[@]}" }

+# usage: getdbinfo repo arch <pkgbase|pkgname> fields +getdbinfo() { + local repo=$1 + local arch=$2 + local pkgbase=$3 + local fields=$4 + + local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + local dbdir="${WORKDIR}/dbfiles/${repo}-${arch}" + if ! [[ -d $dbdir ]]; then + mkdir -p "$dbdir" + bsdtar -xf "$dbfile" -C "$dbdir" --include='*/desc' + fi + + # The grep/xargs is just a fast filter with possible + # false-postives (like a Bloom filter), because awk is + # comparatively slow. You could remove the grep/xargs line, + # and append `"$dbdir"/*/desc` to the awk command, and it + # would do the same thing; just several times slower. + grep -r -lZ -Fx -e "$pkgbase" -- "$dbdir" | xargs -r0 \ + awk -v pkgbase="$pkgbase" -v fields="$fields" -vFS='\n' -vRS='' ' + { dat[gensub(/^%(.*)%$/, "\1", 1, $1)] = gensub(FS, ",", "g", gensub($1 FS, "", 1)) } + ENDFILE { + if (dat["BASE"] == pkgbase || dat["NAME"] == pkgbase) { + n=split(fields, fieldlist, ",") + str = "" + for (i=1; i<=n; i++) + str = str (i==1?"":" ") dat[fieldlist[i]] + print str + } + delete dat + } + ' +} + check_pkgfile() { local pkgfile=$1

diff --git a/db-move b/db-move index 63e5c14..03debfc 100755 --- a/db-move +++ b/db-move @@ -28,32 +28,14 @@ done arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null for pkgbase in "${args[@]:2}"; do arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - if [[ -r ${svnrepo_from}/PKGBUILD ]]; then - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) - if (( ${#pkgnames[@]} < 1 )); then - die "Could not read pkgname" + for tarch in "${ARCHES[@]}"; do + while read -r pkgarch pkgfile; do + svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + if [[ -r ${svnrepo_from}/PKGBUILD ]]; then + getpkgfile "${ftppath_from}/${tarch}/${pkgfile}" >/dev/null + continue 3 fi - - pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) - if [[ -z ${pkgver} ]]; then - die "Could not read pkgver" - fi - - if [[ "${pkgarch}" = any ]]; then - tarches=("${ARCHES[@]}") - else - tarches=("${pkgarch}") - fi - - for pkgname in "${pkgnames[@]}"; do - for tarch in "${tarches[@]}"; do - getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS} >/dev/null - done - done - continue 2 - fi + done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" ARCH,FILENAME) done die "%s not found in %s" "$pkgbase" "$repo_from" done @@ -64,49 +46,36 @@ declare -A add_pkgs declare -A remove_pkgs for pkgbase in "${args[@]:2}"; do tag_list="" - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - - if [[ -f ${svnrepo_from}/PKGBUILD ]]; then - if [[ ${pkgarch} = any ]]; then - tarches=("${ARCHES[@]}") - else - tarches=("${pkgarch}") - fi - msg2 "%s (%s)" "$pkgbase" "${tarches[*]}" - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) - pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) - - if [[ -d ${svnrepo_to} ]]; then - for file in $(arch_svn ls "${svnrepo_to}"); do - arch_svn rm -q "${svnrepo_to}/$file@" + for tarch in "${ARCHES[@]}"; do + while read -r pkgname pkgver pkgarch pkgfile; do + svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" + if [[ -f ${svnrepo_from}/PKGBUILD ]]; then + msg2 "%s (%s)" "$pkgbase" "$pkgarch" + + if [[ -d ${svnrepo_to} ]]; then + for file in $(arch_svn ls "${svnrepo_to}"); do + arch_svn rm -q "${svnrepo_to}/$file@" + done + else + mkdir "${svnrepo_to}" + arch_svn add -q "${svnrepo_to}" + fi + + for file in $(arch_svn ls "${svnrepo_from}"); do + arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" done - else - mkdir "${svnrepo_to}" - arch_svn add -q "${svnrepo_to}" + arch_svn rm --force -q "${svnrepo_from}" + tag_list+=", $pkgarch" fi

- for file in $(arch_svn ls "${svnrepo_from}"); do - arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" - done - arch_svn rm --force -q "${svnrepo_from}" - tag_list+=", $pkgarch" - - for pkgname in "${pkgnames[@]}"; do - for tarch in "${tarches[@]}"; do - pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS}) - pkgfile="${pkgpath##*/}" - - ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" - if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" - fi - add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " - remove_pkgs[${tarch}]+="${pkgname} " - done - done - fi + ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" + if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then + ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" + fi + add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " + remove_pkgs[${tarch}]+="${pkgname} " + done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" NAME,VERSION,ARCH,FILENAME) done tag_list="${tag_list#, }" arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" diff --git a/db-remove b/db-remove index ac9a168..3017026 100755 --- a/db-remove +++ b/db-remove @@ -32,17 +32,16 @@ done remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" - arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null

+ mapfile -t pkgnames < <(getdbinfo "$repo" "${tarches[0]}" "$pkgbase" NAME) + remove_pkgs+=("${pkgnames[@]}") + + arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then - remove_pkgs+=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" else - warning "%s not found in %s" "$pkgbase" "$svnrepo" - warning "Removing only %s from the repo" "$pkgbase" - warning "If it was a split package you have to remove the others yourself!" - remove_pkgs+=("$pkgbase") + warning "pkgbase '%s' not found in svn; unable to commit removal to svn" "$pkgbase" fi done

From: Luke Shumaker lukeshu@parabola.nu

testing2x needs to: 1. Verify that the new version of the package is found in TESTING_REPO 2. Determine which of STABLE_REPOS the old version of the package is in

Currently, to do those things, it checks for the existence of PKGBUILD files in SVN. That information is already stored in the DBEXT files; get it from there, instead of talking to SVN.

"Ignore space change" might be useful when viewing this diff. --- testing2x | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-)

diff --git a/testing2x b/testing2x index d3b096b..f455a93 100755 --- a/testing2x +++ b/testing2x @@ -19,31 +19,25 @@ done declare -A pkgs

for pkgbase in "$@"; do - if [[ ! -d ${WORKDIR}/${pkgbase} ]]; then - arch_svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null - - found_source=false - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/${pkgbase}/${TESTING_REPO}-${pkgarch}" - if [[ -r ${svnrepo_from}/PKGBUILD ]]; then - found_source=true - break + found_source=false + for tarch in "${ARCHES[@]}"; do + if [[ -n $(getdbinfo "$TESTING_REPO" "$tarch" "$pkgbase" NAME) ]]; then + found_source=true + break + fi + done + [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" + found_target=false + for tarch in "${ARCHES[@]}"; do + for repo in "${STABLE_REPOS[@]}"; do + if [[ -n $(getdbinfo "$repo" "$tarch" "$pkgbase" NAME) ]]; then + found_target=true + pkgs[${repo}]+="${pkgbase} " + break 2 fi done - [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" - found_target=false - for pkgarch in "${ARCHES[@]}" 'any'; do - for repo in "${STABLE_REPOS[@]}"; do - svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" - if [[ -r ${svnrepo_to}/PKGBUILD ]]; then - found_target=true - pkgs[${repo}]+="${pkgbase} " - break 2 - fi - done - done - [[ $found_target = true ]] || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" - fi + done + [[ $found_target = true ]] || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" done

for pkgarch in "${ARCHES[@]}"; do

From: Luke Shumaker lukeshu@parabola.nu

Currently, abs_move_preflight_check and abs_move_arch require the exact pkgarch (i.e. "any" or "x86_64"). Modify them so that they take the tarch (i.e. "x86_64") and figure out themselves if it needs to set pkgarch=$tarch or pkgarch=any.

abs_export could probably stand to be modified to do something similar, but it's more work to adjust its callers.

"Ignore space change" might be useful when viewing this diff. --- db-abs | 56 ++++++++++++++++++++++++++++++++------------------------ db-move | 15 +++++++-------- 2 files changed, 39 insertions(+), 32 deletions(-)

diff --git a/db-abs b/db-abs index 5a34b84..c510bad 100644 --- a/db-abs +++ b/db-abs @@ -20,12 +20,12 @@ _abs_checkout() {

abs_move_preflight_check() { local repo_from=$1 - local pkgarch=$2 + local tarch=$2 local pkgbase=$3

_abs_checkout "$pkgbase" - local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - [[ -r ${svnrepo_from}/PKGBUILD ]] + local reposdir="${WORKDIR}/svn/${pkgbase}/repos" + [[ -r ${reposdir}/${repo_from}-${tarch}/PKGBUILD || -r ${reposdir}/${repo_from}-any/PKGBUILD ]] }

abs_move_start() { @@ -37,37 +37,45 @@ abs_move_start() { }

# If the "from" PKGBUILD doesn't exist, this is a no-op (not an -# error), so that it can be run for each arch, and the invoker doesn't -# need to worry about hoisting it out of the loop if arch=(any). If -# the nonexistence is such that it should be an error, we count on -# abs_move_preflight_check having already caught that. +# error). This is because we expect to be called exactly once for +# each tarch (eg: x86_64, i686), but if arch=(any) then we only need +# do the work once; on the subsequent calls the "from" PKGBUILD won't +# exist anymore. If the source PKGBUILD never existed, we expect that +# to have already been caught by abs_move_preflight_check. abs_move_arch() { - local pkgarch=$1 + local tarch=$1

local repo_from=$abs_move_repo_from local repo_to=$abs_move_repo_to local pkgbase=$abs_move_pkgbase

+ local pkgarch + if [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${tarch}/PKGBUILD" ]]; then + pkgarch=$tarch + elif [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-any/PKGBUILD" ]]; then + pkgarch=any + else + return 0 + fi + local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" local svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - if [[ -f ${svnrepo_from}/PKGBUILD ]]; then - msg2 "%s (%s)" "$pkgbase" "$pkgarch" - - if [[ -d ${svnrepo_to} ]]; then - for file in $(arch_svn ls "${svnrepo_to}"); do - arch_svn rm -q "${svnrepo_to}/$file@" - done - else - mkdir "${svnrepo_to}" - arch_svn add -q "${svnrepo_to}" - fi - - for file in $(arch_svn ls "${svnrepo_from}"); do - arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" + msg2 "%s (%s)" "$pkgbase" "$pkgarch" + + if [[ -d ${svnrepo_to} ]]; then + for file in $(arch_svn ls "${svnrepo_to}"); do + arch_svn rm -q "${svnrepo_to}/$file@" done - arch_svn rm --force -q "${svnrepo_from}" - abs_move_tag_list+=", $pkgarch" + else + mkdir "${svnrepo_to}" + arch_svn add -q "${svnrepo_to}" fi + + for file in $(arch_svn ls "${svnrepo_from}"); do + arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" + done + arch_svn rm --force -q "${svnrepo_from}" + abs_move_tag_list+=", $pkgarch" }

abs_move_finish() { diff --git a/db-move b/db-move index 94e6352..40af7c5 100755 --- a/db-move +++ b/db-move @@ -28,10 +28,10 @@ done for pkgbase in "${args[@]:2}"; do found=false for tarch in "${ARCHES[@]}"; do - while read -r pkgarch pkgfile; do + while read -r pkgfile; do

- if ! abs_move_preflight_check "$repo_from" "$pkgarch" "$pkgbase"; then - die "%s not found in %s-%s" "$pkgbase" "$repo_from" "$pkgarch" + if ! abs_move_preflight_check "$repo_from" "$tarch" "$pkgbase"; then + die "%s not found in %s-%s" "$pkgbase" "$repo_from" "$tarch" fi

# getpkgfile will `exit` for us if it fails; @@ -39,7 +39,7 @@ for pkgbase in "${args[@]:2}"; do getpkgfile "${ftppath_from}/${tarch}/${pkgfile}" >/dev/null

found=true - done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" ARCH,FILENAME) + done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" FILENAME) done [[ $found = true ]] || die "%s not found in %s" "$pkgbase" "$repo_from" done @@ -51,16 +51,15 @@ declare -A remove_pkgs for pkgbase in "${args[@]:2}"; do abs_move_start "$repo_from" "$repo_to" "$pkgbase" for tarch in "${ARCHES[@]}"; do - while read -r pkgname pkgver pkgarch pkgfile; do - abs_move_arch "$pkgarch" - + abs_move_arch "$tarch" + while read -r pkgname pkgver pkgfile; do ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" fi add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " remove_pkgs[${tarch}]+="${pkgname} " - done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" NAME,VERSION,ARCH,FILENAME) + done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" NAME,VERSION,FILENAME) done abs_move_finish done