On 12/04/18 at 01:15pm, Eli Schwartz via arch-devops wrote:

On 12/4/18 1:09 PM, Eli Schwartz wrote:

...

Whenever adding new package files to the pool of distributed packages, hardlink a copy of every package it was built with, into a "reproducible" pool, and log which file required it.

Does this also clean up the archive? As in remove packages which are not required for reproducible builds? Since now our archive server is almost running out of space again.

The question becomes, where can I store these? As-is, this will burden the mirror network as well. Unsure how to handle this. Could this be configurable by the mirror, as ISOs are now? Should we exclusively self-host this, and if so, where?

archive.archlinux.org is managed by another service with its own exclusively writable location.

...

Signed-off-by: Eli Schwartz <eschwartz@archlinux.org> --- config | 1 + config.local.svn-community | 1 + config.local.svn-packages | 1 + db-functions | 49 +++++++++++++++++++++++++++++++------- db-update | 4 ++++ 5 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/config b/config index 1cfe11f4..5144fca7 100644 --- a/config +++ b/config @@ -3,6 +3,7 @@ FTP_BASE="/srv/ftp" PKGREPOS=() PKGPOOL='' +EXTRA_PKGPOOLS=() SRCPOOL='' TESTING_REPO='' STABLE_REPOS=() diff --git a/config.local.svn-community b/config.local.svn-community index 5d61b5ea..15bcc17f 100644 --- a/config.local.svn-community +++ b/config.local.svn-community @@ -2,6 +2,7 @@

PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') PKGPOOL='pool/community' +EXTRA_PKGPOOLS=('pool/packages') SRCPOOL='sources/community' SVNREPO='file:///srv/repos/svn-community/svn' SVNUSER='svn-community' diff --git a/config.local.svn-packages b/config.local.svn-packages index 34aab35c..75986b65 100644 --- a/config.local.svn-packages +++ b/config.local.svn-packages @@ -2,6 +2,7 @@

PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') PKGPOOL='pool/packages' +EXTRA_PKGPOOLS=('pool/community') SRCPOOL='sources/packages' SVNREPO='file:///srv/repos/svn-packages/svn' SVNUSER='svn-packages' diff --git a/db-functions b/db-functions index 7aeedced..2b1ae87a 100644 --- a/db-functions +++ b/db-functions @@ -165,20 +165,23 @@ repo_unlock () { #repo_unlock <repo-name> <arch> fi }

+# usage: _grep_all_info pkgfile infofile key +_grep_all_info() { + local _ret=() + + mapfile -t _ret < <(/usr/bin/bsdtar -xOqf "$1" "${2}" | grep "^${3} = ") + + printf '%s\n' "${_ret[@]#${3} = }" +} + # usage: _grep_pkginfo pkgfile pattern _grep_pkginfo() { - local _ret - - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep "^${2} = " | tail -1)" - echo "${_ret#${2} = }" + _grep_all_info "${1}" .PKGINFO "${2}" | tail -1 }

# usage: _grep_buildinfo pkgfile pattern _grep_buildinfo() { - local _ret - - _ret="$(/usr/bin/bsdtar -xOqf "$1" .BUILDINFO | grep "^${2} = " | tail -1)" - echo "${_ret#${2} = }" + _grep_all_info "${1}" .BUILDINFO "${2}" | tail -1 }

# Get the package base or name as fallback @@ -444,4 +447,34 @@ arch_repo_modify() { REPO_MODIFIED=1 }

+# Build an index of dependent packages needed by a given pkgfile +# usage: make_reproducible pkgfile [check] +make_reproducible() { + local pkg dir pkgs=() pkgfile pkgfiles=() + + mapfile -t pkgs < <(_grep_all_info "${1}" .BUILDINFO installed) + + for pkg in "${pkgs[@]}"; do + for dir in "${FTP_BASE}/${PKGPOOL}" "${EXTRA_PKGPOOLS[@]/#/${FTP_BASE}/}" "${STAGING}"/**/; do + if pkgfile="$(getpkgfile "${dir}/${pkg}"${PKGEXTS} 2>/dev/null)"; then + pkgfiles+=("${pkgfile}") + continue 2 + fi + done + error "could not find existing package for %s" "${pkg}" + return 1 + done + + if [[ ${2} = check ]]; then + return 0 + fi + + for pkg in "${pkgfiles[@]}"; do + if [[ ! -f ${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/} ]]; then + ln -L "${pkg}" "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}" + fi + echo "${1}" >> "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}.buildlinks" + done +} + . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}" diff --git a/db-update b/db-update index 313fb999..11ec185f 100755 --- a/db-update +++ b/db-update @@ -61,6 +61,9 @@ for repo in "${repos[@]}"; do if ! check_builddir "${pkg}"; then die "Package %s was not built in a chroot" "$repo/${pkg##*/}" fi + if ! make_reproducible "${pkg}" "check"; then + die "Package %s is not reproducible" "${pkg}" + fi done if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then die "Missing split packages for %s" "$repo" @@ -82,6 +85,7 @@ for repo in "${repos[@]}"; do # any packages might have been moved by the previous run if [[ -f ${pkg} ]]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" + make_reproducible "${FTP_BASE}/${PKGPOOL}${pkg##*/}" fi ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" # also move signatures

-- Eli Schwartz Bug Wrangler and Trusted User

-- Jelle van der Waa