On 12/4/18 1:09 PM, Eli Schwartz wrote:
Whenever adding new package files to the pool of distributed packages, hardlink a copy of every package it was built with, into a "reproducible" pool, and log which file required it.
The question becomes, where can I store these? As-is, this will burden the mirror network as well. Unsure how to handle this. Could this be configurable by the mirror, as ISOs are now? Should we exclusively self-host this, and if so, where? archive.archlinux.org is managed by another service with its own exclusively writable location.
Signed-off-by: Eli Schwartz <eschwartz@archlinux.org> --- config | 1 + config.local.svn-community | 1 + config.local.svn-packages | 1 + db-functions | 49 +++++++++++++++++++++++++++++++------- db-update | 4 ++++ 5 files changed, 48 insertions(+), 8 deletions(-)
diff --git a/config b/config index 1cfe11f4..5144fca7 100644 --- a/config +++ b/config @@ -3,6 +3,7 @@ FTP_BASE="/srv/ftp" PKGREPOS=() PKGPOOL='' +EXTRA_PKGPOOLS=() SRCPOOL='' TESTING_REPO='' STABLE_REPOS=() diff --git a/config.local.svn-community b/config.local.svn-community index 5d61b5ea..15bcc17f 100644 --- a/config.local.svn-community +++ b/config.local.svn-community @@ -2,6 +2,7 @@
PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') PKGPOOL='pool/community' +EXTRA_PKGPOOLS=('pool/packages') SRCPOOL='sources/community' SVNREPO='file:///srv/repos/svn-community/svn' SVNUSER='svn-community' diff --git a/config.local.svn-packages b/config.local.svn-packages index 34aab35c..75986b65 100644 --- a/config.local.svn-packages +++ b/config.local.svn-packages @@ -2,6 +2,7 @@
PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') PKGPOOL='pool/packages' +EXTRA_PKGPOOLS=('pool/community') SRCPOOL='sources/packages' SVNREPO='file:///srv/repos/svn-packages/svn' SVNUSER='svn-packages' diff --git a/db-functions b/db-functions index 7aeedced..2b1ae87a 100644 --- a/db-functions +++ b/db-functions @@ -165,20 +165,23 @@ repo_unlock () { #repo_unlock <repo-name> <arch> fi }
+# usage: _grep_all_info pkgfile infofile key +_grep_all_info() { + local _ret=() + + mapfile -t _ret < <(/usr/bin/bsdtar -xOqf "$1" "${2}" | grep "^${3} = ") + + printf '%s\n' "${_ret[@]#${3} = }" +} + # usage: _grep_pkginfo pkgfile pattern _grep_pkginfo() { - local _ret - - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep "^${2} = " | tail -1)" - echo "${_ret#${2} = }" + _grep_all_info "${1}" .PKGINFO "${2}" | tail -1 }
# usage: _grep_buildinfo pkgfile pattern _grep_buildinfo() { - local _ret - - _ret="$(/usr/bin/bsdtar -xOqf "$1" .BUILDINFO | grep "^${2} = " | tail -1)" - echo "${_ret#${2} = }" + _grep_all_info "${1}" .BUILDINFO "${2}" | tail -1 }
# Get the package base or name as fallback @@ -444,4 +447,34 @@ arch_repo_modify() { REPO_MODIFIED=1 }
+# Build an index of dependent packages needed by a given pkgfile +# usage: make_reproducible pkgfile [check] +make_reproducible() { + local pkg dir pkgs=() pkgfile pkgfiles=() + + mapfile -t pkgs < <(_grep_all_info "${1}" .BUILDINFO installed) + + for pkg in "${pkgs[@]}"; do + for dir in "${FTP_BASE}/${PKGPOOL}" "${EXTRA_PKGPOOLS[@]/#/${FTP_BASE}/}" "${STAGING}"/**/; do + if pkgfile="$(getpkgfile "${dir}/${pkg}"${PKGEXTS} 2>/dev/null)"; then + pkgfiles+=("${pkgfile}") + continue 2 + fi + done + error "could not find existing package for %s" "${pkg}" + return 1 + done + + if [[ ${2} = check ]]; then + return 0 + fi + + for pkg in "${pkgfiles[@]}"; do + if [[ ! -f ${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/} ]]; then + ln -L "${pkg}" "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}" + fi + echo "${1}" >> "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}.buildlinks" + done +} + . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}" diff --git a/db-update b/db-update index 313fb999..11ec185f 100755 --- a/db-update +++ b/db-update @@ -61,6 +61,9 @@ for repo in "${repos[@]}"; do if ! check_builddir "${pkg}"; then die "Package %s was not built in a chroot" "$repo/${pkg##*/}" fi + if ! make_reproducible "${pkg}" "check"; then + die "Package %s is not reproducible" "${pkg}" + fi done if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then die "Missing split packages for %s" "$repo" @@ -82,6 +85,7 @@ for repo in "${repos[@]}"; do # any packages might have been moved by the previous run if [[ -f ${pkg} ]]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" + make_reproducible "${FTP_BASE}/${PKGPOOL}${pkg##*/}" fi ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" # also move signatures
-- Eli Schwartz Bug Wrangler and Trusted User