[arch-devops] [arch-projects] [dbscripts] [PATCH 2/4] Add reproducible archive of packages.

[Eli Schwartz]

On 12/4/18 1:09 PM, Eli Schwartz wrote:
> Whenever adding new package files to the pool of distributed packages,
> hardlink a copy of every package it was built with, into a
> "reproducible" pool, and log which file required it.

The question becomes, where can I store these? As-is, this will burden
the mirror network as well. Unsure how to handle this. Could this be
configurable by the mirror, as ISOs are now? Should we exclusively
self-host this, and if so, where?

archive.archlinux.org is managed by another service with its own
exclusively writable location.

> Signed-off-by: Eli Schwartz <eschwartz at archlinux.org>
> ---
>  config                     |  1 +
>  config.local.svn-community |  1 +
>  config.local.svn-packages  |  1 +
>  db-functions               | 49 +++++++++++++++++++++++++++++++-------
>  db-update                  |  4 ++++
>  5 files changed, 48 insertions(+), 8 deletions(-)
> 
> diff --git a/config b/config
> index 1cfe11f4..5144fca7 100644
> --- a/config
> +++ b/config
> @@ -3,6 +3,7 @@
>  FTP_BASE="/srv/ftp"
>  PKGREPOS=()
>  PKGPOOL=''
> +EXTRA_PKGPOOLS=()
>  SRCPOOL=''
>  TESTING_REPO=''
>  STABLE_REPOS=()
> diff --git a/config.local.svn-community b/config.local.svn-community
> index 5d61b5ea..15bcc17f 100644
> --- a/config.local.svn-community
> +++ b/config.local.svn-community
> @@ -2,6 +2,7 @@
>  
>  PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging')
>  PKGPOOL='pool/community'
> +EXTRA_PKGPOOLS=('pool/packages')
>  SRCPOOL='sources/community'
>  SVNREPO='file:///srv/repos/svn-community/svn'
>  SVNUSER='svn-community'
> diff --git a/config.local.svn-packages b/config.local.svn-packages
> index 34aab35c..75986b65 100644
> --- a/config.local.svn-packages
> +++ b/config.local.svn-packages
> @@ -2,6 +2,7 @@
>  
>  PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable')
>  PKGPOOL='pool/packages'
> +EXTRA_PKGPOOLS=('pool/community')
>  SRCPOOL='sources/packages'
>  SVNREPO='file:///srv/repos/svn-packages/svn'
>  SVNUSER='svn-packages'
> diff --git a/db-functions b/db-functions
> index 7aeedced..2b1ae87a 100644
> --- a/db-functions
> +++ b/db-functions
> @@ -165,20 +165,23 @@ repo_unlock () { #repo_unlock <repo-name> <arch>
>  	fi
>  }
>  
> +# usage: _grep_all_info pkgfile infofile key
> +_grep_all_info() {
> +	local _ret=()
> +
> +	mapfile -t _ret < <(/usr/bin/bsdtar -xOqf "$1" "${2}" | grep "^${3} = ")
> +
> +	printf '%s\n' "${_ret[@]#${3} = }"
> +}
> +
>  # usage: _grep_pkginfo pkgfile pattern
>  _grep_pkginfo() {
> -	local _ret
> -
> -	_ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep "^${2} = " | tail -1)"
> -	echo "${_ret#${2} = }"
> +	_grep_all_info "${1}" .PKGINFO "${2}" | tail -1
>  }
>  
>  # usage: _grep_buildinfo pkgfile pattern
>  _grep_buildinfo() {
> -	local _ret
> -
> -	_ret="$(/usr/bin/bsdtar -xOqf "$1" .BUILDINFO | grep "^${2} = " | tail -1)"
> -	echo "${_ret#${2} = }"
> +	_grep_all_info "${1}" .BUILDINFO "${2}" | tail -1
>  }
>  
>  # Get the package base or name as fallback
> @@ -444,4 +447,34 @@ arch_repo_modify() {
>  	REPO_MODIFIED=1
>  }
>  
> +# Build an index of dependent packages needed by a given pkgfile
> +# usage: make_reproducible pkgfile [check]
> +make_reproducible() {
> +	local pkg dir pkgs=() pkgfile pkgfiles=()
> +
> +	mapfile -t pkgs < <(_grep_all_info "${1}" .BUILDINFO installed)
> +
> +	for pkg in "${pkgs[@]}"; do
> +		for dir in "${FTP_BASE}/${PKGPOOL}" "${EXTRA_PKGPOOLS[@]/#/${FTP_BASE}/}" "${STAGING}"/**/; do
> +			if pkgfile="$(getpkgfile "${dir}/${pkg}"${PKGEXTS} 2>/dev/null)"; then
> +				pkgfiles+=("${pkgfile}")
> +				continue 2
> +			fi
> +		done
> +		error "could not find existing package for %s" "${pkg}"
> +		return 1
> +	done
> +
> +	if [[ ${2} = check ]]; then
> +		return 0
> +	fi
> +
> +	for pkg in "${pkgfiles[@]}"; do
> +		if [[ ! -f ${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/} ]]; then
> +			ln -L "${pkg}" "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}"
> +		fi
> +		echo "${1}" >> "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}.buildlinks"
> +	done
> +}
> +
>  . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}"
> diff --git a/db-update b/db-update
> index 313fb999..11ec185f 100755
> --- a/db-update
> +++ b/db-update
> @@ -61,6 +61,9 @@ for repo in "${repos[@]}"; do
>  			if ! check_builddir "${pkg}"; then
>  				die "Package %s was not built in a chroot" "$repo/${pkg##*/}"
>  			fi
> +			if ! make_reproducible "${pkg}" "check"; then
> +				die "Package %s is not reproducible" "${pkg}"
> +			fi
>  		done
>  		if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then
>  			die "Missing split packages for %s" "$repo"
> @@ -82,6 +85,7 @@ for repo in "${repos[@]}"; do
>  			# any packages might have been moved by the previous run
>  			if [[ -f ${pkg} ]]; then
>  				mv "${pkg}" "$FTP_BASE/${PKGPOOL}"
> +				make_reproducible "${FTP_BASE}/${PKGPOOL}${pkg##*/}"
>  			fi
>  			ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}"
>  			# also move signatures
> 


-- 
Eli Schwartz
Bug Wrangler and Trusted User

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: OpenPGP digital signature
URL: <https://lists.archlinux.org/pipermail/arch-devops/attachments/20181204/6a708130/attachment.asc>