[arch-devops] [arch-projects] [dbscripts] [PATCH 2/4] Add reproducible archive of packages.

[Jelle van der Waa]

On 12/04/18 at 01:15pm, Eli Schwartz via arch-devops wrote:
> On 12/4/18 1:09 PM, Eli Schwartz wrote:
> > Whenever adding new package files to the pool of distributed packages,
> > hardlink a copy of every package it was built with, into a
> > "reproducible" pool, and log which file required it.

Does this also clean up the archive? As in remove packages which are not
required for reproducible builds? Since now our archive server is almost
running out of space again.

> The question becomes, where can I store these? As-is, this will burden
> the mirror network as well. Unsure how to handle this. Could this be
> configurable by the mirror, as ISOs are now? Should we exclusively
> self-host this, and if so, where?
> 
> archive.archlinux.org is managed by another service with its own
> exclusively writable location.
> 
> > Signed-off-by: Eli Schwartz <eschwartz at archlinux.org>
> > ---
> >  config                     |  1 +
> >  config.local.svn-community |  1 +
> >  config.local.svn-packages  |  1 +
> >  db-functions               | 49 +++++++++++++++++++++++++++++++-------
> >  db-update                  |  4 ++++
> >  5 files changed, 48 insertions(+), 8 deletions(-)
> > 
> > diff --git a/config b/config
> > index 1cfe11f4..5144fca7 100644
> > --- a/config
> > +++ b/config
> > @@ -3,6 +3,7 @@
> >  FTP_BASE="/srv/ftp"
> >  PKGREPOS=()
> >  PKGPOOL=''
> > +EXTRA_PKGPOOLS=()
> >  SRCPOOL=''
> >  TESTING_REPO=''
> >  STABLE_REPOS=()
> > diff --git a/config.local.svn-community b/config.local.svn-community
> > index 5d61b5ea..15bcc17f 100644
> > --- a/config.local.svn-community
> > +++ b/config.local.svn-community
> > @@ -2,6 +2,7 @@
> >  
> >  PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging')
> >  PKGPOOL='pool/community'
> > +EXTRA_PKGPOOLS=('pool/packages')
> >  SRCPOOL='sources/community'
> >  SVNREPO='file:///srv/repos/svn-community/svn'
> >  SVNUSER='svn-community'
> > diff --git a/config.local.svn-packages b/config.local.svn-packages
> > index 34aab35c..75986b65 100644
> > --- a/config.local.svn-packages
> > +++ b/config.local.svn-packages
> > @@ -2,6 +2,7 @@
> >  
> >  PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable')
> >  PKGPOOL='pool/packages'
> > +EXTRA_PKGPOOLS=('pool/community')
> >  SRCPOOL='sources/packages'
> >  SVNREPO='file:///srv/repos/svn-packages/svn'
> >  SVNUSER='svn-packages'
> > diff --git a/db-functions b/db-functions
> > index 7aeedced..2b1ae87a 100644
> > --- a/db-functions
> > +++ b/db-functions
> > @@ -165,20 +165,23 @@ repo_unlock () { #repo_unlock <repo-name> <arch>
> >  	fi
> >  }
> >  
> > +# usage: _grep_all_info pkgfile infofile key
> > +_grep_all_info() {
> > +	local _ret=()
> > +
> > +	mapfile -t _ret < <(/usr/bin/bsdtar -xOqf "$1" "${2}" | grep "^${3} = ")
> > +
> > +	printf '%s\n' "${_ret[@]#${3} = }"
> > +}
> > +
> >  # usage: _grep_pkginfo pkgfile pattern
> >  _grep_pkginfo() {
> > -	local _ret
> > -
> > -	_ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep "^${2} = " | tail -1)"
> > -	echo "${_ret#${2} = }"
> > +	_grep_all_info "${1}" .PKGINFO "${2}" | tail -1
> >  }
> >  
> >  # usage: _grep_buildinfo pkgfile pattern
> >  _grep_buildinfo() {
> > -	local _ret
> > -
> > -	_ret="$(/usr/bin/bsdtar -xOqf "$1" .BUILDINFO | grep "^${2} = " | tail -1)"
> > -	echo "${_ret#${2} = }"
> > +	_grep_all_info "${1}" .BUILDINFO "${2}" | tail -1
> >  }
> >  
> >  # Get the package base or name as fallback
> > @@ -444,4 +447,34 @@ arch_repo_modify() {
> >  	REPO_MODIFIED=1
> >  }
> >  
> > +# Build an index of dependent packages needed by a given pkgfile
> > +# usage: make_reproducible pkgfile [check]
> > +make_reproducible() {
> > +	local pkg dir pkgs=() pkgfile pkgfiles=()
> > +
> > +	mapfile -t pkgs < <(_grep_all_info "${1}" .BUILDINFO installed)
> > +
> > +	for pkg in "${pkgs[@]}"; do
> > +		for dir in "${FTP_BASE}/${PKGPOOL}" "${EXTRA_PKGPOOLS[@]/#/${FTP_BASE}/}" "${STAGING}"/**/; do
> > +			if pkgfile="$(getpkgfile "${dir}/${pkg}"${PKGEXTS} 2>/dev/null)"; then
> > +				pkgfiles+=("${pkgfile}")
> > +				continue 2
> > +			fi
> > +		done
> > +		error "could not find existing package for %s" "${pkg}"
> > +		return 1
> > +	done
> > +
> > +	if [[ ${2} = check ]]; then
> > +		return 0
> > +	fi
> > +
> > +	for pkg in "${pkgfiles[@]}"; do
> > +		if [[ ! -f ${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/} ]]; then
> > +			ln -L "${pkg}" "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}"
> > +		fi
> > +		echo "${1}" >> "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}.buildlinks"
> > +	done
> > +}
> > +
> >  . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}"
> > diff --git a/db-update b/db-update
> > index 313fb999..11ec185f 100755
> > --- a/db-update
> > +++ b/db-update
> > @@ -61,6 +61,9 @@ for repo in "${repos[@]}"; do
> >  			if ! check_builddir "${pkg}"; then
> >  				die "Package %s was not built in a chroot" "$repo/${pkg##*/}"
> >  			fi
> > +			if ! make_reproducible "${pkg}" "check"; then
> > +				die "Package %s is not reproducible" "${pkg}"
> > +			fi
> >  		done
> >  		if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then
> >  			die "Missing split packages for %s" "$repo"
> > @@ -82,6 +85,7 @@ for repo in "${repos[@]}"; do
> >  			# any packages might have been moved by the previous run
> >  			if [[ -f ${pkg} ]]; then
> >  				mv "${pkg}" "$FTP_BASE/${PKGPOOL}"
> > +				make_reproducible "${FTP_BASE}/${PKGPOOL}${pkg##*/}"
> >  			fi
> >  			ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}"
> >  			# also move signatures
> > 
> 
> 
> -- 
> Eli Schwartz
> Bug Wrangler and Trusted User
> 




-- 
Jelle van der Waa
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: not available
URL: <https://lists.archlinux.org/pipermail/arch-devops/attachments/20181212/84dae3a6/attachment-0001.asc>