[arch-projects] [dbscripts] [PATCH 2/4] Add reproducible archive of packages.

[Eli Schwartz]

Whenever adding new package files to the pool of distributed packages,
hardlink a copy of every package it was built with, into a
"reproducible" pool, and log which file required it.

Signed-off-by: Eli Schwartz <eschwartz at archlinux.org>
---
 config                     |  1 +
 config.local.svn-community |  1 +
 config.local.svn-packages  |  1 +
 db-functions               | 49 +++++++++++++++++++++++++++++++-------
 db-update                  |  4 ++++
 5 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/config b/config
index 1cfe11f4..5144fca7 100644
--- a/config
+++ b/config
@@ -3,6 +3,7 @@
 FTP_BASE="/srv/ftp"
 PKGREPOS=()
 PKGPOOL=''
+EXTRA_PKGPOOLS=()
 SRCPOOL=''
 TESTING_REPO=''
 STABLE_REPOS=()
diff --git a/config.local.svn-community b/config.local.svn-community
index 5d61b5ea..15bcc17f 100644
--- a/config.local.svn-community
+++ b/config.local.svn-community
@@ -2,6 +2,7 @@
 
 PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging')
 PKGPOOL='pool/community'
+EXTRA_PKGPOOLS=('pool/packages')
 SRCPOOL='sources/community'
 SVNREPO='file:///srv/repos/svn-community/svn'
 SVNUSER='svn-community'
diff --git a/config.local.svn-packages b/config.local.svn-packages
index 34aab35c..75986b65 100644
--- a/config.local.svn-packages
+++ b/config.local.svn-packages
@@ -2,6 +2,7 @@
 
 PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable')
 PKGPOOL='pool/packages'
+EXTRA_PKGPOOLS=('pool/community')
 SRCPOOL='sources/packages'
 SVNREPO='file:///srv/repos/svn-packages/svn'
 SVNUSER='svn-packages'
diff --git a/db-functions b/db-functions
index 7aeedced..2b1ae87a 100644
--- a/db-functions
+++ b/db-functions
@@ -165,20 +165,23 @@ repo_unlock () { #repo_unlock <repo-name> <arch>
 	fi
 }
 
+# usage: _grep_all_info pkgfile infofile key
+_grep_all_info() {
+	local _ret=()
+
+	mapfile -t _ret < <(/usr/bin/bsdtar -xOqf "$1" "${2}" | grep "^${3} = ")
+
+	printf '%s\n' "${_ret[@]#${3} = }"
+}
+
 # usage: _grep_pkginfo pkgfile pattern
 _grep_pkginfo() {
-	local _ret
-
-	_ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep "^${2} = " | tail -1)"
-	echo "${_ret#${2} = }"
+	_grep_all_info "${1}" .PKGINFO "${2}" | tail -1
 }
 
 # usage: _grep_buildinfo pkgfile pattern
 _grep_buildinfo() {
-	local _ret
-
-	_ret="$(/usr/bin/bsdtar -xOqf "$1" .BUILDINFO | grep "^${2} = " | tail -1)"
-	echo "${_ret#${2} = }"
+	_grep_all_info "${1}" .BUILDINFO "${2}" | tail -1
 }
 
 # Get the package base or name as fallback
@@ -444,4 +447,34 @@ arch_repo_modify() {
 	REPO_MODIFIED=1
 }
 
+# Build an index of dependent packages needed by a given pkgfile
+# usage: make_reproducible pkgfile [check]
+make_reproducible() {
+	local pkg dir pkgs=() pkgfile pkgfiles=()
+
+	mapfile -t pkgs < <(_grep_all_info "${1}" .BUILDINFO installed)
+
+	for pkg in "${pkgs[@]}"; do
+		for dir in "${FTP_BASE}/${PKGPOOL}" "${EXTRA_PKGPOOLS[@]/#/${FTP_BASE}/}" "${STAGING}"/**/; do
+			if pkgfile="$(getpkgfile "${dir}/${pkg}"${PKGEXTS} 2>/dev/null)"; then
+				pkgfiles+=("${pkgfile}")
+				continue 2
+			fi
+		done
+		error "could not find existing package for %s" "${pkg}"
+		return 1
+	done
+
+	if [[ ${2} = check ]]; then
+		return 0
+	fi
+
+	for pkg in "${pkgfiles[@]}"; do
+		if [[ ! -f ${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/} ]]; then
+			ln -L "${pkg}" "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}"
+		fi
+		echo "${1}" >> "${FTP_BASE}/${PKGPOOL}-reproducible/${pkg##*/}.buildlinks"
+	done
+}
+
 . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}"
diff --git a/db-update b/db-update
index 313fb999..11ec185f 100755
--- a/db-update
+++ b/db-update
@@ -61,6 +61,9 @@ for repo in "${repos[@]}"; do
 			if ! check_builddir "${pkg}"; then
 				die "Package %s was not built in a chroot" "$repo/${pkg##*/}"
 			fi
+			if ! make_reproducible "${pkg}" "check"; then
+				die "Package %s is not reproducible" "${pkg}"
+			fi
 		done
 		if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then
 			die "Missing split packages for %s" "$repo"
@@ -82,6 +85,7 @@ for repo in "${repos[@]}"; do
 			# any packages might have been moved by the previous run
 			if [[ -f ${pkg} ]]; then
 				mv "${pkg}" "$FTP_BASE/${PKGPOOL}"
+				make_reproducible "${FTP_BASE}/${PKGPOOL}${pkg##*/}"
 			fi
 			ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}"
 			# also move signatures
-- 
2.19.2