[arch-projects] [dbscripts] [PATCH v3 2/2] Add reproducible archive of packages.

[Eli Schwartz]

Whenever adding new package files to the pool of distributed packages,
copy the file into a longterm archive. This is the first step to merging
the functionality of archivetools, as this implements the shared pool
while also guaranteeing that all packages are archived at the time of
entry rather than once per day if they still exist.

Signed-off-by: Eli Schwartz <eschwartz at archlinux.org>
---

v3: string changes, consistently use BASH_SOURCE

 config                    |  2 ++
 db-archive                | 21 +++++++++++++++++++++
 db-functions              | 39 +++++++++++++++++++++++++++++++--------
 db-update                 |  5 +++++
 test/cases/db-update.bats |  6 ++++++
 5 files changed, 65 insertions(+), 8 deletions(-)
 create mode 100755 db-archive

diff --git a/config b/config
index 1cfe11f4..57a2cc47 100644
--- a/config
+++ b/config
@@ -1,6 +1,8 @@
 #!/hint/bash
 
 FTP_BASE="/srv/ftp"
+ARCHIVE_BASE="/srv/archive"
+ARCHIVEUSER='archive'
 PKGREPOS=()
 PKGPOOL=''
 SRCPOOL=''
diff --git a/db-archive b/db-archive
new file mode 100755
index 00000000..f3fa2fea
--- /dev/null
+++ b/db-archive
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+. "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/config"
+
+if (( $# != 1 )); then
+	echo "usage: %s <pkgfile>" "${0##*/}"
+	exit 1
+fi
+
+if [[ -n ${ARCHIVEUSER} ]]; then
+	exec sudo -u "${ARCHIVEUSER}" bash "${BASH_SOURCE[0]}" "${@}"
+fi
+
+pkgfile=${1##*/}
+pkgname=${pkgfile%-*-*-*}
+archive_dir="${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}"
+
+if [[ ! -f ${archive_dir}/${pkgfile} ]]; then
+	mkdir -p "${archive_dir}"
+	cp -np "${1}"{,.sig} "${archive_dir}/"
+fi
diff --git a/db-functions b/db-functions
index 7aeedced..0e4dd939 100644
--- a/db-functions
+++ b/db-functions
@@ -165,20 +165,23 @@ repo_unlock () { #repo_unlock <repo-name> <arch>
 	fi
 }
 
+# usage: _grep_all_info pkgfile infofile key
+_grep_all_info() {
+	local _ret=()
+
+	mapfile -t _ret < <(/usr/bin/bsdtar -xOqf "$1" "${2}" | grep "^${3} = ")
+
+	printf '%s\n' "${_ret[@]#${3} = }"
+}
+
 # usage: _grep_pkginfo pkgfile pattern
 _grep_pkginfo() {
-	local _ret
-
-	_ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep "^${2} = " | tail -1)"
-	echo "${_ret#${2} = }"
+	_grep_all_info "${1}" .PKGINFO "${2}" | tail -1
 }
 
 # usage: _grep_buildinfo pkgfile pattern
 _grep_buildinfo() {
-	local _ret
-
-	_ret="$(/usr/bin/bsdtar -xOqf "$1" .BUILDINFO | grep "^${2} = " | tail -1)"
-	echo "${_ret#${2} = }"
+	_grep_all_info "${1}" .BUILDINFO "${2}" | tail -1
 }
 
 # Get the package base or name as fallback
@@ -444,4 +447,24 @@ arch_repo_modify() {
 	REPO_MODIFIED=1
 }
 
+# Verify the existence of dependent packages needed by a given pkgfile
+# usage: check_reproducible pkgfile
+check_reproducible() {
+	local pkg dir pkgs=() pkgfile pkgfiles=()
+
+	mapfile -t pkgs < <(_grep_all_info "${1}" .BUILDINFO installed)
+
+	for pkg in "${pkgs[@]}"; do
+		local pkgname=${pkg%-*-*-*}
+		for dir in "${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}" "${STAGING}"/**/; do
+			if pkgfile="$(getpkgfile "${dir}/${pkg}"${PKGEXTS} 2>/dev/null)"; then
+				pkgfiles+=("${pkgfile}")
+				continue 2
+			fi
+		done
+		error "could not find existing or staged package for dependency %s" "${pkg}"
+		return 1
+	done
+}
+
 . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}"
diff --git a/db-update b/db-update
index 313fb999..3a66c859 100755
--- a/db-update
+++ b/db-update
@@ -61,6 +61,10 @@ for repo in "${repos[@]}"; do
 			if ! check_builddir "${pkg}"; then
 				die "Package %s was not built in a chroot" "$repo/${pkg##*/}"
 			fi
+			if ! check_reproducible "${pkg}"; then
+				error "Package %s is not reproducible." "${pkg}"
+				die "Ensure that all dependencies are available in the repositories or are added in the same db-update."
+			fi
 		done
 		if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then
 			die "Missing split packages for %s" "$repo"
@@ -82,6 +86,7 @@ for repo in "${repos[@]}"; do
 			# any packages might have been moved by the previous run
 			if [[ -f ${pkg} ]]; then
 				mv "${pkg}" "$FTP_BASE/${PKGPOOL}"
+				"$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-archive" "${FTP_BASE}/${PKGPOOL}/${pkg##*/}"
 			fi
 			ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}"
 			# also move signatures
diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats
index 9ee06321..bc978302 100644
--- a/test/cases/db-update.bats
+++ b/test/cases/db-update.bats
@@ -87,6 +87,12 @@ load ../lib/common
 	checkPackage testing pkg-any-a 1-2
 }
 
+ at test "archive package when releasing" {
+	releasePackage extra pkg-any-a
+	db-update
+	[[ -f ${ARCHIVE_BASE}/packages/p/pkg-any-a/pkg-any-a-1-1-any${PKGEXT} ]]
+}
+
 @test "update same any package to same repository fails" {
 	releasePackage extra pkg-any-a
 	db-update
-- 
2.20.1