[pacman-dev] [PATCH v2] repo-add: always create files database

27 Jan 2015

Signed-off-by: Allan McRae <allan@archlinux.org>
---

v2 - fix updating repos

 scripts/repo-add.sh.in | 220 +++++++++++++++++++++++++++----------------------
 1 file changed, 122 insertions(+), 98 deletions(-)

diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in
index 21cb2aa..8601fbb 100644
--- a/scripts/repo-add.sh.in
+++ b/scripts/repo-add.sh.in
@@ -32,7 +32,6 @@ QUIET=0
 DELTA=0
 ONLYADDNEW=0
 RMEXISTING=0
-WITHFILES=0
 SIGN=0
 KEY=0
 VERIFY=0
@@ -61,7 +60,6 @@ Multiple packages to add can be specified on the command line.\n")"
 		printf -- "$(gettext "  -d, --delta       generate and add delta for package update\n")"
 		printf -- "$(gettext "  -n, --new         only add packages that are not already in the database\n")"
 		printf -- "$(gettext "  -R, --remove      remove old package file from disk after updating database\n")"
-		printf -- "$(gettext "  -f, --files       update database's file list\n")"
 	elif [[ $cmd == "repo-remove" ]] ; then
 		printf -- "$(gettext "Usage: repo-remove [options] <path-to-db> <packagename|delta> ...\n")"
 		printf -- "\n"
@@ -115,7 +113,7 @@ format_entry() {
 find_pkgentry() {
 	local pkgname=$1
 	local pkgentry
-	for pkgentry in "$tmpdir/tree/$pkgname"*; do
+	for pkgentry in "$tmpdir/repo/$pkgname"*; do
 		name=${pkgentry##*/}
 		if [[ ${name%-*-*} = $pkgname ]]; then
 			echo $pkgentry
@@ -145,6 +143,7 @@ db_write_delta() {
 		return 1
 	fi
 	deltas=$pkgentry/deltas
+	files_deltas=$(echo "$deltas" | sed 's/\(.*\)\/repo\//\1\/files\//')
 	if [[ ! -f $deltas ]]; then
 		echo -e "%DELTAS%" >"$deltas"
 	fi
@@ -161,6 +160,7 @@ db_write_delta() {
 	fi
 	msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile"
 	echo "${deltafile##*/} $md5sum $csize $oldfile $newfile" >> "$deltas"
+	cp "$deltas" "$files_deltas"
 
 	return 0
 } # end db_write_delta
@@ -177,16 +177,18 @@ db_remove_delta() {
 		return 1
 	fi
 	deltas=$pkgentry/deltas
+	files_deltas=$(echo "${deltas}" | sed 's/\(.*\)\/repo\//\1\/files\//')
 	if [[ ! -f $deltas ]]; then
 		return 1
 	fi
 	if grep -q "$filename" "$deltas"; then
-		sed -i.backup "/$filename/d" "$deltas" && rm -f "$deltas.backup"
 		msg2 "$(gettext "Removing existing entry '%s'...")" "$filename"
+		sed -i.backup "/$filename/d" "$deltas" && rm -f "$deltas.backup"
+		cp "$deltas" "$files_deltas"
 		# empty deltas file contains only "%DELTAS%"
 		if (( $(wc -l < "$deltas") == 1 )); then
 			msg2 "$(gettext "Removing empty deltas file ...")"
-			rm "$deltas"
+			rm "$deltas" "$files_deltas"
 		fi
 		return 0
 	fi
@@ -256,11 +258,11 @@ verify_repo_extension() {
 	local repofile=$1
 
 	case $repofile in
-		*.@(db|files).tar.gz)  TAR_OPT="z" ;;
-		*.@(db|files).tar.bz2) TAR_OPT="j" ;;
-		*.@(db|files).tar.xz)  TAR_OPT="J" ;;
-		*.@(db|files).tar.Z)   TAR_OPT="Z" ;;
-		*.@(db|files).tar)     TAR_OPT="" ;;
+		*.db.tar.gz)  TAR_OPT="z" ;;
+		*.db.tar.bz2) TAR_OPT="j" ;;
+		*.db.tar.xz)  TAR_OPT="J" ;;
+		*.db.tar.Z)   TAR_OPT="Z" ;;
+		*.db.tar)     TAR_OPT="" ;;
 		*) error "$(gettext "'%s' does not have a valid database archive extension.")" \
 				"$repofile"
 			exit 1 ;;
@@ -306,7 +308,7 @@ db_write_entry() {
 		return 1
 	fi
 
-	if [[ -d $tmpdir/tree/$pkgname-$pkgver ]]; then
+	if [[ -d $tmpdir/repo/$pkgname-$pkgver ]]; then
 		warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver"
 		if (( ONLYADDNEW )); then
 			return 0;
@@ -353,7 +355,7 @@ db_write_entry() {
 	db_remove_entry "$pkgname"
 
 	# create package directory
-	pushd "$tmpdir/tree" >/dev/null
+	pushd "$tmpdir/repo" >/dev/null
 	mkdir "$pkgname-$pkgver"
 	pushd "$pkgname-$pkgver" >/dev/null
 
@@ -402,12 +404,11 @@ db_write_entry() {
 	popd >/dev/null
 
 	# create files file if wanted
-	if (( WITHFILES )); then
-		msg2 "$(gettext "Creating '%s' db entry...")" 'files'
-		local files_path="$tmpdir/tree/$pkgname-$pkgver/files"
-		echo "%FILES%" >"$files_path"
-		bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_path"
-	fi
+	msg2 "$(gettext "Creating '%s' db entry...")" 'files'
+	local files_dir="$tmpdir/files/$pkgname-$pkgver"
+	mkdir "${files_dir}"
+	echo "%FILES%" >"$files_dir/files"
+	bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_dir/files"
 
 	# create a delta file
 	if (( DELTA )); then
@@ -423,6 +424,10 @@ db_write_entry() {
 		fi
 	fi
 
+	# sync files db with main db
+	rm -f $tmpdir/files/$pkgname-$pkgver/{desc,depends,deltas}
+	cp $tmpdir/repo/$pkgname-$pkgver/* $tmpdir/files/$pkgname-$pkgver/
+
 	if (( RMEXISTING )); then
 		msg2 "$(gettext "Removing old package file '%s'")" "$oldfilename"
 		rm -f ${oldfile} ${oldfile}.sig
@@ -437,14 +442,16 @@ db_remove_entry() {
 	local pkgname=$1
 	local notfound=1
 	local pkgentry=$(find_pkgentry "$pkgname")
+
 	while [[ -n $pkgentry ]]; do
 		notfound=0
+		local files_pkgentry=$(echo "$pkgentry" | sed 's/\(.*\)\/repo\//\1\/files\//')
 		if [[ -f $pkgentry/deltas ]]; then
-			mv "$pkgentry/deltas" "$tmpdir/tree/$pkgname.deltas"
+			mv "$pkgentry/deltas" "$tmpdir/repo/$pkgname.deltas"
 		fi
 		msg2 "$(gettext "Removing existing entry '%s'...")" \
 		"${pkgentry##*/}"
-		rm -rf "$pkgentry"
+		rm -rf "$pkgentry" "$files_pkgentry"
 		pkgentry=$(find_pkgentry "$pkgname")
 	done
 	return $notfound
@@ -465,57 +472,63 @@ elephant() {
 	esac | openssl base64 -d | gzip -d
 }
 
-check_repo_db() {
-	local repodir
-
-	# ensure the path to the DB exists; $LOCKFILE is always an absolute path
-	repodir=${LOCKFILE%/*}/
-
-	if [[ ! -d $repodir ]]; then
-		error "$(gettext "%s does not exist or is not a directory.")" "$repodir"
-		exit 1
-	fi
-
-	# check lock file
-	if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then
-		CLEAN_LOCK=1
-	else
-		error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE"
-		[[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat "$LOCKFILE")"
-		exit 1
-	fi
+validate_repo() {
+	local type=$1 repo=$2
 
-	if [[ -f $REPO_DB_FILE ]]; then
+	if [[ -f $repo ]]; then
 		# there are two situations we can have here- a DB with some entries,
 		# or a DB with no contents at all.
-		if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then
+		if ! bsdtar -tqf "$repo" '*/desc' >/dev/null 2>&1; then
 			# check empty case
-			if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then
+			if [[ -n $(bsdtar -tqf "$repo" '*' 2>/dev/null) ]]; then
 				error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE"
 				exit 1
 			fi
 		fi
-		verify_signature "$REPO_DB_FILE"
-		msg "$(gettext "Extracting database to a temporary location...")"
-		bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir/tree"
+		verify_signature "$repo"
+		bsdtar -xf "$repo" -C "$tmpdir/$type"
 	else
 		case $cmd in
 			repo-remove)
-			error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE"
+			error "$(gettext "Repository file '%s' was not found.")" "$repo"
 			exit 1
 			;;
 			repo-add)
 			# check if the file can be created (write permission, directory existence, etc)
-			if ! touch "$REPO_DB_FILE"; then
-				error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE"
+			if ! touch "$repo"; then
+				error "$(gettext "Repository file '%s' could not be created.")" "$repo"
 				exit 1
 			fi
-			rm -f "$REPO_DB_FILE"
+			rm -f "$repo"
 			;;
 		esac
 	fi
 }
 
+check_repo_dbs() {
+	local repodir
+
+	# ensure the path to the DB exists; $LOCKFILE is always an absolute path
+	repodir=${LOCKFILE%/*}/
+
+	if [[ ! -d $repodir ]]; then
+		error "$(gettext "%s does not exist or is not a directory.")" "$repodir"
+		exit 1
+	fi
+
+	# check lock file
+	if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then
+		CLEAN_LOCK=1
+	else
+		error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE"
+		[[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat "$LOCKFILE")"
+		exit 1
+	fi
+
+	validate_repo repo $REPO_DB_FILE
+	validate_repo files $FILES_DB_FILE
+}
+
 add() {
 	if [[ ! -f $1 ]]; then
 		error "$(gettext "File '%s' not found.")" "$1"
@@ -563,7 +576,7 @@ remove() {
 	msg "$(gettext "Searching for package '%s'...")" "$pkgname"
 
 	if db_remove_entry "$pkgname"; then
-		rm -f "$tmpdir/tree/$pkgname.deltas"
+		rm -f "$tmpdir/repo/$pkgname.deltas"
 		return 0
 	else
 		error "$(gettext "Package matching '%s' not found.")" "$pkgname"
@@ -571,40 +584,64 @@ remove() {
 	fi
 }
 
-rotate_db() {
-	filename=${REPO_DB_FILE##*/}
-	tempname=$dirname/.tmp.$filename
+create_db() {
+		local type=$1 db=$2
 
-	# hardlink or move the previous version of the database and signature to .old
-	# extension as a backup measure
-	if [[ -f $REPO_DB_FILE ]]; then
-		ln -f "$REPO_DB_FILE" "$REPO_DB_FILE.old" 2>/dev/null || \
-			mv -f "$REPO_DB_FILE" "$REPO_DB_FILE.old"
+		dirname=${LOCKFILE%/*}
+		filename=${db##*/}
+		# this ensures we create it on the same filesystem, making moves atomic
+		tempname=$dirname/.tmp.$filename
 
-		if [[ -f $REPO_DB_FILE.sig ]]; then
-			ln -f "$REPO_DB_FILE.sig" "$REPO_DB_FILE.old.sig" 2>/dev/null || \
-				mv -f "$REPO_DB_FILE.sig" "$REPO_DB_FILE.old.sig"
+		pushd "$tmpdir/$type" >/dev/null
+		if ( shopt -s nullglob; files=(*); (( ${#files[*]} )) ); then
+			bsdtar -c${TAR_OPT}f "$tempname" *
 		else
-			rm -f "$REPO_DB_FILE.old.sig"
+			# we have no packages remaining? zip up some emptyness
+			warning "$(gettext "No packages remain, creating empty database.")"
+			bsdtar -c${TAR_OPT}f "$tempname" -T /dev/null
 		fi
-	fi
+		popd >/dev/null
 
-	# rotate the newly-created database and signature into place
-	mv "$tempname" "$REPO_DB_FILE"
-	if [[ -f $tempname.sig ]]; then
-		mv "$tempname.sig" "$REPO_DB_FILE.sig"
-	fi
+		create_signature "$tempname"
+}
 
-	dblink=${REPO_DB_FILE%.tar*}
-	rm -f "$dblink" "$dblink.sig"
-	ln -s "$filename" "$dblink" 2>/dev/null || \
-		ln "$filename" "$dblink" 2>/dev/null || \
-		cp "$REPO_DB_FILE" "$dblink"
-	if [[ -f "$REPO_DB_FILE.sig" ]]; then
-		ln -s "$filename.sig" "$dblink.sig" 2>/dev/null || \
-			ln "$filename.sig" "$dblink.sig" 2>/dev/null || \
-			cp "$REPO_DB_FILE.sig" "$dblink.sig"
-	fi
+rotate_dbs() {
+	for db in "$REPO_DB_FILE" "$FILES_DB_FILE"; do
+
+		filename=${db##*/}
+		tempname=$dirname/.tmp.$filename
+
+		# hardlink or move the previous version of the database and signature to .old
+		# extension as a backup measure
+		if [[ -f $db ]]; then
+			ln -f "$db" "$db.old" 2>/dev/null || \
+				mv -f "$db" "$db.old"
+
+			if [[ -f $db.sig ]]; then
+				ln -f "$db.sig" "$db.old.sig" 2>/dev/null || \
+					mv -f "$db.sig" "$db.old.sig"
+			else
+				rm -f "$db.old.sig"
+			fi
+		fi
+
+		# rotate the newly-created database and signature into place
+		mv "$tempname" "$db"
+		if [[ -f $tempname.sig ]]; then
+			mv "$tempname.sig" "$db.sig"
+		fi
+
+		dblink=${db%.tar*}
+		rm -f "$dblink" "$dblink.sig"
+		ln -s "$filename" "$dblink" 2>/dev/null || \
+			ln "$filename" "$dblink" 2>/dev/null || \
+			cp "$db" "$dblink"
+		if [[ -f "$db.sig" ]]; then
+			ln -s "$filename.sig" "$dblink.sig" 2>/dev/null || \
+				ln "$filename.sig" "$dblink.sig" 2>/dev/null || \
+				cp "$db.sig" "$dblink.sig"
+		fi
+	done
 }
 
 trap_exit() {
@@ -657,7 +694,7 @@ fi
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/repo-tools.XXXXXXXXXX") || (\
 	error "$(gettext "Cannot create temp directory for database building.")"; \
 	exit 1)
-mkdir "$tmpdir/tree"
+mkdir "$tmpdir/repo" "$tmpdir/files"
 
 trap 'clean_up' EXIT
 for signal in TERM HUP QUIT; do
@@ -675,7 +712,6 @@ while (( $# )); do
 		-d|--delta) DELTA=1;;
 		-n|--new) ONLYADDNEW=1;;
 		-R|--remove) RMEXISTING=1;;
-		-f|--files) WITHFILES=1;;
 		--nocolor) USE_COLOR='n';;
 		-s|--sign)
 			SIGN=1
@@ -703,6 +739,8 @@ if [[ -z $REPO_DB_FILE ]]; then
 	exit 1
 fi
 
+FILES_DB_FILE=${REPO_DB_FILE%db.tar*}files.tar${REPO_DB_FILE##*.db.tar}
+
 if [[ $REPO_DB_FILE == /* ]]; then
 	LOCKFILE=$REPO_DB_FILE.lck
 else
@@ -710,7 +748,7 @@ else
 fi
 
 verify_repo_extension "$REPO_DB_FILE" >/dev/null
-check_repo_db
+check_repo_dbs
 
 if (( SIGN || KEY || VERIFY )); then
 	check_gpg
@@ -729,24 +767,10 @@ if (( success )); then
 
 	TAR_OPT=$(verify_repo_extension "$REPO_DB_FILE")
 	# $LOCKFILE is already guaranteed to be absolute so this is safe
-	dirname=${LOCKFILE%/*}
-	filename=${REPO_DB_FILE##*/}
-	# this ensures we create it on the same filesystem, making moves atomic
-	tempname=$dirname/.tmp.$filename
-
-	pushd "$tmpdir/tree" >/dev/null
-	if ( shopt -s nullglob; files=(*); (( ${#files[*]} )) ); then
-		bsdtar -c${TAR_OPT}f "$tempname" *
-	else
-		# we have no packages remaining? zip up some emptyness
-		warning "$(gettext "No packages remain, creating empty database.")"
-		bsdtar -c${TAR_OPT}f "$tempname" -T /dev/null
-	fi
-	popd >/dev/null
-
-	create_signature "$tempname"
 
-	rotate_db
+	create_db repo $REPO_DB_FILE
+	create_db files $FILES_DB_FILE
+	rotate_dbs
 else
 	msg "$(gettext "No packages modified, nothing to do.")"
 	exit 1
-- 
2.2.2

    