[pacman-dev] [PATCH 6/7] Always create files database
Allan McRae
allan at archlinux.org
Sun Mar 15 10:49:06 UTC 2015
Both the "db" and "files" databases are created in one call to repo-add.
Only the "foo.db.tar.xz" name is passed to repo-add.
Signed-off-by: Allan McRae <allan at archlinux.org>
---
scripts/repo-add.sh.in | 259 ++++++++++++++++++++++++++++++-------------------
1 file changed, 157 insertions(+), 102 deletions(-)
diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in
index 7512136..68f9829 100644
--- a/scripts/repo-add.sh.in
+++ b/scripts/repo-add.sh.in
@@ -32,11 +32,12 @@ QUIET=0
DELTA=0
ONLYADDNEW=0
RMEXISTING=0
-WITHFILES=0
SIGN=0
KEY=0
VERIFY=0
REPO_DB_FILE=
+REPO_DB_PREFIX=
+REPO_DB_SUFFIX=
LOCKFILE=
CLEAN_LOCK=0
USE_COLOR='y'
@@ -61,7 +62,6 @@ Multiple packages to add can be specified on the command line.\n")"
printf -- "$(gettext " -d, --delta generate and add delta for package update\n")"
printf -- "$(gettext " -n, --new only add packages that are not already in the database\n")"
printf -- "$(gettext " -R, --remove remove old package file from disk after updating database\n")"
- printf -- "$(gettext " -f, --files update database's file list\n")"
elif [[ $cmd == "repo-remove" ]] ; then
printf -- "$(gettext "Usage: repo-remove [options] <path-to-db> <packagename|delta> ...\n")"
printf -- "\n"
@@ -99,6 +99,7 @@ This is free software; see the source for copying conditions.\n\
There is NO WARRANTY, to the extent permitted by law.\n")"
}
+
# format a metadata entry
# arg1 - Entry name
# ... - value(s)
@@ -115,7 +116,8 @@ format_entry() {
find_pkgentry() {
local pkgname=$1
local pkgentry
- for pkgentry in "$tmpdir/tree/$pkgname"*; do
+
+ for pkgentry in "$tmpdir/db/$pkgname"*; do
name=${pkgentry##*/}
if [[ ${name%-*-*} = $pkgname ]]; then
echo $pkgentry
@@ -162,6 +164,12 @@ db_write_delta() {
msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile"
echo "${deltafile##*/} $md5sum $csize $oldfile $newfile" >> "$deltas"
+ # copy updated deltas entry into "files" database
+ # TODO: remove duplication across databases
+ local filesentry=$(echo "$pkgentry" | sed 's/\(.*\)\/db\//\1\/files\//')
+ mkdir -p "$filesentry"
+ cp $deltas "$filesentry"
+
return 0
} # end db_write_delta
@@ -188,6 +196,17 @@ db_remove_delta() {
msg2 "$(gettext "Removing empty deltas file ...")"
rm "$deltas"
fi
+
+ # copy updated deltas entry into "files" database
+ # TODO: remove duplication across databases
+ local filesentry=$(echo "$pkgentry" | sed 's/\(.*\)\/db\//\1\/files\//')
+ if [[ -f $deltas ]]; then
+ mkdir -p "$filesentry"
+ cp $deltas "$filesentry"
+ else
+ rm -f "$filesentry/deltas"
+ fi
+
return 0
fi
@@ -218,12 +237,14 @@ check_xdelta() {
if (( DELTA )); then
need_xdelta=1
else
- if [[ $cmd == "repo-add" ]];
+ if [[ $cmd == "repo-add" ]]; then
for f in ${args[@]:1}; do
case $f in
*.delta) need_xdelta=1 ;;
*) ;;
+ esac
done
+ fi
fi
if (( need_xdelta )); then
@@ -239,7 +260,7 @@ create_signature() {
(( ! SIGN )) && return
local dbfile=$1
local ret=0
- msg "$(gettext "Signing database...")"
+ msg "$(gettext "Signing database '%s'...")" "${dbfile##*/.tmp.}"
local SIGNWITHKEY=""
if [[ -n $GPGKEY ]]; then
@@ -250,7 +271,7 @@ create_signature() {
if (( ! ret )); then
msg2 "$(gettext "Created signature file '%s'")" "${dbfile##*/.tmp.}.sig"
else
- warning "$(gettext "Failed to sign package database.")"
+ warning "$(gettext "Failed to sign package database file '%s'")" "${dbfile##*/.tmp.}"
fi
}
@@ -278,11 +299,11 @@ verify_repo_extension() {
local repofile=$1
case $repofile in
- *.@(db|files).tar.gz) TAR_OPT="z" ;;
- *.@(db|files).tar.bz2) TAR_OPT="j" ;;
- *.@(db|files).tar.xz) TAR_OPT="J" ;;
- *.@(db|files).tar.Z) TAR_OPT="Z" ;;
- *.@(db|files).tar) TAR_OPT="" ;;
+ *.db.tar.gz) TAR_OPT="z" ;;
+ *.db.tar.bz2) TAR_OPT="j" ;;
+ *.db.tar.xz) TAR_OPT="J" ;;
+ *.db.tar.Z) TAR_OPT="Z" ;;
+ *.db.tar) TAR_OPT="" ;;
*) error "$(gettext "'%s' does not have a valid database archive extension.")" \
"$repofile"
exit 1 ;;
@@ -328,7 +349,7 @@ db_write_entry() {
return 1
fi
- if [[ -d $tmpdir/tree/$pkgname-$pkgver ]]; then
+ if [[ -d $tmpdir/db/$pkgname-$pkgver ]]; then
warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver"
if (( ONLYADDNEW )); then
return 0;
@@ -367,7 +388,7 @@ db_write_entry() {
db_remove_entry "$pkgname"
# create package directory
- pushd "$tmpdir/tree" >/dev/null
+ pushd "$tmpdir/db" >/dev/null
mkdir "$pkgname-$pkgver"
pushd "$pkgname-$pkgver" >/dev/null
@@ -415,14 +436,6 @@ db_write_entry() {
popd >/dev/null
popd >/dev/null
- # create files file if wanted
- if (( WITHFILES )); then
- msg2 "$(gettext "Creating '%s' db entry...")" 'files'
- local files_path="$tmpdir/tree/$pkgname-$pkgver/files"
- echo "%FILES%" >"$files_path"
- bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_path"
- fi
-
# create a delta file
if (( DELTA )); then
if [[ -n $oldfilename ]]; then
@@ -437,6 +450,16 @@ db_write_entry() {
fi
fi
+ # copy updated package entry into "files" database
+ # TODO: remove duplication across databases
+ cp -a "$tmpdir/db/$pkgname-$pkgver" "$tmpdir/files/$pkgname-$pkgver"
+
+ # create files file
+ msg2 "$(gettext "Creating '%s' db entry...")" 'files'
+ local files_path="$tmpdir/files/$pkgname-$pkgver/files"
+ echo "%FILES%" >"$files_path"
+ bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_path"
+
if (( RMEXISTING )); then
msg2 "$(gettext "Removing old package file '%s'")" "$oldfilename"
rm -f ${oldfile} ${oldfile}.sig
@@ -454,11 +477,17 @@ db_remove_entry() {
while [[ -n $pkgentry ]]; do
notfound=0
if [[ -f $pkgentry/deltas ]]; then
- mv "$pkgentry/deltas" "$tmpdir/tree/$pkgname.deltas"
+ mv "$pkgentry/deltas" "$tmpdir/db/$pkgname.deltas"
fi
msg2 "$(gettext "Removing existing entry '%s'...")" \
"${pkgentry##*/}"
rm -rf "$pkgentry"
+
+ # remove entries in "files" database
+ # TODO: remove duplication across databases
+ local filesentry=$(echo "$pkgentry" | sed 's/\(.*\)\/db\//\1\/files\//')
+ rm -rf "$filesentry"
+
pkgentry=$(find_pkgentry "$pkgname")
done
return $notfound
@@ -479,8 +508,8 @@ elephant() {
esac | openssl base64 -d | gzip -d
}
-check_repo_db() {
- local repodir
+prepare_repo_db() {
+ local repodir dbfile
# ensure the path to the DB exists; $LOCKFILE is always an absolute path
repodir=${LOCKFILE%/*}/
@@ -499,35 +528,43 @@ check_repo_db() {
exit 1
fi
- if [[ -f $REPO_DB_FILE ]]; then
- # there are two situations we can have here- a DB with some entries,
- # or a DB with no contents at all.
- if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then
- # check empty case
- if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then
- error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE"
- exit 1
+ for repo in "db" "files"; do
+ dbfile=${repodir}/$REPO_DB_PREFIX.$repo.$REPO_DB_SUFFIX
+
+ if [[ -f $dbfile ]]; then
+ # there are two situations we can have here:
+ # a DB with some entries, or a DB with no contents at all.
+ if ! bsdtar -tqf "$dbfile" '*/desc' >/dev/null 2>&1; then
+ # check empty case
+ if [[ -n $(bsdtar -tqf "$dbfile" '*' 2>/dev/null) ]]; then
+ error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$dbfile"
+ exit 1
+ fi
fi
+ verify_signature "$dbfile"
+ msg "$(gettext "Extracting database to a temporary location...")"
+ bsdtar -xf "$dbfile" -C "$tmpdir/$repo"
+ else
+ case $cmd in
+ repo-remove)
+ # only a missing "db" database is currently an error
+ # TODO: remove if statement
+ if [[ $repo == "db" ]]; then
+ error "$(gettext "Repository file '%s' was not found.")" "$dbfile"
+ fi
+ exit 1
+ ;;
+ repo-add)
+ # check if the file can be created (write permission, directory existence, etc)
+ if ! touch "$dbfile"; then
+ error "$(gettext "Repository file '%s' could not be created.")" "$dbfile"
+ exit 1
+ fi
+ rm -f "$dbfile"
+ ;;
+ esac
fi
- verify_signature "$REPO_DB_FILE"
- msg "$(gettext "Extracting database to a temporary location...")"
- bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir/tree"
- else
- case $cmd in
- repo-remove)
- error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE"
- exit 1
- ;;
- repo-add)
- # check if the file can be created (write permission, directory existence, etc)
- if ! touch "$REPO_DB_FILE"; then
- error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE"
- exit 1
- fi
- rm -f "$REPO_DB_FILE"
- ;;
- esac
- fi
+ done
}
add() {
@@ -536,7 +573,7 @@ add() {
return 1
fi
- if [[ ${1##*.} == "delta" ]]; then
+ if [[ $1 = *-*-*_to_*-*-*.delta ]]; then
deltafile=$1
msg "$(gettext "Adding delta '%s'")" "$deltafile"
if db_write_delta "$deltafile"; then
@@ -582,60 +619,71 @@ remove() {
}
rotate_db() {
- filename=${REPO_DB_FILE##*/}
- tempname=$dirname/.tmp.$filename
-
- # hardlink or move the previous version of the database and signature to .old
- # extension as a backup measure
- if [[ -f $REPO_DB_FILE ]]; then
- ln -f "$REPO_DB_FILE" "$REPO_DB_FILE.old" 2>/dev/null || \
- mv -f "$REPO_DB_FILE" "$REPO_DB_FILE.old"
-
- if [[ -f $REPO_DB_FILE.sig ]]; then
- ln -f "$REPO_DB_FILE.sig" "$REPO_DB_FILE.old.sig" 2>/dev/null || \
- mv -f "$REPO_DB_FILE.sig" "$REPO_DB_FILE.old.sig"
- else
- rm -f "$REPO_DB_FILE.old.sig"
+ dirname=${LOCKFILE%/*}
+
+ pushd $dirname >/dev/null
+
+ for repo in "db" "files"; do
+ filename=${REPO_DB_PREFIX}.${repo}.${REPO_DB_SUFFIX}
+ tempname=$dirname/.tmp.$filename
+
+ # hardlink or move the previous version of the database and signature to .old
+ # extension as a backup measure
+ if [[ -f $filename ]]; then
+ ln -f "$filename" "$filename.old" 2>/dev/null || \
+ mv -f "$filename" "$filename.old"
+
+ if [[ -f $filename.sig ]]; then
+ ln -f "$filename.sig" "$filename.old.sig" 2>/dev/null || \
+ mv -f "$filename.sig" "$filename.old.sig"
+ else
+ rm -f "$filename.old.sig"
+ fi
fi
- fi
- # rotate the newly-created database and signature into place
- mv "$tempname" "$REPO_DB_FILE"
- if [[ -f $tempname.sig ]]; then
- mv "$tempname.sig" "$REPO_DB_FILE.sig"
- fi
+ # rotate the newly-created database and signature into place
+ mv "$tempname" "$filename"
+ if [[ -f $tempname.sig ]]; then
+ mv "$tempname.sig" "$filename.sig"
+ fi
- dblink=${REPO_DB_FILE%.tar*}
- rm -f "$dblink" "$dblink.sig"
- ln -s "$filename" "$dblink" 2>/dev/null || \
- ln "$filename" "$dblink" 2>/dev/null || \
- cp "$REPO_DB_FILE" "$dblink"
- if [[ -f "$REPO_DB_FILE.sig" ]]; then
- ln -s "$filename.sig" "$dblink.sig" 2>/dev/null || \
- ln "$filename.sig" "$dblink.sig" 2>/dev/null || \
- cp "$REPO_DB_FILE.sig" "$dblink.sig"
- fi
+ dblink=${filename%.tar*}
+ rm -f "$dblink" "$dblink.sig"
+ ln -s "$filename" "$dblink" 2>/dev/null || \
+ ln "$filename" "$dblink" 2>/dev/null || \
+ cp "$filename" "$dblink"
+ if [[ -f "$filename.sig" ]]; then
+ ln -s "$filename.sig" "$dblink.sig" 2>/dev/null || \
+ ln "$filename.sig" "$dblink.sig" 2>/dev/null || \
+ cp "$filename.sig" "$dblink.sig"
+ fi
+ done
+
+ popd >/dev/null
}
create_db() {
TAR_OPT=$(verify_repo_extension "$REPO_DB_FILE")
# $LOCKFILE is already guaranteed to be absolute so this is safe
dirname=${LOCKFILE%/*}
- filename=${REPO_DB_FILE##*/}
- # this ensures we create it on the same filesystem, making moves atomic
- tempname=$dirname/.tmp.$filename
- pushd "$tmpdir/tree" >/dev/null
- if ( shopt -s nullglob; files=(*); (( ${#files[*]} )) ); then
- bsdtar -c${TAR_OPT}f "$tempname" *
- else
- # we have no packages remaining? zip up some emptyness
- warning "$(gettext "No packages remain, creating empty database.")"
- bsdtar -c${TAR_OPT}f "$tempname" -T /dev/null
- fi
- popd >/dev/null
+ for repo in "db" "files"; do
+ filename=${REPO_DB_PREFIX}.${repo}.${REPO_DB_SUFFIX}
+ # this ensures we create it on the same filesystem, making moves atomic
+ tempname=$dirname/.tmp.$filename
+
+ pushd "$tmpdir/$repo" >/dev/null
+ if ( shopt -s nullglob; files=(*); (( ${#files[*]} )) ); then
+ bsdtar -c${TAR_OPT}f "$tempname" *
+ else
+ # we have no packages remaining? zip up some emptyness
+ warning "$(gettext "No packages remain, creating empty database.")"
+ bsdtar -c${TAR_OPT}f "$tempname" -T /dev/null
+ fi
+ popd >/dev/null
- create_signature "$tempname"
+ create_signature "$tempname"
+ done
}
trap_exit() {
@@ -659,6 +707,7 @@ clean_up() {
exit $exit_code
}
+
# PROGRAM START
# determine whether we have gettext; make it a no-op if we do not
@@ -688,7 +737,10 @@ fi
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/repo-tools.XXXXXXXXXX") || (\
error "$(gettext "Cannot create temp directory for database building.")"; \
exit 1)
-mkdir "$tmpdir/tree"
+
+for repo in "db" "files"; do
+ mkdir "$tmpdir/$repo"
+done
trap 'clean_up' EXIT
for signal in TERM HUP QUIT; do
@@ -705,7 +757,6 @@ while (( $# )); do
-d|--delta) DELTA=1;;
-n|--new) ONLYADDNEW=1;;
-R|--remove) RMEXISTING=1;;
- -f|--files) WITHFILES=1;;
--nocolor) USE_COLOR='n';;
-s|--sign)
SIGN=1
@@ -740,23 +791,27 @@ else
fi
verify_repo_extension "$REPO_DB_FILE" >/dev/null
-check_repo_db
-if (( SIGN || KEY || VERIFY )); then
+REPO_DB_PREFIX=${REPO_DB_FILE##*/}
+REPO_DB_PREFIX=${REPO_DB_PREFIX%.db.*}
+REPO_DB_SUFFIX=${REPO_DB_FILE##*.db.}
+
+if (( SIGN || VERIFY )); then
check_gpg
fi
check_xdelta
-fail=0
+prepare_repo_db
+
for arg in "${args[@]:1}"; do
case $cmd in
repo-add) add "$arg" ;;
repo-remove) remove "$arg" ;;
- esac || fail=1
+ esac && success=1
done
-# if the whole operation was a success, re-zip and rotate database
+# if the whole operation was a success, re-zip and rotate databases
if (( !fail )); then
msg "$(gettext "Creating updated database file '%s'")" "$REPO_DB_FILE"
create_db
--
2.3.3
More information about the pacman-dev
mailing list