[pacman-dev] [PATCH v2] repo-add: always create files database
Allan McRae
allan at archlinux.org
Tue Jan 27 12:40:01 UTC 2015
Signed-off-by: Allan McRae <allan at archlinux.org>
---
v2 - fix updating repos
scripts/repo-add.sh.in | 220 +++++++++++++++++++++++++++----------------------
1 file changed, 122 insertions(+), 98 deletions(-)
diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in
index 21cb2aa..8601fbb 100644
--- a/scripts/repo-add.sh.in
+++ b/scripts/repo-add.sh.in
@@ -32,7 +32,6 @@ QUIET=0
DELTA=0
ONLYADDNEW=0
RMEXISTING=0
-WITHFILES=0
SIGN=0
KEY=0
VERIFY=0
@@ -61,7 +60,6 @@ Multiple packages to add can be specified on the command line.\n")"
printf -- "$(gettext " -d, --delta generate and add delta for package update\n")"
printf -- "$(gettext " -n, --new only add packages that are not already in the database\n")"
printf -- "$(gettext " -R, --remove remove old package file from disk after updating database\n")"
- printf -- "$(gettext " -f, --files update database's file list\n")"
elif [[ $cmd == "repo-remove" ]] ; then
printf -- "$(gettext "Usage: repo-remove [options] <path-to-db> <packagename|delta> ...\n")"
printf -- "\n"
@@ -115,7 +113,7 @@ format_entry() {
find_pkgentry() {
local pkgname=$1
local pkgentry
- for pkgentry in "$tmpdir/tree/$pkgname"*; do
+ for pkgentry in "$tmpdir/repo/$pkgname"*; do
name=${pkgentry##*/}
if [[ ${name%-*-*} = $pkgname ]]; then
echo $pkgentry
@@ -145,6 +143,7 @@ db_write_delta() {
return 1
fi
deltas=$pkgentry/deltas
+ files_deltas=$(echo "$deltas" | sed 's/\(.*\)\/repo\//\1\/files\//')
if [[ ! -f $deltas ]]; then
echo -e "%DELTAS%" >"$deltas"
fi
@@ -161,6 +160,7 @@ db_write_delta() {
fi
msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile"
echo "${deltafile##*/} $md5sum $csize $oldfile $newfile" >> "$deltas"
+ cp "$deltas" "$files_deltas"
return 0
} # end db_write_delta
@@ -177,16 +177,18 @@ db_remove_delta() {
return 1
fi
deltas=$pkgentry/deltas
+ files_deltas=$(echo "${deltas}" | sed 's/\(.*\)\/repo\//\1\/files\//')
if [[ ! -f $deltas ]]; then
return 1
fi
if grep -q "$filename" "$deltas"; then
- sed -i.backup "/$filename/d" "$deltas" && rm -f "$deltas.backup"
msg2 "$(gettext "Removing existing entry '%s'...")" "$filename"
+ sed -i.backup "/$filename/d" "$deltas" && rm -f "$deltas.backup"
+ cp "$deltas" "$files_deltas"
# empty deltas file contains only "%DELTAS%"
if (( $(wc -l < "$deltas") == 1 )); then
msg2 "$(gettext "Removing empty deltas file ...")"
- rm "$deltas"
+ rm "$deltas" "$files_deltas"
fi
return 0
fi
@@ -256,11 +258,11 @@ verify_repo_extension() {
local repofile=$1
case $repofile in
- *.@(db|files).tar.gz) TAR_OPT="z" ;;
- *.@(db|files).tar.bz2) TAR_OPT="j" ;;
- *.@(db|files).tar.xz) TAR_OPT="J" ;;
- *.@(db|files).tar.Z) TAR_OPT="Z" ;;
- *.@(db|files).tar) TAR_OPT="" ;;
+ *.db.tar.gz) TAR_OPT="z" ;;
+ *.db.tar.bz2) TAR_OPT="j" ;;
+ *.db.tar.xz) TAR_OPT="J" ;;
+ *.db.tar.Z) TAR_OPT="Z" ;;
+ *.db.tar) TAR_OPT="" ;;
*) error "$(gettext "'%s' does not have a valid database archive extension.")" \
"$repofile"
exit 1 ;;
@@ -306,7 +308,7 @@ db_write_entry() {
return 1
fi
- if [[ -d $tmpdir/tree/$pkgname-$pkgver ]]; then
+ if [[ -d $tmpdir/repo/$pkgname-$pkgver ]]; then
warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver"
if (( ONLYADDNEW )); then
return 0;
@@ -353,7 +355,7 @@ db_write_entry() {
db_remove_entry "$pkgname"
# create package directory
- pushd "$tmpdir/tree" >/dev/null
+ pushd "$tmpdir/repo" >/dev/null
mkdir "$pkgname-$pkgver"
pushd "$pkgname-$pkgver" >/dev/null
@@ -402,12 +404,11 @@ db_write_entry() {
popd >/dev/null
# create files file if wanted
- if (( WITHFILES )); then
- msg2 "$(gettext "Creating '%s' db entry...")" 'files'
- local files_path="$tmpdir/tree/$pkgname-$pkgver/files"
- echo "%FILES%" >"$files_path"
- bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_path"
- fi
+ msg2 "$(gettext "Creating '%s' db entry...")" 'files'
+ local files_dir="$tmpdir/files/$pkgname-$pkgver"
+ mkdir "${files_dir}"
+ echo "%FILES%" >"$files_dir/files"
+ bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_dir/files"
# create a delta file
if (( DELTA )); then
@@ -423,6 +424,10 @@ db_write_entry() {
fi
fi
+ # sync files db with main db
+ rm -f $tmpdir/files/$pkgname-$pkgver/{desc,depends,deltas}
+ cp $tmpdir/repo/$pkgname-$pkgver/* $tmpdir/files/$pkgname-$pkgver/
+
if (( RMEXISTING )); then
msg2 "$(gettext "Removing old package file '%s'")" "$oldfilename"
rm -f ${oldfile} ${oldfile}.sig
@@ -437,14 +442,16 @@ db_remove_entry() {
local pkgname=$1
local notfound=1
local pkgentry=$(find_pkgentry "$pkgname")
+
while [[ -n $pkgentry ]]; do
notfound=0
+ local files_pkgentry=$(echo "$pkgentry" | sed 's/\(.*\)\/repo\//\1\/files\//')
if [[ -f $pkgentry/deltas ]]; then
- mv "$pkgentry/deltas" "$tmpdir/tree/$pkgname.deltas"
+ mv "$pkgentry/deltas" "$tmpdir/repo/$pkgname.deltas"
fi
msg2 "$(gettext "Removing existing entry '%s'...")" \
"${pkgentry##*/}"
- rm -rf "$pkgentry"
+ rm -rf "$pkgentry" "$files_pkgentry"
pkgentry=$(find_pkgentry "$pkgname")
done
return $notfound
@@ -465,57 +472,63 @@ elephant() {
esac | openssl base64 -d | gzip -d
}
-check_repo_db() {
- local repodir
-
- # ensure the path to the DB exists; $LOCKFILE is always an absolute path
- repodir=${LOCKFILE%/*}/
-
- if [[ ! -d $repodir ]]; then
- error "$(gettext "%s does not exist or is not a directory.")" "$repodir"
- exit 1
- fi
-
- # check lock file
- if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then
- CLEAN_LOCK=1
- else
- error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE"
- [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat "$LOCKFILE")"
- exit 1
- fi
+validate_repo() {
+ local type=$1 repo=$2
- if [[ -f $REPO_DB_FILE ]]; then
+ if [[ -f $repo ]]; then
# there are two situations we can have here- a DB with some entries,
# or a DB with no contents at all.
- if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then
+ if ! bsdtar -tqf "$repo" '*/desc' >/dev/null 2>&1; then
# check empty case
- if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then
+ if [[ -n $(bsdtar -tqf "$repo" '*' 2>/dev/null) ]]; then
error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE"
exit 1
fi
fi
- verify_signature "$REPO_DB_FILE"
- msg "$(gettext "Extracting database to a temporary location...")"
- bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir/tree"
+ verify_signature "$repo"
+ bsdtar -xf "$repo" -C "$tmpdir/$type"
else
case $cmd in
repo-remove)
- error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE"
+ error "$(gettext "Repository file '%s' was not found.")" "$repo"
exit 1
;;
repo-add)
# check if the file can be created (write permission, directory existence, etc)
- if ! touch "$REPO_DB_FILE"; then
- error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE"
+ if ! touch "$repo"; then
+ error "$(gettext "Repository file '%s' could not be created.")" "$repo"
exit 1
fi
- rm -f "$REPO_DB_FILE"
+ rm -f "$repo"
;;
esac
fi
}
+check_repo_dbs() {
+ local repodir
+
+ # ensure the path to the DB exists; $LOCKFILE is always an absolute path
+ repodir=${LOCKFILE%/*}/
+
+ if [[ ! -d $repodir ]]; then
+ error "$(gettext "%s does not exist or is not a directory.")" "$repodir"
+ exit 1
+ fi
+
+ # check lock file
+ if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then
+ CLEAN_LOCK=1
+ else
+ error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE"
+ [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat "$LOCKFILE")"
+ exit 1
+ fi
+
+ validate_repo repo $REPO_DB_FILE
+ validate_repo files $FILES_DB_FILE
+}
+
add() {
if [[ ! -f $1 ]]; then
error "$(gettext "File '%s' not found.")" "$1"
@@ -563,7 +576,7 @@ remove() {
msg "$(gettext "Searching for package '%s'...")" "$pkgname"
if db_remove_entry "$pkgname"; then
- rm -f "$tmpdir/tree/$pkgname.deltas"
+ rm -f "$tmpdir/repo/$pkgname.deltas"
return 0
else
error "$(gettext "Package matching '%s' not found.")" "$pkgname"
@@ -571,40 +584,64 @@ remove() {
fi
}
-rotate_db() {
- filename=${REPO_DB_FILE##*/}
- tempname=$dirname/.tmp.$filename
+create_db() {
+ local type=$1 db=$2
- # hardlink or move the previous version of the database and signature to .old
- # extension as a backup measure
- if [[ -f $REPO_DB_FILE ]]; then
- ln -f "$REPO_DB_FILE" "$REPO_DB_FILE.old" 2>/dev/null || \
- mv -f "$REPO_DB_FILE" "$REPO_DB_FILE.old"
+ dirname=${LOCKFILE%/*}
+ filename=${db##*/}
+ # this ensures we create it on the same filesystem, making moves atomic
+ tempname=$dirname/.tmp.$filename
- if [[ -f $REPO_DB_FILE.sig ]]; then
- ln -f "$REPO_DB_FILE.sig" "$REPO_DB_FILE.old.sig" 2>/dev/null || \
- mv -f "$REPO_DB_FILE.sig" "$REPO_DB_FILE.old.sig"
+ pushd "$tmpdir/$type" >/dev/null
+ if ( shopt -s nullglob; files=(*); (( ${#files[*]} )) ); then
+ bsdtar -c${TAR_OPT}f "$tempname" *
else
- rm -f "$REPO_DB_FILE.old.sig"
+ # we have no packages remaining? zip up some emptyness
+ warning "$(gettext "No packages remain, creating empty database.")"
+ bsdtar -c${TAR_OPT}f "$tempname" -T /dev/null
fi
- fi
+ popd >/dev/null
- # rotate the newly-created database and signature into place
- mv "$tempname" "$REPO_DB_FILE"
- if [[ -f $tempname.sig ]]; then
- mv "$tempname.sig" "$REPO_DB_FILE.sig"
- fi
+ create_signature "$tempname"
+}
- dblink=${REPO_DB_FILE%.tar*}
- rm -f "$dblink" "$dblink.sig"
- ln -s "$filename" "$dblink" 2>/dev/null || \
- ln "$filename" "$dblink" 2>/dev/null || \
- cp "$REPO_DB_FILE" "$dblink"
- if [[ -f "$REPO_DB_FILE.sig" ]]; then
- ln -s "$filename.sig" "$dblink.sig" 2>/dev/null || \
- ln "$filename.sig" "$dblink.sig" 2>/dev/null || \
- cp "$REPO_DB_FILE.sig" "$dblink.sig"
- fi
+rotate_dbs() {
+ for db in "$REPO_DB_FILE" "$FILES_DB_FILE"; do
+
+ filename=${db##*/}
+ tempname=$dirname/.tmp.$filename
+
+ # hardlink or move the previous version of the database and signature to .old
+ # extension as a backup measure
+ if [[ -f $db ]]; then
+ ln -f "$db" "$db.old" 2>/dev/null || \
+ mv -f "$db" "$db.old"
+
+ if [[ -f $db.sig ]]; then
+ ln -f "$db.sig" "$db.old.sig" 2>/dev/null || \
+ mv -f "$db.sig" "$db.old.sig"
+ else
+ rm -f "$db.old.sig"
+ fi
+ fi
+
+ # rotate the newly-created database and signature into place
+ mv "$tempname" "$db"
+ if [[ -f $tempname.sig ]]; then
+ mv "$tempname.sig" "$db.sig"
+ fi
+
+ dblink=${db%.tar*}
+ rm -f "$dblink" "$dblink.sig"
+ ln -s "$filename" "$dblink" 2>/dev/null || \
+ ln "$filename" "$dblink" 2>/dev/null || \
+ cp "$db" "$dblink"
+ if [[ -f "$db.sig" ]]; then
+ ln -s "$filename.sig" "$dblink.sig" 2>/dev/null || \
+ ln "$filename.sig" "$dblink.sig" 2>/dev/null || \
+ cp "$db.sig" "$dblink.sig"
+ fi
+ done
}
trap_exit() {
@@ -657,7 +694,7 @@ fi
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/repo-tools.XXXXXXXXXX") || (\
error "$(gettext "Cannot create temp directory for database building.")"; \
exit 1)
-mkdir "$tmpdir/tree"
+mkdir "$tmpdir/repo" "$tmpdir/files"
trap 'clean_up' EXIT
for signal in TERM HUP QUIT; do
@@ -675,7 +712,6 @@ while (( $# )); do
-d|--delta) DELTA=1;;
-n|--new) ONLYADDNEW=1;;
-R|--remove) RMEXISTING=1;;
- -f|--files) WITHFILES=1;;
--nocolor) USE_COLOR='n';;
-s|--sign)
SIGN=1
@@ -703,6 +739,8 @@ if [[ -z $REPO_DB_FILE ]]; then
exit 1
fi
+FILES_DB_FILE=${REPO_DB_FILE%db.tar*}files.tar${REPO_DB_FILE##*.db.tar}
+
if [[ $REPO_DB_FILE == /* ]]; then
LOCKFILE=$REPO_DB_FILE.lck
else
@@ -710,7 +748,7 @@ else
fi
verify_repo_extension "$REPO_DB_FILE" >/dev/null
-check_repo_db
+check_repo_dbs
if (( SIGN || KEY || VERIFY )); then
check_gpg
@@ -729,24 +767,10 @@ if (( success )); then
TAR_OPT=$(verify_repo_extension "$REPO_DB_FILE")
# $LOCKFILE is already guaranteed to be absolute so this is safe
- dirname=${LOCKFILE%/*}
- filename=${REPO_DB_FILE##*/}
- # this ensures we create it on the same filesystem, making moves atomic
- tempname=$dirname/.tmp.$filename
-
- pushd "$tmpdir/tree" >/dev/null
- if ( shopt -s nullglob; files=(*); (( ${#files[*]} )) ); then
- bsdtar -c${TAR_OPT}f "$tempname" *
- else
- # we have no packages remaining? zip up some emptyness
- warning "$(gettext "No packages remain, creating empty database.")"
- bsdtar -c${TAR_OPT}f "$tempname" -T /dev/null
- fi
- popd >/dev/null
-
- create_signature "$tempname"
- rotate_db
+ create_db repo $REPO_DB_FILE
+ create_db files $FILES_DB_FILE
+ rotate_dbs
else
msg "$(gettext "No packages modified, nothing to do.")"
exit 1
--
2.2.2
More information about the pacman-dev
mailing list