Signed-off-by: Allan McRae <allan@archlinux.org> --- v2 - fix updating repos scripts/repo-add.sh.in | 220 +++++++++++++++++++++++++++---------------------- 1 file changed, 122 insertions(+), 98 deletions(-) diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in index 21cb2aa..8601fbb 100644 --- a/scripts/repo-add.sh.in +++ b/scripts/repo-add.sh.in @@ -32,7 +32,6 @@ QUIET=0 DELTA=0 ONLYADDNEW=0 RMEXISTING=0 -WITHFILES=0 SIGN=0 KEY=0 VERIFY=0 @@ -61,7 +60,6 @@ Multiple packages to add can be specified on the command line.\n")" printf -- "$(gettext " -d, --delta generate and add delta for package update\n")" printf -- "$(gettext " -n, --new only add packages that are not already in the database\n")" printf -- "$(gettext " -R, --remove remove old package file from disk after updating database\n")" - printf -- "$(gettext " -f, --files update database's file list\n")" elif [[ $cmd == "repo-remove" ]] ; then printf -- "$(gettext "Usage: repo-remove [options] <path-to-db> <packagename|delta> ...\n")" printf -- "\n" @@ -115,7 +113,7 @@ format_entry() { find_pkgentry() { local pkgname=$1 local pkgentry - for pkgentry in "$tmpdir/tree/$pkgname"*; do + for pkgentry in "$tmpdir/repo/$pkgname"*; do name=${pkgentry##*/} if [[ ${name%-*-*} = $pkgname ]]; then echo $pkgentry @@ -145,6 +143,7 @@ db_write_delta() { return 1 fi deltas=$pkgentry/deltas + files_deltas=$(echo "$deltas" | sed 's/\(.*\)\/repo\//\1\/files\//') if [[ ! -f $deltas ]]; then echo -e "%DELTAS%" >"$deltas" fi @@ -161,6 +160,7 @@ db_write_delta() { fi msg2 "$(gettext "Adding 'deltas' entry : %s -> %s")" "$oldfile" "$newfile" echo "${deltafile##*/} $md5sum $csize $oldfile $newfile" >> "$deltas" + cp "$deltas" "$files_deltas" return 0 } # end db_write_delta @@ -177,16 +177,18 @@ db_remove_delta() { return 1 fi deltas=$pkgentry/deltas + files_deltas=$(echo "${deltas}" | sed 's/\(.*\)\/repo\//\1\/files\//') if [[ ! -f $deltas ]]; then return 1 fi if grep -q "$filename" "$deltas"; then - sed -i.backup "/$filename/d" "$deltas" && rm -f "$deltas.backup" msg2 "$(gettext "Removing existing entry '%s'...")" "$filename" + sed -i.backup "/$filename/d" "$deltas" && rm -f "$deltas.backup" + cp "$deltas" "$files_deltas" # empty deltas file contains only "%DELTAS%" if (( $(wc -l < "$deltas") == 1 )); then msg2 "$(gettext "Removing empty deltas file ...")" - rm "$deltas" + rm "$deltas" "$files_deltas" fi return 0 fi @@ -256,11 +258,11 @@ verify_repo_extension() { local repofile=$1 case $repofile in - *.@(db|files).tar.gz) TAR_OPT="z" ;; - *.@(db|files).tar.bz2) TAR_OPT="j" ;; - *.@(db|files).tar.xz) TAR_OPT="J" ;; - *.@(db|files).tar.Z) TAR_OPT="Z" ;; - *.@(db|files).tar) TAR_OPT="" ;; + *.db.tar.gz) TAR_OPT="z" ;; + *.db.tar.bz2) TAR_OPT="j" ;; + *.db.tar.xz) TAR_OPT="J" ;; + *.db.tar.Z) TAR_OPT="Z" ;; + *.db.tar) TAR_OPT="" ;; *) error "$(gettext "'%s' does not have a valid database archive extension.")" \ "$repofile" exit 1 ;; @@ -306,7 +308,7 @@ db_write_entry() { return 1 fi - if [[ -d $tmpdir/tree/$pkgname-$pkgver ]]; then + if [[ -d $tmpdir/repo/$pkgname-$pkgver ]]; then warning "$(gettext "An entry for '%s' already existed")" "$pkgname-$pkgver" if (( ONLYADDNEW )); then return 0; @@ -353,7 +355,7 @@ db_write_entry() { db_remove_entry "$pkgname" # create package directory - pushd "$tmpdir/tree" >/dev/null + pushd "$tmpdir/repo" >/dev/null mkdir "$pkgname-$pkgver" pushd "$pkgname-$pkgver" >/dev/null @@ -402,12 +404,11 @@ db_write_entry() { popd >/dev/null # create files file if wanted - if (( WITHFILES )); then - msg2 "$(gettext "Creating '%s' db entry...")" 'files' - local files_path="$tmpdir/tree/$pkgname-$pkgver/files" - echo "%FILES%" >"$files_path" - bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_path" - fi + msg2 "$(gettext "Creating '%s' db entry...")" 'files' + local files_dir="$tmpdir/files/$pkgname-$pkgver" + mkdir "${files_dir}" + echo "%FILES%" >"$files_dir/files" + bsdtar --exclude='^.*' -tf "$pkgfile" >>"$files_dir/files" # create a delta file if (( DELTA )); then @@ -423,6 +424,10 @@ db_write_entry() { fi fi + # sync files db with main db + rm -f $tmpdir/files/$pkgname-$pkgver/{desc,depends,deltas} + cp $tmpdir/repo/$pkgname-$pkgver/* $tmpdir/files/$pkgname-$pkgver/ + if (( RMEXISTING )); then msg2 "$(gettext "Removing old package file '%s'")" "$oldfilename" rm -f ${oldfile} ${oldfile}.sig @@ -437,14 +442,16 @@ db_remove_entry() { local pkgname=$1 local notfound=1 local pkgentry=$(find_pkgentry "$pkgname") + while [[ -n $pkgentry ]]; do notfound=0 + local files_pkgentry=$(echo "$pkgentry" | sed 's/\(.*\)\/repo\//\1\/files\//') if [[ -f $pkgentry/deltas ]]; then - mv "$pkgentry/deltas" "$tmpdir/tree/$pkgname.deltas" + mv "$pkgentry/deltas" "$tmpdir/repo/$pkgname.deltas" fi msg2 "$(gettext "Removing existing entry '%s'...")" \ "${pkgentry##*/}" - rm -rf "$pkgentry" + rm -rf "$pkgentry" "$files_pkgentry" pkgentry=$(find_pkgentry "$pkgname") done return $notfound @@ -465,57 +472,63 @@ elephant() { esac | openssl base64 -d | gzip -d } -check_repo_db() { - local repodir - - # ensure the path to the DB exists; $LOCKFILE is always an absolute path - repodir=${LOCKFILE%/*}/ - - if [[ ! -d $repodir ]]; then - error "$(gettext "%s does not exist or is not a directory.")" "$repodir" - exit 1 - fi - - # check lock file - if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then - CLEAN_LOCK=1 - else - error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" - [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat "$LOCKFILE")" - exit 1 - fi +validate_repo() { + local type=$1 repo=$2 - if [[ -f $REPO_DB_FILE ]]; then + if [[ -f $repo ]]; then # there are two situations we can have here- a DB with some entries, # or a DB with no contents at all. - if ! bsdtar -tqf "$REPO_DB_FILE" '*/desc' >/dev/null 2>&1; then + if ! bsdtar -tqf "$repo" '*/desc' >/dev/null 2>&1; then # check empty case - if [[ -n $(bsdtar -tqf "$REPO_DB_FILE" '*' 2>/dev/null) ]]; then + if [[ -n $(bsdtar -tqf "$repo" '*' 2>/dev/null) ]]; then error "$(gettext "Repository file '%s' is not a proper pacman database.")" "$REPO_DB_FILE" exit 1 fi fi - verify_signature "$REPO_DB_FILE" - msg "$(gettext "Extracting database to a temporary location...")" - bsdtar -xf "$REPO_DB_FILE" -C "$tmpdir/tree" + verify_signature "$repo" + bsdtar -xf "$repo" -C "$tmpdir/$type" else case $cmd in repo-remove) - error "$(gettext "Repository file '%s' was not found.")" "$REPO_DB_FILE" + error "$(gettext "Repository file '%s' was not found.")" "$repo" exit 1 ;; repo-add) # check if the file can be created (write permission, directory existence, etc) - if ! touch "$REPO_DB_FILE"; then - error "$(gettext "Repository file '%s' could not be created.")" "$REPO_DB_FILE" + if ! touch "$repo"; then + error "$(gettext "Repository file '%s' could not be created.")" "$repo" exit 1 fi - rm -f "$REPO_DB_FILE" + rm -f "$repo" ;; esac fi } +check_repo_dbs() { + local repodir + + # ensure the path to the DB exists; $LOCKFILE is always an absolute path + repodir=${LOCKFILE%/*}/ + + if [[ ! -d $repodir ]]; then + error "$(gettext "%s does not exist or is not a directory.")" "$repodir" + exit 1 + fi + + # check lock file + if ( set -o noclobber; echo "$$" > "$LOCKFILE") 2> /dev/null; then + CLEAN_LOCK=1 + else + error "$(gettext "Failed to acquire lockfile: %s.")" "$LOCKFILE" + [[ -f $LOCKFILE ]] && error "$(gettext "Held by process %s")" "$(cat "$LOCKFILE")" + exit 1 + fi + + validate_repo repo $REPO_DB_FILE + validate_repo files $FILES_DB_FILE +} + add() { if [[ ! -f $1 ]]; then error "$(gettext "File '%s' not found.")" "$1" @@ -563,7 +576,7 @@ remove() { msg "$(gettext "Searching for package '%s'...")" "$pkgname" if db_remove_entry "$pkgname"; then - rm -f "$tmpdir/tree/$pkgname.deltas" + rm -f "$tmpdir/repo/$pkgname.deltas" return 0 else error "$(gettext "Package matching '%s' not found.")" "$pkgname" @@ -571,40 +584,64 @@ remove() { fi } -rotate_db() { - filename=${REPO_DB_FILE##*/} - tempname=$dirname/.tmp.$filename +create_db() { + local type=$1 db=$2 - # hardlink or move the previous version of the database and signature to .old - # extension as a backup measure - if [[ -f $REPO_DB_FILE ]]; then - ln -f "$REPO_DB_FILE" "$REPO_DB_FILE.old" 2>/dev/null || \ - mv -f "$REPO_DB_FILE" "$REPO_DB_FILE.old" + dirname=${LOCKFILE%/*} + filename=${db##*/} + # this ensures we create it on the same filesystem, making moves atomic + tempname=$dirname/.tmp.$filename - if [[ -f $REPO_DB_FILE.sig ]]; then - ln -f "$REPO_DB_FILE.sig" "$REPO_DB_FILE.old.sig" 2>/dev/null || \ - mv -f "$REPO_DB_FILE.sig" "$REPO_DB_FILE.old.sig" + pushd "$tmpdir/$type" >/dev/null + if ( shopt -s nullglob; files=(*); (( ${#files[*]} )) ); then + bsdtar -c${TAR_OPT}f "$tempname" * else - rm -f "$REPO_DB_FILE.old.sig" + # we have no packages remaining? zip up some emptyness + warning "$(gettext "No packages remain, creating empty database.")" + bsdtar -c${TAR_OPT}f "$tempname" -T /dev/null fi - fi + popd >/dev/null - # rotate the newly-created database and signature into place - mv "$tempname" "$REPO_DB_FILE" - if [[ -f $tempname.sig ]]; then - mv "$tempname.sig" "$REPO_DB_FILE.sig" - fi + create_signature "$tempname" +} - dblink=${REPO_DB_FILE%.tar*} - rm -f "$dblink" "$dblink.sig" - ln -s "$filename" "$dblink" 2>/dev/null || \ - ln "$filename" "$dblink" 2>/dev/null || \ - cp "$REPO_DB_FILE" "$dblink" - if [[ -f "$REPO_DB_FILE.sig" ]]; then - ln -s "$filename.sig" "$dblink.sig" 2>/dev/null || \ - ln "$filename.sig" "$dblink.sig" 2>/dev/null || \ - cp "$REPO_DB_FILE.sig" "$dblink.sig" - fi +rotate_dbs() { + for db in "$REPO_DB_FILE" "$FILES_DB_FILE"; do + + filename=${db##*/} + tempname=$dirname/.tmp.$filename + + # hardlink or move the previous version of the database and signature to .old + # extension as a backup measure + if [[ -f $db ]]; then + ln -f "$db" "$db.old" 2>/dev/null || \ + mv -f "$db" "$db.old" + + if [[ -f $db.sig ]]; then + ln -f "$db.sig" "$db.old.sig" 2>/dev/null || \ + mv -f "$db.sig" "$db.old.sig" + else + rm -f "$db.old.sig" + fi + fi + + # rotate the newly-created database and signature into place + mv "$tempname" "$db" + if [[ -f $tempname.sig ]]; then + mv "$tempname.sig" "$db.sig" + fi + + dblink=${db%.tar*} + rm -f "$dblink" "$dblink.sig" + ln -s "$filename" "$dblink" 2>/dev/null || \ + ln "$filename" "$dblink" 2>/dev/null || \ + cp "$db" "$dblink" + if [[ -f "$db.sig" ]]; then + ln -s "$filename.sig" "$dblink.sig" 2>/dev/null || \ + ln "$filename.sig" "$dblink.sig" 2>/dev/null || \ + cp "$db.sig" "$dblink.sig" + fi + done } trap_exit() { @@ -657,7 +694,7 @@ fi tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/repo-tools.XXXXXXXXXX") || (\ error "$(gettext "Cannot create temp directory for database building.")"; \ exit 1) -mkdir "$tmpdir/tree" +mkdir "$tmpdir/repo" "$tmpdir/files" trap 'clean_up' EXIT for signal in TERM HUP QUIT; do @@ -675,7 +712,6 @@ while (( $# )); do -d|--delta) DELTA=1;; -n|--new) ONLYADDNEW=1;; -R|--remove) RMEXISTING=1;; - -f|--files) WITHFILES=1;; --nocolor) USE_COLOR='n';; -s|--sign) SIGN=1 @@ -703,6 +739,8 @@ if [[ -z $REPO_DB_FILE ]]; then exit 1 fi +FILES_DB_FILE=${REPO_DB_FILE%db.tar*}files.tar${REPO_DB_FILE##*.db.tar} + if [[ $REPO_DB_FILE == /* ]]; then LOCKFILE=$REPO_DB_FILE.lck else @@ -710,7 +748,7 @@ else fi verify_repo_extension "$REPO_DB_FILE" >/dev/null -check_repo_db +check_repo_dbs if (( SIGN || KEY || VERIFY )); then check_gpg @@ -729,24 +767,10 @@ if (( success )); then TAR_OPT=$(verify_repo_extension "$REPO_DB_FILE") # $LOCKFILE is already guaranteed to be absolute so this is safe - dirname=${LOCKFILE%/*} - filename=${REPO_DB_FILE##*/} - # this ensures we create it on the same filesystem, making moves atomic - tempname=$dirname/.tmp.$filename - - pushd "$tmpdir/tree" >/dev/null - if ( shopt -s nullglob; files=(*); (( ${#files[*]} )) ); then - bsdtar -c${TAR_OPT}f "$tempname" * - else - # we have no packages remaining? zip up some emptyness - warning "$(gettext "No packages remain, creating empty database.")" - bsdtar -c${TAR_OPT}f "$tempname" -T /dev/null - fi - popd >/dev/null - - create_signature "$tempname" - rotate_db + create_db repo $REPO_DB_FILE + create_db files $FILES_DB_FILE + rotate_dbs else msg "$(gettext "No packages modified, nothing to do.")" exit 1 -- 2.2.2