[pacman-dev] [PATCH 2/2] Allow repo-add to create delta files.
Nathan Jones
nathanj at insightbb.com
Fri Feb 15 20:38:31 EST 2008
This adds a -d|--delta flag to repo-add which will create delta files
for the packages being added to the repository. Deltas will only be
created for the package with the latest version, so that `repo-add db
*.pkg.tar.gz' will not generate a delta for older packages.
The delta creation code was refactored out of makepkg into a new file
create-delta. This new file is sourced into both makepkg and repo-add.
Signed-off-by: Nathan Jones <nathanj at insightbb.com>
---
doc/repo-add.8.txt | 10 +++-
scripts/.gitignore | 1 +
scripts/Makefile.am | 6 ++
scripts/create-delta.sh.in | 141 ++++++++++++++++++++++++++++++++++++++++++++
scripts/makepkg.sh.in | 59 ++----------------
scripts/repo-add.sh.in | 54 ++++++++++++++++-
6 files changed, 213 insertions(+), 58 deletions(-)
create mode 100644 scripts/create-delta.sh.in
diff --git a/doc/repo-add.8.txt b/doc/repo-add.8.txt
index 80faef4..283aa2c 100644
--- a/doc/repo-add.8.txt
+++ b/doc/repo-add.8.txt
@@ -16,7 +16,7 @@ repo-add - package database maintenance utility
Synopsis
--------
-repo-add <path-to-db> <package> ...
+repo-add [options] <path-to-db> <package> ...
repo-remove <path-to-db> <packagename> ...
@@ -34,6 +34,14 @@ specified on the command line. Multiple packages to remove can be specified
on the command line.
+Options
+-------
+*-d, --delta* (repo-add only)::
+ Generate delta files for added packages. Delta files will only be
+ generated for the latest version of the package. A delta file
+ will only be generated if it does not already exist.
+
+
See Also
--------
linkman:makepkg[8], linkman:pacman[8]
diff --git a/scripts/.gitignore b/scripts/.gitignore
index 214c23d..5c0bf5a 100644
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@@ -1,4 +1,5 @@
apply-delta
+create-delta
makepkg
pacman-optimize
rankmirrors
diff --git a/scripts/Makefile.am b/scripts/Makefile.am
index ddcbd7f..e6194b9 100644
--- a/scripts/Makefile.am
+++ b/scripts/Makefile.am
@@ -1,8 +1,11 @@
# enforce that all scripts have a --help and --version option
AUTOMAKE_OPTIONS = std-options
+cachedir = ${localstatedir}/cache/pacman/pkg/
+
bin_SCRIPTS = \
apply-delta \
+ create-delta \
makepkg \
pacman-optimize \
rankmirrors \
@@ -11,6 +14,7 @@ bin_SCRIPTS = \
EXTRA_DIST = \
apply-delta.sh.in \
+ create-delta.sh.in \
makepkg.sh.in \
pacman-optimize.sh.in \
rankmirrors.py.in \
@@ -30,6 +34,7 @@ edit = sed \
-e 's|@PACKAGE_BUGREPORT[@]|$(PACKAGE_BUGREPORT)|g' \
-e 's|@PACKAGE_NAME[@]|$(PACKAGE_NAME)|g' \
-e 's|@DBEXT[@]|$(DBEXT)|g' \
+ -e 's|@CACHEDIR[@]|$(cachedir)|g' \
-e 's|@configure_input[@]|Generated from $@.in; do not edit by hand.|g'
## All the scripts depend on Makefile so that they are rebuilt when the
@@ -47,6 +52,7 @@ $(bin_SCRIPTS): Makefile
mv $@.tmp $@
apply-delta: $(srcdir)/apply-delta.sh.in
+create-delta: $(srcdir)/create-delta.sh.in
makepkg: $(srcdir)/makepkg.sh.in
pacman-optimize: $(srcdir)/pacman-optimize.sh.in
rankmirrors: $(srcdir)/rankmirrors.py.in
diff --git a/scripts/create-delta.sh.in b/scripts/create-delta.sh.in
new file mode 100644
index 0000000..38711d3
--- /dev/null
+++ b/scripts/create-delta.sh.in
@@ -0,0 +1,141 @@
+#!/bin/bash -e
+#
+# create-delta - create xdelta patches for packages
+# @configure_input@
+#
+# Copyright (c) 2008 by Judd Vinet <jvinet at zeroflux.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# This file is meant to be sourced into other scripts. The calling script must
+# define the functions msg, msg2, warning, and error.
+
+# create_xdelta_latest - will find the latest package with the given pkgname
+# and create a delta for it.
+#
+# params:
+# $1 - the name of the package
+# $2 - the arch of the package
+# $3 - the directory where the package is located
+# $4 - the extension of packages
+create_xdelta_latest() {
+ local pkgname=$1
+ local arch=$2
+ local pkgdest=$3
+ local pkgext=$4
+ local cache_dir="@CACHEDIR@"
+ local pkginfo="$(mktemp -t xdelta-pkginfo.XXXXXXXXX)"
+
+ local old_file old_version latest_version latest_file prev_version prev_file
+ for old_file in $(ls {"$cache_dir","$pkgdest"}/${pkgname}-*-*$pkgext 2>/dev/null); do
+ bsdtar -xOf "$old_file" .PKGINFO > "$pkginfo" || continue
+ if [ "$(cat "$pkginfo" | grep '^pkgname = ')" != "pkgname = $pkgname" ]; then
+ continue # Package name does not match.
+ elif [ "$(cat "$pkginfo" | grep '^arch = ')" != "arch = $arch" ] ; then
+ continue # Not same arch.
+ fi
+
+ old_version="$(cat "$pkginfo" | grep '^pkgver = ' | sed 's/^pkgver = //')"
+
+ local vercmp=$(vercmp "$old_version" "$latest_version")
+ if [ $vercmp -gt 0 ]; then
+ prev_version=$latest_version
+ prev_file=$latest_file
+ latest_version=$old_version
+ latest_file=$old_file
+ fi
+ done
+
+ rm -f "$pkginfo"
+
+ if [ -n "$latest_file" -a -n "$prev_file" ]; then
+ create_xdelta_file "$latest_file" "$arch" "$latest_version" "$pkgdest" "$pkgext" 0 "$prev_file" "$prev_version"
+ fi
+}
+
+
+# create_xdelta_file - will create a delta for the package filename given.
+#
+# params:
+# $1 - the filename of the package
+# $2 - the arch of the package
+# $3 - the version and release of the package
+# $4 - the directory where the package is located
+# $5 - the extension of packages
+# $6 - 0 if an existing delta file should not be overwritten
+# $7 - the filename of the previous package (blank if not known)
+# $8 - the version of the previous package (blank if not known)
+create_xdelta_file() {
+ if [ ! "$(type -p xdelta)" ]; then
+ error "$(gettext "Cannot find the xdelta binary! Is xdelta installed?")"
+ return
+ fi
+
+ local pkg_file=$1
+ local arch=$2
+ local pkgverrel=$3
+ local pkgdest=$4
+ local pkgext=$5
+ local overwrite=$6
+ local prev_file=$7
+ local prev_version=$8
+ local cache_dir="@CACHEDIR@"
+ local pkginfo="$(mktemp -t xdelta-pkginfo.XXXXXXXXX)"
+
+ if [ -z "$prev_file" -o -z "$prev_version" ]; then
+ local old_file old_version
+ for old_file in $(ls {"$cache_dir","$pkgdest"}/${pkgname}-*-*$pkgext 2>/dev/null); do
+ bsdtar -xOf "$old_file" .PKGINFO > "$pkginfo" || continue
+ if [ "$(cat "$pkginfo" | grep '^pkgname = ')" != "pkgname = $pkgname" ]; then
+ continue # Package name does not match.
+ elif [ "$(cat "$pkginfo" | grep '^arch = ')" != "arch = $arch" ] ; then
+ continue # Not same arch.
+ fi
+
+ old_version="$(cat "$pkginfo" | grep '^pkgver = ' | sed 's/^pkgver = //')"
+
+ # old_version may include the target package, only use the old versions
+ local vercmp=$(vercmp "$old_version" "$prev_version")
+ if [ "$old_version" != "$pkgverrel" -a $vercmp -gt 0 ]; then
+ prev_version=$old_version
+ prev_file=$old_file
+ fi
+ done
+ fi
+
+ rm -f "$pkginfo"
+
+ if [ -n "$prev_file" ]; then
+ local delta_file="$pkgdest/$pkgname-${prev_version}_to_$pkgverrel-$arch.delta"
+ local ret=0
+
+ # return if not overwriting an existing delta
+ if [ $overwrite -eq 0 -a -e "$delta_file" ]; then
+ return
+ fi
+
+ msg "$(gettext "Creating delta from %s to %s...")" "$pkgname-$prev_version" "$pkgname-$pkgverrel"
+ msg2 "$(gettext "NOTE: the delta should ONLY be distributed with this tarball")"
+
+ # xdelta will decompress prev_file & pkg_file into TMPDIR (or /tmp if
+ # TMPDIR is unset) then perform the delta on the resulting tars
+ xdelta delta "$prev_file" "$pkg_file" "$delta_file" || ret=$?
+
+ if [ $ret -ne 0 -a $ret -ne 1 ]; then
+ warning "$(gettext "Delta was not able to be created.")"
+ fi
+ fi
+}
+
+# vim: set ts=2 sw=2 noet:
diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in
index b9b3ed7..ce4d960 100644
--- a/scripts/makepkg.sh.in
+++ b/scripts/makepkg.sh.in
@@ -65,6 +65,9 @@ FORCE_VER=""
PACMAN_OPTS=
+# load the delta generation functions
+source "$(dirname $0)/create-delta"
+
### SUBROUTINES ###
plain() {
@@ -875,58 +878,6 @@ create_package() {
fi
}
-create_xdelta() {
- if [ "$(check_buildenv xdelta)" != "y" ]; then
- return
- elif [ ! "$(type -p xdelta)" ]; then
- error "$(gettext "Cannot find the xdelta binary! Is xdelta installed?")"
- return
- fi
-
- local pkg_file=$1
- local cache_dir="/var/cache/pacman/pkg" # TODO: autoconf me
- local pkginfo="$(mktemp "$startdir"/xdelta-pkginfo.XXXXXXXXX)"
-
- local old_file old_version
- for old_file in $(ls {"$cache_dir","$PKGDEST"}/${pkgname}-*-*{,-$CARCH}$PKGEXT 2>/dev/null); do
- bsdtar -xOf "$old_file" .PKGINFO > "$pkginfo" || continue
- if [ "$(cat "$pkginfo" | grep '^pkgname = ')" != "pkgname = $pkgname" ]; then
- continue # Package name does not match.
- elif [ "$(cat "$pkginfo" | grep '^arch = ')" != "arch = $CARCH" ] ; then
- continue # Not same arch.
- fi
-
- old_version="$(cat "$pkginfo" | grep '^pkgver = ' | sed 's/^pkgver = //')"
-
- # old_version may include the target package, only use the old versions
- local vercmp=$(vercmp "$old_version" "$latest_version")
- if [ "$old_version" != "$pkgver-$pkgrel" -a $vercmp -gt 0 ]; then
- local latest_version=$old_version
- local base_file=$old_file
- fi
- done
-
- rm -f "$pkginfo"
-
- if [ "$base_file" != "" ]; then
- msg "$(gettext "Making delta from version %s...")" "$latest_version"
- msg2 "$(gettext "NOTE: the delta should ONLY be distributed with this tarball")"
-
- local delta_file="$PKGDEST/$pkgname-${latest_version}_to_$pkgver-$pkgrel-$CARCH.delta"
- local ret=0
-
- # xdelta will decompress base_file & pkg_file into TMPDIR (or /tmp if
- # TMPDIR is unset) then perform the delta on the resulting tars
- xdelta delta "$base_file" "$pkg_file" "$delta_file" || ret=$?
-
- if [ $ret -ne 0 -a $ret -ne 1 ]; then
- warning "$(gettext "Delta was not able to be created.")"
- fi
- else
- warning "$(gettext "No previous version found, skipping xdelta.")"
- fi
-}
-
create_srcpackage() {
cd "$startdir"
msg "$(gettext "Creating source package...")"
@@ -1468,7 +1419,9 @@ else
fi
fi
- create_xdelta "$PKGDEST/${pkgname}-${pkgver}-${pkgrel}-${CARCH}${PKGEXT}"
+ if [ "$(check_buildenv xdelta)" = "y" ]; then
+ create_xdelta_file "$PKGDEST/${pkgname}-${pkgver}-${pkgrel}-${CARCH}${PKGEXT}" "$CARCH" "$pkgver-$pkgrel" "$PKGDEST" "$PKGEXT" 1 "" ""
+ fi
fi
msg "$(gettext "Finished making: %s")" "$pkgname ($(date))"
diff --git a/scripts/repo-add.sh.in b/scripts/repo-add.sh.in
index 00eec7e..6d61aa4 100644
--- a/scripts/repo-add.sh.in
+++ b/scripts/repo-add.sh.in
@@ -27,6 +27,9 @@ confdir='@sysconfdir@'
REPO_DB_FILE=""
+# load the delta generation functions
+source "$(dirname $0)/create-delta"
+
# ensure we have a sane umask set
umask 0022
@@ -53,10 +56,12 @@ error() {
# print usage instructions
usage() {
printf "repo-add (pacman) %s\n\n" "$myver"
- printf "$(gettext "Usage: %s <path-to-db> <package> ...\n\n")" "$0"
+ printf "$(gettext "Usage: %s [options] <path-to-db> <package> ...\n\n")" "$0"
printf "$(gettext "\
repo-add will update a package database by reading a package file.\n\
Multiple packages to add can be specified on the command line.\n\n")"
+ printf "$(gettext "Options:\n")"
+ printf "$(gettext " -d, --delta Generate delta files for packages\n\n")"
echo "$(gettext "Example: repo-add /path/to/repo.db.tar.gz pacman-3.0.0.pkg.tar.gz")"
}
@@ -123,6 +128,38 @@ db_write_delta()
echo -e "$fromver $tover $csize $filename $md5sum" >>deltas
} # end db_write_delta
+# create a delta for the latest package
+# arg1 - path to package
+create_xdelta() {
+ # blank out all variables and set pkgfile
+ local pkgfile=$(readlink -f "$1")
+ local pkgname pkgver pkgdesc url builddate packager csize size \
+ group depend backup arch license replaces provides conflict force \
+ _groups _depends _backups _licenses _replaces _provides _conflicts \
+ pkgdir
+
+ local OLDIFS="$IFS"
+ # IFS (field separator) is only the newline character
+ IFS="
+"
+
+ # read info from the zipped package
+ local line
+ for line in $(bsdtar -xOf "$pkgfile" .PKGINFO | \
+ grep -v "^#" | sed 's|\(\w*\)\s*=\s*\(.*\)|\1="\2"|'); do
+ eval "$line"
+ done
+
+ IFS=$OLDIFS
+
+ # ensure $pkgname and $pkgver variables were found
+ if [ -z "$pkgname" -o -z "$pkgver" ]; then
+ return 1
+ fi
+
+ pkgdir=$(pwd)
+ create_xdelta_latest "$pkgname" "$arch" "$pkgdir" "$PKGEXT"
+}
# write an entry to the pacman database
# arg1 - path to package
@@ -131,7 +168,7 @@ db_write_entry()
# blank out all variables and set pkgfile
local pkgfile=$(readlink -f "$1")
local pkgname pkgver pkgdesc url builddate packager csize size \
- group depend backup license replaces provides conflict force \
+ group depend backup arch license replaces provides conflict force \
_groups _depends _backups _licenses _replaces _provides _conflicts \
startdir
@@ -229,9 +266,9 @@ db_write_entry()
# write this delta entry
if db_write_delta "$delta"; then
- msg2 "$(gettext "Added delta '%s'")" "$(basename "$delta")"
+ msg2 "$(gettext "Adding delta '%s'...")" "$(basename "$delta")"
else
- msg2 "$(gettext "Could not add delta '%s'")" "$(basename "$delta")"
+ msg2 "$(gettext "Could not add delta '%s'.")" "$(basename "$delta")"
fi
fi
done
@@ -283,11 +320,18 @@ gstmpdir=$(mktemp -d /tmp/repo-add.XXXXXXXXXX) || (\
exit 1)
success=0
+optdelta=""
# parse arguments
for arg in "$@"; do
if [ "$arg" == "--force" -o "$arg" == "-f" ]; then
warning "$(gettext "the -f and --force options are no longer recognized")"
msg2 "$(gettext "use options=(force) in the PKGBUILD instead")"
+ elif [ "$arg" == "--delta" -o "$arg" == "-d" ]; then
+ if [ ! "$(type -p xdelta)" ]; then
+ error "$(gettext "Cannot find the xdelta binary! Is xdelta installed?")"
+ exit 1
+ fi
+ optdelta="y"
elif [ -z "$REPO_DB_FILE" ]; then
REPO_DB_FILE=$(readlink -f "$arg")
if ! test_repo_db_file; then
@@ -302,6 +346,8 @@ for arg in "$@"; do
if ! bsdtar -tf "$arg" .PKGINFO 2>&1 >/dev/null; then
error "$(gettext "'%s' is not a package file, skipping")" "$arg"
else
+ [ -n "$optdelta" ] && create_xdelta "$arg"
+
msg "$(gettext "Adding package '%s'")" "$arg"
if db_write_entry "$arg"; then
--
1.5.4
More information about the pacman-dev
mailing list