[pacman-dev] [PATCH 4/5] Download delta files if UseDelta is set.
Nathan Jones
nathanj at insightbb.com
Fri Oct 19 13:17:53 EDT 2007
Delta files will be used if the size is smaller than a percent
(MAX_DELTA_RATIO) of the package size.
Signed-off-by: Nathan Jones <nathanj at insightbb.com>
---
doc/pacman.conf.5.txt | 4 +
lib/libalpm/alpm.h | 11 ++
lib/libalpm/error.c | 5 +
lib/libalpm/handle.c | 6 +
lib/libalpm/handle.h | 1 +
lib/libalpm/sync.c | 248 ++++++++++++++++++++++++++++++++++++++++++++++++-
lib/libalpm/util.h | 3 +
src/pacman/callback.c | 23 +++++-
src/pacman/pacman.c | 3 +
9 files changed, 300 insertions(+), 4 deletions(-)
diff --git a/doc/pacman.conf.5.txt b/doc/pacman.conf.5.txt
index d422a4b..06ba81d 100644
--- a/doc/pacman.conf.5.txt
+++ b/doc/pacman.conf.5.txt
@@ -105,6 +105,10 @@ Options
*ShowSize*::
Display the size of individual packages for '\--sync' and '\--query' modes.
+*UseDelta*::
+ Download delta files instead of complete packages if possible. Requires
+ the xdelta program to be installed.
+
Repository Sections
-------------------
diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h
index 5f37d82..a4eaafa 100644
--- a/lib/libalpm/alpm.h
+++ b/lib/libalpm/alpm.h
@@ -138,6 +138,7 @@ void alpm_option_set_xfercommand(const char *cmd);
unsigned short alpm_option_get_nopassiveftp();
void alpm_option_set_nopassiveftp(unsigned short nopasv);
+void alpm_option_set_usedelta(unsigned short usedelta);
pmdb_t *alpm_option_get_localdb();
alpm_list_t *alpm_option_get_syncdbs();
@@ -294,6 +295,13 @@ typedef enum _pmtransevt_t {
PM_TRANS_EVT_EXTRACT_DONE,
PM_TRANS_EVT_INTEGRITY_START,
PM_TRANS_EVT_INTEGRITY_DONE,
+ PM_TRANS_EVT_DELTA_INTEGRITY_START,
+ PM_TRANS_EVT_DELTA_INTEGRITY_DONE,
+ PM_TRANS_EVT_DELTA_PATCHES_START,
+ PM_TRANS_EVT_DELTA_PATCHES_DONE,
+ PM_TRANS_EVT_DELTA_PATCH_START,
+ PM_TRANS_EVT_DELTA_PATCH_DONE,
+ PM_TRANS_EVT_DELTA_PATCH_FAILED,
PM_TRANS_EVT_PRINTURI,
PM_TRANS_EVT_RETRIEVE_START,
} pmtransevt_t;
@@ -442,6 +450,9 @@ enum _pmerrno_t {
PM_ERR_PKG_INVALID_NAME,
PM_ERR_PKG_CORRUPTED,
PM_ERR_PKG_REPO_NOT_FOUND,
+ /* Deltas */
+ PM_ERR_DLT_CORRUPTED,
+ PM_ERR_DLT_PATCHFAILED,
/* Groups */
PM_ERR_GRP_NOT_FOUND,
/* Dependencies */
diff --git a/lib/libalpm/error.c b/lib/libalpm/error.c
index f81d22d..44d1c60 100644
--- a/lib/libalpm/error.c
+++ b/lib/libalpm/error.c
@@ -123,6 +123,11 @@ const char SYMEXPORT *alpm_strerror(int err)
return _("corrupted package");
case PM_ERR_PKG_REPO_NOT_FOUND:
return _("no such repository");
+ /* Deltas */
+ case PM_ERR_DLT_CORRUPTED:
+ return _("corrupted delta");
+ case PM_ERR_DLT_PATCHFAILED:
+ return _("delta patch failed");
/* Groups */
case PM_ERR_GRP_NOT_FOUND:
return _("group not found");
diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c
index e8f2147..242bbe5 100644
--- a/lib/libalpm/handle.c
+++ b/lib/libalpm/handle.c
@@ -64,6 +64,7 @@ pmhandle_t *_alpm_handle_new()
handle->cachedirs = NULL;
handle->lockfile = NULL;
handle->logfile = NULL;
+ handle->usedelta = 0;
return(handle);
}
@@ -496,4 +497,9 @@ void SYMEXPORT alpm_option_set_nopassiveftp(unsigned short nopasv)
handle->nopassiveftp = nopasv;
}
+void SYMEXPORT alpm_option_set_usedelta(unsigned short usedelta)
+{
+ handle->usedelta = usedelta;
+}
+
/* vim: set ts=2 sw=2 noet: */
diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h
index cf2e9d5..d8edf00 100644
--- a/lib/libalpm/handle.h
+++ b/lib/libalpm/handle.h
@@ -61,6 +61,7 @@ typedef struct _pmhandle_t {
unsigned short nopassiveftp; /* Don't use PASV ftp connections */
time_t upgradedelay; /* Time to wait before upgrading a package */
char *xfercommand; /* External download command */
+ unsigned short usedelta; /* Download deltas if possible */
} pmhandle_t;
extern pmhandle_t *handle;
diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c
index abc90de..d5f6500 100644
--- a/lib/libalpm/sync.c
+++ b/lib/libalpm/sync.c
@@ -48,6 +48,7 @@
#include "handle.h"
#include "alpm.h"
#include "server.h"
+#include "delta.h"
pmsyncpkg_t *_alpm_sync_new(int type, pmpkg_t *spkg, void *data)
{
@@ -700,6 +701,148 @@ cleanup:
return(ret);
}
+/** Returns a list of deltas that should be downloaded instead of the
+ * package.
+ *
+ * It first tests if a delta path exists between the currently installed
+ * version (if any) and the version to upgrade to. If so, the delta path
+ * is used if its size is below a set percentage (MAX_DELTA_RATIO) of
+ * the package size, Otherwise, an empty list is returned.
+ *
+ * @param newpkg the new package to upgrade to
+ * @param db_local the local database
+ *
+ * @return the list of pmdelta_t * objects. NULL (the empty list) is
+ * returned if the package should be downloaded instead of deltas.
+ */
+static alpm_list_t *pkg_upgrade_delta_path(pmpkg_t *newpkg, pmdb_t *db_local)
+{
+ pmpkg_t *oldpkg = alpm_db_get_pkg(db_local, newpkg->name);
+ alpm_list_t *ret = NULL;
+
+ if(oldpkg) {
+ const char *oldname = alpm_pkg_get_filename(oldpkg);
+ char *oldpath = _alpm_filecache_find(oldname);
+
+ if(oldpath) {
+ alpm_list_t *deltas = _alpm_shortest_delta_path(
+ alpm_pkg_get_deltas(newpkg),
+ alpm_pkg_get_version(oldpkg),
+ alpm_pkg_get_version(newpkg));
+
+ if(deltas) {
+ unsigned long dltsize = _alpm_delta_path_size(deltas);
+ unsigned long pkgsize = alpm_pkg_get_size(newpkg);
+
+ if(dltsize < pkgsize * MAX_DELTA_RATIO) {
+ ret = deltas;
+ } else {
+ ret = NULL;
+ alpm_list_free(deltas);
+ }
+ }
+
+ FREE(oldpath);
+ }
+ }
+
+ return(ret);
+}
+
+/** Applies delta files to create an upgraded package file.
+ *
+ * All intermediate files are deleted, leaving only the starting and
+ * ending package files.
+ *
+ * @param trans the transaction
+ * @param patches A list of alternating pmpkg_t * and pmdelta_t *
+ * objects. The patch command will be built using the pmpkg_t, pmdelta_t
+ * pair.
+ *
+ * @return 0 if all delta files were able to be applied, 1 otherwise.
+ */
+static int apply_deltas(pmtrans_t *trans, alpm_list_t *patches)
+{
+ /* keep track of the previous package in the loop to decide if a
+ * package file should be deleted */
+ pmpkg_t *lastpkg = NULL;
+ int lastpkg_failed = 0;
+ int ret = 0;
+ const char *cachedir = _alpm_filecache_setup();
+
+ alpm_list_t *p = patches;
+ while(p) {
+ pmpkg_t *pkg;
+ pmdelta_t *d;
+ char command[PATH_MAX], fname[PATH_MAX];
+ char pkgfilename[PKG_FILENAME_LEN];
+
+ pkg = alpm_list_getdata(p);
+ p = alpm_list_next(p);
+
+ d = alpm_list_getdata(p);
+ p = alpm_list_next(p);
+
+ /* if patching fails, ignore the rest of that package's deltas */
+ if(lastpkg_failed) {
+ if(pkg == lastpkg) {
+ continue;
+ } else {
+ lastpkg_failed = 0;
+ }
+ }
+
+ /* an example of the patch command: (using /cache for cachedir)
+ * xdelta patch /cache/pacman_3.0.0-1_to_3.0.1-1-i686.delta \
+ * /cache/pacman-3.0.0-1-i686.pkg.tar.gz \
+ * /cache/pacman-3.0.1-1-i686.pkg.tar.gz
+ */
+
+ /* build the patch command */
+ snprintf(command, PATH_MAX,
+ "xdelta patch" /* the command */
+ " %s/%s" /* the delta */
+ " %s/%s-%s-%s" PKGEXT /* the 'from' package */
+ " %s/%s-%s-%s" PKGEXT, /* the 'to' package */
+ cachedir, d->filename,
+ cachedir, pkg->name, d->from, pkg->arch,
+ cachedir, pkg->name, d->to, pkg->arch);
+
+ _alpm_log(PM_LOG_DEBUG, _("command: %s\n"), command);
+
+ snprintf(pkgfilename, PKG_FILENAME_LEN, "%s-%s-%s" PKGEXT,
+ pkg->name, d->to, pkg->arch);
+
+ EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_START, pkgfilename, d->filename);
+
+ if(system(command) == 0) {
+ EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_DONE, NULL, NULL);
+
+ /* delete the delta file */
+ snprintf(fname, PATH_MAX, "%s/%s", cachedir, d->filename);
+ unlink(fname);
+
+ /* Delete the 'from' package but only if it is an intermediate
+ * package. The starting 'from' package should be kept, just
+ * as if deltas were not used. Delete the package file if the
+ * previous iteration of the loop used the same package. */
+ if(pkg == lastpkg) {
+ snprintf(fname, PATH_MAX, "%s/%s-%s-%s" PKGEXT,
+ cachedir, pkg->name, d->from, pkg->arch);
+ unlink(fname);
+ } else {
+ lastpkg = pkg;
+ }
+ } else {
+ EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_FAILED, NULL, NULL);
+ lastpkg_failed = 1;
+ ret = 1;
+ }
+ }
+
+ return(ret);
+}
+
/** Compares the md5sum of a file to the expected value.
*
* If the md5sum does not match, the user is asked whether the file
@@ -762,6 +905,29 @@ static int test_md5sum(pmtrans_t *trans, const char *filename,
return(ret);
}
+/** Compares the md5sum of a delta to the expected value.
+ *
+ * @param trans the transaction
+ * @param delta the delta to test
+ * @param data data to write the error messages to
+ *
+ * @return 0 if the md5sum matched, 1 otherwise
+ */
+static int test_delta_md5sum(pmtrans_t *trans, pmdelta_t *delta,
+ alpm_list_t **data)
+{
+ const char *filename;
+ char *md5sum;
+ int ret = 0;
+
+ filename = alpm_delta_get_filename(delta);
+ md5sum = alpm_delta_get_md5sum(delta);
+
+ ret = test_md5sum(trans, filename, md5sum, data);
+
+ return(ret);
+}
+
/** Compares the md5sum of a package to the expected value.
*
* @param trans the transaction
@@ -787,6 +953,7 @@ static int test_pkg_md5sum(pmtrans_t *trans, pmpkg_t *pkg, alpm_list_t **data)
int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
{
alpm_list_t *i, *j, *files = NULL;
+ alpm_list_t *patches = NULL, *deltas = NULL;
pmtrans_t *tr = NULL;
int replaces = 0, retval = 0;
const char *cachedir = NULL;
@@ -817,8 +984,42 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
} else {
char *fpath = _alpm_filecache_find(fname);
if(!fpath) {
- /* file is not in the cache dir, so add it to the list */
- files = alpm_list_add(files, strdup(fname));
+ if(handle->usedelta) {
+ alpm_list_t *delta_path = pkg_upgrade_delta_path(spkg, db_local);
+
+ if(delta_path) {
+ alpm_list_t *dlts = NULL;
+
+ for(dlts = delta_path; dlts; dlts = alpm_list_next(dlts)) {
+ pmdelta_t *d = (pmdelta_t *)alpm_list_getdata(dlts);
+ char *fpath2 = _alpm_filecache_find(d->filename);
+
+ if(!fpath2) {
+ /* add the delta filename to the download list if
+ * it's not in the cache*/
+ files = alpm_list_add(files, strdup(d->filename));
+ }
+
+ /* save the package and delta so that the xdelta patch
+ * command can be run after the downloads finish */
+ patches = alpm_list_add(patches, spkg);
+ patches = alpm_list_add(patches, d);
+
+ /* keep a list of the delta files for md5sums */
+ deltas = alpm_list_add(deltas, d);
+ }
+
+ alpm_list_free(delta_path);
+ delta_path = NULL;
+ } else {
+ /* no deltas to download, so add the file to the
+ * download list */
+ files = alpm_list_add(files, strdup(fname));
+ }
+ } else {
+ /* not using deltas, so add the file to the download list */
+ files = alpm_list_add(files, strdup(fname));
+ }
}
FREE(fpath);
}
@@ -839,7 +1040,48 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data)
return(0);
}
- /* Check integrity of files */
+ if(handle->usedelta) {
+ int ret = 0;
+
+ /* only output if there are deltas to work with */
+ if(deltas) {
+ /* Check integrity of deltas */
+ EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_START, NULL, NULL);
+
+ for(i = deltas; i; i = i->next) {
+ pmdelta_t *d = alpm_list_getdata(i);
+
+ ret = test_delta_md5sum(trans, d, data);
+
+ if(ret == 1) {
+ retval = 1;
+ } else if(ret == -1) { /* -1 is for serious errors */
+ RET_ERR(pm_errno, -1);
+ }
+ }
+ if(retval) {
+ pm_errno = PM_ERR_DLT_CORRUPTED;
+ goto error;
+ }
+ EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_DONE, NULL, NULL);
+
+ /* Use the deltas to generate the packages */
+ EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_START, NULL, NULL);
+ ret = apply_deltas(trans, patches);
+ EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_DONE, NULL, NULL);
+
+ alpm_list_free(patches);
+ patches = NULL;
+ alpm_list_free(deltas);
+ deltas = NULL;
+ }
+ if(ret) {
+ pm_errno = PM_ERR_DLT_PATCHFAILED;
+ goto error;
+ }
+ }
+
+ /* Check integrity of packages */
EVENT(trans, PM_TRANS_EVT_INTEGRITY_START, NULL, NULL);
for(i = trans->packages; i; i = i->next) {
diff --git a/lib/libalpm/util.h b/lib/libalpm/util.h
index 5ebc70c..17fa123 100644
--- a/lib/libalpm/util.h
+++ b/lib/libalpm/util.h
@@ -67,6 +67,9 @@ char *strsep(char **, const char *);
#define SYMEXPORT __attribute__((visibility("default")))
#define SYMHIDDEN __attribute__((visibility("internal")))
+/* max percent of package size to download deltas */
+#define MAX_DELTA_RATIO 0.7
+
#endif /* _ALPM_UTIL_H */
/* vim: set ts=2 sw=2 noet: */
diff --git a/src/pacman/callback.c b/src/pacman/callback.c
index 6d25713..aec9753 100644
--- a/src/pacman/callback.c
+++ b/src/pacman/callback.c
@@ -236,6 +236,27 @@ void cb_trans_evt(pmtransevt_t event, void *data1, void *data2)
case PM_TRANS_EVT_INTEGRITY_DONE:
printf(_("done.\n"));
break;
+ case PM_TRANS_EVT_DELTA_INTEGRITY_START:
+ printf(_("checking delta integrity... "));
+ break;
+ case PM_TRANS_EVT_DELTA_INTEGRITY_DONE:
+ printf(_("done.\n"));
+ break;
+ case PM_TRANS_EVT_DELTA_PATCHES_START:
+ printf(_("applying deltas...\n"));
+ break;
+ case PM_TRANS_EVT_DELTA_PATCHES_DONE:
+ /* nothing */
+ break;
+ case PM_TRANS_EVT_DELTA_PATCH_START:
+ printf(_("generating %s with %s... "), (char *)data1, (char *)data2);
+ break;
+ case PM_TRANS_EVT_DELTA_PATCH_DONE:
+ printf(_("done.\n"));
+ break;
+ case PM_TRANS_EVT_DELTA_PATCH_FAILED:
+ printf(_("failed.\n"));
+ break;
case PM_TRANS_EVT_PRINTURI:
printf("%s/%s\n", (char*)data1, (char*)data2);
break;
@@ -309,7 +330,7 @@ void cb_trans_conv(pmtransconv_t event, void *data1, void *data2,
break;
case PM_TRANS_CONV_CORRUPTED_PKG:
if(!config->noconfirm) {
- snprintf(str, LOG_STR_LEN, _(":: Archive %s is corrupted. Do you want to delete it? [Y/n] "),
+ snprintf(str, LOG_STR_LEN, _(":: File %s is corrupted. Do you want to delete it? [Y/n] "),
(char *)data1);
*response = yesno(str);
} else {
diff --git a/src/pacman/pacman.c b/src/pacman/pacman.c
index 8474020..2f6f928 100644
--- a/src/pacman/pacman.c
+++ b/src/pacman/pacman.c
@@ -548,6 +548,9 @@ static int _parseconfig(const char *file, const char *givensection,
} else if(strcmp(key, "ShowSize") == 0 || strcmp(upperkey, "SHOWSIZE") == 0) {
config->showsize = 1;
pm_printf(PM_LOG_DEBUG, "config: showsize\n");
+ } else if(strcmp(key, "UseDelta") == 0 || strcmp(upperkey, "USEDELTA") == 0) {
+ alpm_option_set_usedelta(1);
+ pm_printf(PM_LOG_DEBUG, "config: usedelta\n");
} else {
pm_printf(PM_LOG_ERROR, _("config file %s, line %d: directive '%s' not recognized.\n"),
file, linenum, key);
--
1.5.3.4
More information about the pacman-dev
mailing list