[pacman-dev] [PATCH 4/5] Download delta files if UseDelta is set.

Delta files will be used if the size is smaller than a percent (MAX_DELTA_RATIO) of the package size. Signed-off-by: Nathan Jones <nathanj@insightbb.com> --- doc/pacman.conf.5.txt | 4 + lib/libalpm/alpm.h | 11 ++ lib/libalpm/error.c | 5 + lib/libalpm/handle.c | 6 + lib/libalpm/handle.h | 1 + lib/libalpm/sync.c | 248 ++++++++++++++++++++++++++++++++++++++++++++++++- lib/libalpm/util.h | 3 + src/pacman/callback.c | 23 +++++- src/pacman/pacman.c | 3 + 9 files changed, 300 insertions(+), 4 deletions(-) diff --git a/doc/pacman.conf.5.txt b/doc/pacman.conf.5.txt index d422a4b..06ba81d 100644 --- a/doc/pacman.conf.5.txt +++ b/doc/pacman.conf.5.txt @@ -105,6 +105,10 @@ Options *ShowSize*:: Display the size of individual packages for '\--sync' and '\--query' modes. +*UseDelta*:: + Download delta files instead of complete packages if possible. Requires + the xdelta program to be installed. + Repository Sections ------------------- diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 5f37d82..a4eaafa 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -138,6 +138,7 @@ void alpm_option_set_xfercommand(const char *cmd); unsigned short alpm_option_get_nopassiveftp(); void alpm_option_set_nopassiveftp(unsigned short nopasv); +void alpm_option_set_usedelta(unsigned short usedelta); pmdb_t *alpm_option_get_localdb(); alpm_list_t *alpm_option_get_syncdbs(); @@ -294,6 +295,13 @@ typedef enum _pmtransevt_t { PM_TRANS_EVT_EXTRACT_DONE, PM_TRANS_EVT_INTEGRITY_START, PM_TRANS_EVT_INTEGRITY_DONE, + PM_TRANS_EVT_DELTA_INTEGRITY_START, + PM_TRANS_EVT_DELTA_INTEGRITY_DONE, + PM_TRANS_EVT_DELTA_PATCHES_START, + PM_TRANS_EVT_DELTA_PATCHES_DONE, + PM_TRANS_EVT_DELTA_PATCH_START, + PM_TRANS_EVT_DELTA_PATCH_DONE, + PM_TRANS_EVT_DELTA_PATCH_FAILED, PM_TRANS_EVT_PRINTURI, PM_TRANS_EVT_RETRIEVE_START, } pmtransevt_t; @@ -442,6 +450,9 @@ enum _pmerrno_t { PM_ERR_PKG_INVALID_NAME, PM_ERR_PKG_CORRUPTED, PM_ERR_PKG_REPO_NOT_FOUND, + /* Deltas */ + PM_ERR_DLT_CORRUPTED, + PM_ERR_DLT_PATCHFAILED, /* Groups */ PM_ERR_GRP_NOT_FOUND, /* Dependencies */ diff --git a/lib/libalpm/error.c b/lib/libalpm/error.c index f81d22d..44d1c60 100644 --- a/lib/libalpm/error.c +++ b/lib/libalpm/error.c @@ -123,6 +123,11 @@ const char SYMEXPORT *alpm_strerror(int err) return _("corrupted package"); case PM_ERR_PKG_REPO_NOT_FOUND: return _("no such repository"); + /* Deltas */ + case PM_ERR_DLT_CORRUPTED: + return _("corrupted delta"); + case PM_ERR_DLT_PATCHFAILED: + return _("delta patch failed"); /* Groups */ case PM_ERR_GRP_NOT_FOUND: return _("group not found"); diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c index e8f2147..242bbe5 100644 --- a/lib/libalpm/handle.c +++ b/lib/libalpm/handle.c @@ -64,6 +64,7 @@ pmhandle_t *_alpm_handle_new() handle->cachedirs = NULL; handle->lockfile = NULL; handle->logfile = NULL; + handle->usedelta = 0; return(handle); } @@ -496,4 +497,9 @@ void SYMEXPORT alpm_option_set_nopassiveftp(unsigned short nopasv) handle->nopassiveftp = nopasv; } +void SYMEXPORT alpm_option_set_usedelta(unsigned short usedelta) +{ + handle->usedelta = usedelta; +} + /* vim: set ts=2 sw=2 noet: */ diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h index cf2e9d5..d8edf00 100644 --- a/lib/libalpm/handle.h +++ b/lib/libalpm/handle.h @@ -61,6 +61,7 @@ typedef struct _pmhandle_t { unsigned short nopassiveftp; /* Don't use PASV ftp connections */ time_t upgradedelay; /* Time to wait before upgrading a package */ char *xfercommand; /* External download command */ + unsigned short usedelta; /* Download deltas if possible */ } pmhandle_t; extern pmhandle_t *handle; diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index abc90de..d5f6500 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -48,6 +48,7 @@ #include "handle.h" #include "alpm.h" #include "server.h" +#include "delta.h" pmsyncpkg_t *_alpm_sync_new(int type, pmpkg_t *spkg, void *data) { @@ -700,6 +701,148 @@ cleanup: return(ret); } +/** Returns a list of deltas that should be downloaded instead of the + * package. + * + * It first tests if a delta path exists between the currently installed + * version (if any) and the version to upgrade to. If so, the delta path + * is used if its size is below a set percentage (MAX_DELTA_RATIO) of + * the package size, Otherwise, an empty list is returned. + * + * @param newpkg the new package to upgrade to + * @param db_local the local database + * + * @return the list of pmdelta_t * objects. NULL (the empty list) is + * returned if the package should be downloaded instead of deltas. + */ +static alpm_list_t *pkg_upgrade_delta_path(pmpkg_t *newpkg, pmdb_t *db_local) +{ + pmpkg_t *oldpkg = alpm_db_get_pkg(db_local, newpkg->name); + alpm_list_t *ret = NULL; + + if(oldpkg) { + const char *oldname = alpm_pkg_get_filename(oldpkg); + char *oldpath = _alpm_filecache_find(oldname); + + if(oldpath) { + alpm_list_t *deltas = _alpm_shortest_delta_path( + alpm_pkg_get_deltas(newpkg), + alpm_pkg_get_version(oldpkg), + alpm_pkg_get_version(newpkg)); + + if(deltas) { + unsigned long dltsize = _alpm_delta_path_size(deltas); + unsigned long pkgsize = alpm_pkg_get_size(newpkg); + + if(dltsize < pkgsize * MAX_DELTA_RATIO) { + ret = deltas; + } else { + ret = NULL; + alpm_list_free(deltas); + } + } + + FREE(oldpath); + } + } + + return(ret); +} + +/** Applies delta files to create an upgraded package file. + * + * All intermediate files are deleted, leaving only the starting and + * ending package files. + * + * @param trans the transaction + * @param patches A list of alternating pmpkg_t * and pmdelta_t * + * objects. The patch command will be built using the pmpkg_t, pmdelta_t + * pair. + * + * @return 0 if all delta files were able to be applied, 1 otherwise. + */ +static int apply_deltas(pmtrans_t *trans, alpm_list_t *patches) +{ + /* keep track of the previous package in the loop to decide if a + * package file should be deleted */ + pmpkg_t *lastpkg = NULL; + int lastpkg_failed = 0; + int ret = 0; + const char *cachedir = _alpm_filecache_setup(); + + alpm_list_t *p = patches; + while(p) { + pmpkg_t *pkg; + pmdelta_t *d; + char command[PATH_MAX], fname[PATH_MAX]; + char pkgfilename[PKG_FILENAME_LEN]; + + pkg = alpm_list_getdata(p); + p = alpm_list_next(p); + + d = alpm_list_getdata(p); + p = alpm_list_next(p); + + /* if patching fails, ignore the rest of that package's deltas */ + if(lastpkg_failed) { + if(pkg == lastpkg) { + continue; + } else { + lastpkg_failed = 0; + } + } + + /* an example of the patch command: (using /cache for cachedir) + * xdelta patch /cache/pacman_3.0.0-1_to_3.0.1-1-i686.delta \ + * /cache/pacman-3.0.0-1-i686.pkg.tar.gz \ + * /cache/pacman-3.0.1-1-i686.pkg.tar.gz + */ + + /* build the patch command */ + snprintf(command, PATH_MAX, + "xdelta patch" /* the command */ + " %s/%s" /* the delta */ + " %s/%s-%s-%s" PKGEXT /* the 'from' package */ + " %s/%s-%s-%s" PKGEXT, /* the 'to' package */ + cachedir, d->filename, + cachedir, pkg->name, d->from, pkg->arch, + cachedir, pkg->name, d->to, pkg->arch); + + _alpm_log(PM_LOG_DEBUG, _("command: %s\n"), command); + + snprintf(pkgfilename, PKG_FILENAME_LEN, "%s-%s-%s" PKGEXT, + pkg->name, d->to, pkg->arch); + + EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_START, pkgfilename, d->filename); + + if(system(command) == 0) { + EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_DONE, NULL, NULL); + + /* delete the delta file */ + snprintf(fname, PATH_MAX, "%s/%s", cachedir, d->filename); + unlink(fname); + + /* Delete the 'from' package but only if it is an intermediate + * package. The starting 'from' package should be kept, just + * as if deltas were not used. Delete the package file if the + * previous iteration of the loop used the same package. */ + if(pkg == lastpkg) { + snprintf(fname, PATH_MAX, "%s/%s-%s-%s" PKGEXT, + cachedir, pkg->name, d->from, pkg->arch); + unlink(fname); + } else { + lastpkg = pkg; + } + } else { + EVENT(trans, PM_TRANS_EVT_DELTA_PATCH_FAILED, NULL, NULL); + lastpkg_failed = 1; + ret = 1; + } + } + + return(ret); +} + /** Compares the md5sum of a file to the expected value. * * If the md5sum does not match, the user is asked whether the file @@ -762,6 +905,29 @@ static int test_md5sum(pmtrans_t *trans, const char *filename, return(ret); } +/** Compares the md5sum of a delta to the expected value. + * + * @param trans the transaction + * @param delta the delta to test + * @param data data to write the error messages to + * + * @return 0 if the md5sum matched, 1 otherwise + */ +static int test_delta_md5sum(pmtrans_t *trans, pmdelta_t *delta, + alpm_list_t **data) +{ + const char *filename; + char *md5sum; + int ret = 0; + + filename = alpm_delta_get_filename(delta); + md5sum = alpm_delta_get_md5sum(delta); + + ret = test_md5sum(trans, filename, md5sum, data); + + return(ret); +} + /** Compares the md5sum of a package to the expected value. * * @param trans the transaction @@ -787,6 +953,7 @@ static int test_pkg_md5sum(pmtrans_t *trans, pmpkg_t *pkg, alpm_list_t **data) int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data) { alpm_list_t *i, *j, *files = NULL; + alpm_list_t *patches = NULL, *deltas = NULL; pmtrans_t *tr = NULL; int replaces = 0, retval = 0; const char *cachedir = NULL; @@ -817,8 +984,42 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data) } else { char *fpath = _alpm_filecache_find(fname); if(!fpath) { - /* file is not in the cache dir, so add it to the list */ - files = alpm_list_add(files, strdup(fname)); + if(handle->usedelta) { + alpm_list_t *delta_path = pkg_upgrade_delta_path(spkg, db_local); + + if(delta_path) { + alpm_list_t *dlts = NULL; + + for(dlts = delta_path; dlts; dlts = alpm_list_next(dlts)) { + pmdelta_t *d = (pmdelta_t *)alpm_list_getdata(dlts); + char *fpath2 = _alpm_filecache_find(d->filename); + + if(!fpath2) { + /* add the delta filename to the download list if + * it's not in the cache*/ + files = alpm_list_add(files, strdup(d->filename)); + } + + /* save the package and delta so that the xdelta patch + * command can be run after the downloads finish */ + patches = alpm_list_add(patches, spkg); + patches = alpm_list_add(patches, d); + + /* keep a list of the delta files for md5sums */ + deltas = alpm_list_add(deltas, d); + } + + alpm_list_free(delta_path); + delta_path = NULL; + } else { + /* no deltas to download, so add the file to the + * download list */ + files = alpm_list_add(files, strdup(fname)); + } + } else { + /* not using deltas, so add the file to the download list */ + files = alpm_list_add(files, strdup(fname)); + } } FREE(fpath); } @@ -839,7 +1040,48 @@ int _alpm_sync_commit(pmtrans_t *trans, pmdb_t *db_local, alpm_list_t **data) return(0); } - /* Check integrity of files */ + if(handle->usedelta) { + int ret = 0; + + /* only output if there are deltas to work with */ + if(deltas) { + /* Check integrity of deltas */ + EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_START, NULL, NULL); + + for(i = deltas; i; i = i->next) { + pmdelta_t *d = alpm_list_getdata(i); + + ret = test_delta_md5sum(trans, d, data); + + if(ret == 1) { + retval = 1; + } else if(ret == -1) { /* -1 is for serious errors */ + RET_ERR(pm_errno, -1); + } + } + if(retval) { + pm_errno = PM_ERR_DLT_CORRUPTED; + goto error; + } + EVENT(trans, PM_TRANS_EVT_DELTA_INTEGRITY_DONE, NULL, NULL); + + /* Use the deltas to generate the packages */ + EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_START, NULL, NULL); + ret = apply_deltas(trans, patches); + EVENT(trans, PM_TRANS_EVT_DELTA_PATCHES_DONE, NULL, NULL); + + alpm_list_free(patches); + patches = NULL; + alpm_list_free(deltas); + deltas = NULL; + } + if(ret) { + pm_errno = PM_ERR_DLT_PATCHFAILED; + goto error; + } + } + + /* Check integrity of packages */ EVENT(trans, PM_TRANS_EVT_INTEGRITY_START, NULL, NULL); for(i = trans->packages; i; i = i->next) { diff --git a/lib/libalpm/util.h b/lib/libalpm/util.h index 5ebc70c..17fa123 100644 --- a/lib/libalpm/util.h +++ b/lib/libalpm/util.h @@ -67,6 +67,9 @@ char *strsep(char **, const char *); #define SYMEXPORT __attribute__((visibility("default"))) #define SYMHIDDEN __attribute__((visibility("internal"))) +/* max percent of package size to download deltas */ +#define MAX_DELTA_RATIO 0.7 + #endif /* _ALPM_UTIL_H */ /* vim: set ts=2 sw=2 noet: */ diff --git a/src/pacman/callback.c b/src/pacman/callback.c index 6d25713..aec9753 100644 --- a/src/pacman/callback.c +++ b/src/pacman/callback.c @@ -236,6 +236,27 @@ void cb_trans_evt(pmtransevt_t event, void *data1, void *data2) case PM_TRANS_EVT_INTEGRITY_DONE: printf(_("done.\n")); break; + case PM_TRANS_EVT_DELTA_INTEGRITY_START: + printf(_("checking delta integrity... ")); + break; + case PM_TRANS_EVT_DELTA_INTEGRITY_DONE: + printf(_("done.\n")); + break; + case PM_TRANS_EVT_DELTA_PATCHES_START: + printf(_("applying deltas...\n")); + break; + case PM_TRANS_EVT_DELTA_PATCHES_DONE: + /* nothing */ + break; + case PM_TRANS_EVT_DELTA_PATCH_START: + printf(_("generating %s with %s... "), (char *)data1, (char *)data2); + break; + case PM_TRANS_EVT_DELTA_PATCH_DONE: + printf(_("done.\n")); + break; + case PM_TRANS_EVT_DELTA_PATCH_FAILED: + printf(_("failed.\n")); + break; case PM_TRANS_EVT_PRINTURI: printf("%s/%s\n", (char*)data1, (char*)data2); break; @@ -309,7 +330,7 @@ void cb_trans_conv(pmtransconv_t event, void *data1, void *data2, break; case PM_TRANS_CONV_CORRUPTED_PKG: if(!config->noconfirm) { - snprintf(str, LOG_STR_LEN, _(":: Archive %s is corrupted. Do you want to delete it? [Y/n] "), + snprintf(str, LOG_STR_LEN, _(":: File %s is corrupted. Do you want to delete it? [Y/n] "), (char *)data1); *response = yesno(str); } else { diff --git a/src/pacman/pacman.c b/src/pacman/pacman.c index 8474020..2f6f928 100644 --- a/src/pacman/pacman.c +++ b/src/pacman/pacman.c @@ -548,6 +548,9 @@ static int _parseconfig(const char *file, const char *givensection, } else if(strcmp(key, "ShowSize") == 0 || strcmp(upperkey, "SHOWSIZE") == 0) { config->showsize = 1; pm_printf(PM_LOG_DEBUG, "config: showsize\n"); + } else if(strcmp(key, "UseDelta") == 0 || strcmp(upperkey, "USEDELTA") == 0) { + alpm_option_set_usedelta(1); + pm_printf(PM_LOG_DEBUG, "config: usedelta\n"); } else { pm_printf(PM_LOG_ERROR, _("config file %s, line %d: directive '%s' not recognized.\n"), file, linenum, key); -- 1.5.3.4