- Pacman-dev - lists.archlinux.org

[pacman-dev] [PATCH 3/3] More consistent printing of off_t and time_t
by Xavier Chantry 16 Nov '09

16 Nov '09

time_t : %ld off_t : %jd and cast to intmax_t Signed-off-by: Xavier Chantry <shiningxc(a)gmail.com> --- lib/libalpm/be_files.c | 6 +++--- lib/libalpm/delta.c | 2 +- lib/libalpm/dload.c | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/libalpm/be_files.c b/lib/libalpm/be_files.c index 5cbc881..7fc20b8 100644 --- a/lib/libalpm/be_files.c +++ b/lib/libalpm/be_files.c @@ -25,7 +25,7 @@ #include <stdlib.h> #include <errno.h> #include <string.h> -#include <stdint.h> /* uintmax_t, intmax_t */ +#include <stdint.h> /* intmax_t */ #include <sys/stat.h> #include <dirent.h> #include <ctype.h> @@ -839,11 +839,11 @@ int _alpm_db_write(pmdb_t *db, pmpkg_t *info, pmdbinfrq_t inforeq) } if(info->builddate) { fprintf(fp, "%%BUILDDATE%%\n" - "%ju\n\n", (uintmax_t)info->builddate); + "%ld\n\n", info->builddate); } if(info->installdate) { fprintf(fp, "%%INSTALLDATE%%\n" - "%ju\n\n", (uintmax_t)info->installdate); + "%ld\n\n", info->installdate); } if(info->packager) { fprintf(fp, "%%PACKAGER%%\n" diff --git a/lib/libalpm/delta.c b/lib/libalpm/delta.c index 0fc37d9..a5ce611 100644 --- a/lib/libalpm/delta.c +++ b/lib/libalpm/delta.c @@ -337,7 +337,7 @@ pmdelta_t *_alpm_delta_parse(char *line) tmp2 = tmp; STRDUP(delta->to, tmp2, RET_ERR(PM_ERR_MEMORY, NULL)); - _alpm_log(PM_LOG_DEBUG, "delta : %s %s '%lld'\n", delta->from, delta->to, (long long)delta->delta_size); + _alpm_log(PM_LOG_DEBUG, "delta : %s %s '%jd'\n", delta->from, delta->to, (intmax_t)delta->delta_size); return(delta); } diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 05555f2..bc5cdfd 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -186,8 +186,8 @@ static int download_internal(const char *url, const char *localpath, _alpm_log(PM_LOG_DEBUG, "ust.mtime: %ld local_time: %ld compare: %ld\n", ust.mtime, local_time, local_time - ust.mtime); - _alpm_log(PM_LOG_DEBUG, "ust.size: %"PRId64" local_size: %"PRId64" compare: %"PRId64"\n", - ust.size, local_size, local_size - ust.size); + _alpm_log(PM_LOG_DEBUG, "ust.size: %jd local_size: %jd compare: %jd\n", + (intmax_t)ust.size, (intmax_t)local_size, (intmax_t)(local_size - ust.size)); if(!force && ust.mtime && ust.mtime == local_time && ust.size && ust.size == local_size) { /* the remote time and size values agreed with what we have, so move on @@ -220,7 +220,7 @@ static int download_internal(const char *url, const char *localpath, fclose(localf); localf = NULL; } else if(fileurl->offset) { - _alpm_log(PM_LOG_DEBUG, "resuming download at position %"PRId64"\n", fileurl->offset); + _alpm_log(PM_LOG_DEBUG, "resuming download at position %jd\n", (intmax_t)fileurl->offset); } -- 1.6.5.2

1 0

[pacman-dev] [PATCH 2/2] More consistent printing of off_t and time_t
by Xavier Chantry 16 Nov '09

16 Nov '09

%ld for type_t %"PRId64" for off_t Signed-off-by: Xavier Chantry <shiningxc(a)gmail.com> --- lib/libalpm/add.c | 7 +++---- lib/libalpm/be_files.c | 12 ++++++------ lib/libalpm/delta.c | 6 +++--- lib/libalpm/dload.c | 4 ++-- lib/libalpm/sync.c | 6 +++--- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/lib/libalpm/add.c b/lib/libalpm/add.c index ebcd6a5..05bb8d4 100644 --- a/lib/libalpm/add.c +++ b/lib/libalpm/add.c @@ -28,8 +28,7 @@ #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> -#include <inttypes.h> /* int64_t */ -#include <stdint.h> /* intmax_t */ +#include <inttypes.h> /* int64_t,PRId64 */ /* libarchive */ #include <archive.h> @@ -602,8 +601,8 @@ static int commit_single_pkg(pmpkg_t *newpkg, int pkg_current, int pkg_count, int64_t pos = archive_position_compressed(archive); percent = (double)pos / (double)newpkg->size; _alpm_log(PM_LOG_DEBUG, "decompression progress: " - "%f%% (%"PRId64" / %jd)\n", - percent*100.0, pos, (intmax_t)newpkg->size); + "%f%% (%"PRId64" / %"PRId64")\n", + percent*100.0, pos, newpkg->size); if(percent >= 1.0) { percent = 1.0; } diff --git a/lib/libalpm/be_files.c b/lib/libalpm/be_files.c index 90e97a5..dc95bdb 100644 --- a/lib/libalpm/be_files.c +++ b/lib/libalpm/be_files.c @@ -25,7 +25,7 @@ #include <stdlib.h> #include <errno.h> #include <string.h> -#include <stdint.h> /* uintmax_t, intmax_t */ +#include <inttypes.h> /* PRId64 */ #include <sys/stat.h> #include <dirent.h> #include <ctype.h> @@ -697,11 +697,11 @@ int _alpm_db_write(pmdb_t *db, pmpkg_t *info, pmdbinfrq_t inforeq) } if(info->builddate) { fprintf(fp, "%%BUILDDATE%%\n" - "%ju\n\n", (uintmax_t)info->builddate); + "%ld\n\n", info->builddate); } if(info->installdate) { fprintf(fp, "%%INSTALLDATE%%\n" - "%ju\n\n", (uintmax_t)info->installdate); + "%ld\n\n", info->installdate); } if(info->packager) { fprintf(fp, "%%PACKAGER%%\n" @@ -710,7 +710,7 @@ int _alpm_db_write(pmdb_t *db, pmpkg_t *info, pmdbinfrq_t inforeq) if(info->isize) { /* only write installed size, csize is irrelevant once installed */ fprintf(fp, "%%SIZE%%\n" - "%jd\n\n", (intmax_t)info->isize); + "%"PRId64"\n\n", info->isize); } if(info->reason) { fprintf(fp, "%%REASON%%\n" @@ -719,11 +719,11 @@ int _alpm_db_write(pmdb_t *db, pmpkg_t *info, pmdbinfrq_t inforeq) } else { if(info->size) { fprintf(fp, "%%CSIZE%%\n" - "%jd\n\n", (intmax_t)info->size); + "%"PRId64"\n\n", info->size); } if(info->isize) { fprintf(fp, "%%ISIZE%%\n" - "%jd\n\n", (intmax_t)info->isize); + "%"PRId64"\n\n", info->isize); } if(info->md5sum) { fprintf(fp, "%%MD5SUM%%\n" diff --git a/lib/libalpm/delta.c b/lib/libalpm/delta.c index 523968e..fc4afd1 100644 --- a/lib/libalpm/delta.c +++ b/lib/libalpm/delta.c @@ -22,7 +22,7 @@ #include <stdlib.h> #include <string.h> -#include <stdint.h> /* intmax_t */ +#include <inttypes.h> /* PRId64 */ #include <limits.h> #include <sys/types.h> #include <regex.h> @@ -238,7 +238,7 @@ off_t _alpm_shortest_delta_path(alpm_list_t *deltas, dijkstra(vertices); bestsize = shortest_path(vertices, to, &bestpath); - _alpm_log(PM_LOG_DEBUG, "delta shortest-path search complete : '%jd'\n", (intmax_t)bestsize); + _alpm_log(PM_LOG_DEBUG, "delta shortest-path search complete : '%"PRId64"'\n", bestsize); alpm_list_free_inner(vertices, _alpm_graph_free); alpm_list_free(vertices); @@ -297,7 +297,7 @@ pmdelta_t *_alpm_delta_parse(char *line) tmp2 = tmp; STRDUP(delta->to, tmp2, RET_ERR(PM_ERR_MEMORY, NULL)); - _alpm_log(PM_LOG_DEBUG, "delta : %s %s '%lld'\n", delta->from, delta->to, (long long)delta->delta_size); + _alpm_log(PM_LOG_DEBUG, "delta : %s %s '%"PRId64"'\n", delta->from, delta->to, delta->delta_size); return(delta); } diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 05555f2..717954f 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -271,8 +271,8 @@ static int download_internal(const char *url, const char *localpath, if (ust.size != -1 && dl_thisfile < ust.size) { pm_errno = PM_ERR_RETRIEVE; - _alpm_log(PM_LOG_ERROR, _("%s appears to be truncated: %jd/%jd bytes\n"), - filename, (intmax_t)dl_thisfile, (intmax_t)ust.size); + _alpm_log(PM_LOG_ERROR, _("%s appears to be truncated: %"PRId64"/%"PRId64" bytes\n"), + filename, dl_thisfile, ust.size); ret = -1; goto cleanup; } diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 2cdcd47..e3155fc 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -28,7 +28,7 @@ #include <stdio.h> #include <fcntl.h> #include <string.h> -#include <stdint.h> /* intmax_t */ +#include <inttypes.h> /* PRId64 */ #include <unistd.h> #include <time.h> #include <dirent.h> @@ -389,8 +389,8 @@ static int compute_download_size(pmpkg_t *newpkg) size = alpm_pkg_get_size(newpkg); } - _alpm_log(PM_LOG_DEBUG, "setting download size %jd for pkg %s\n", - (intmax_t)size, alpm_pkg_get_name(newpkg)); + _alpm_log(PM_LOG_DEBUG, "setting download size %"PRId64" for pkg %s\n", + size, alpm_pkg_get_name(newpkg)); newpkg->download_size = size; return(0); -- 1.6.5.2

3 2

[pacman-dev] [PATCH] Add valgrind usage for testing purpose into the HACKING guide
by Laszlo Papp 16 Nov '09

16 Nov '09

Signed-off-by: Laszlo Papp <djszapi(a)archlinux.us> --- HACKING | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diff --git a/HACKING b/HACKING index 09e782d..954d4b7 100644 --- a/HACKING +++ b/HACKING @@ -133,6 +133,16 @@ For pacman: #include "anythingelse.h" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +How to test pacman more using valgrind? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Try to run some commands using valgrind with a command like: + +valgrind --leak-check=full --trace-children=yes -- src/pacman/.libs/pacman -Syu + +Be careful, valgrind reports lots of errors, even some errors that may not be related to +pacman. You should have a little experience of valgrind to ignore the noise. + ///// vim: set ts=2 sw=2 syntax=asciidoc et: ///// -- 1.6.5

5 5

[pacman-dev] [PATCH] download: major refactor to address lingering issues
by Dan McGee 16 Nov '09

16 Nov '09

Sorry for this being such a huge patch, but I believe it is necessary for quite a few reasons which I will attempt to explain herein. I've been mulling this over for a while, but wasn't super happy with making the download interface more complex. Instead, if we carefully order things in the internal download code, we can actually make the interface simpler. 1. FS#15657 - This involves `name.db.tar.gz.part` files being left around the filesystem, and then causing all sorts of issues when someone attempts to rerun the operation they canceled. We need to ensure that if we resume a download, we are resuming it on exactly the same file; if we cannot be almost postive of that then we need to start over. 2. http://www.mail-archive.com/pacman-dev@archlinux.org/msg03536.html - Here we have a lighttpd bug to ruin the day. If we send both a Range: header and If-Modified-Since: header across the wire in a GET request, lighttpd doesn't do what we want in several cases. If the file hadn't been modified, it returns a '304 Not Modified' instead of a '206 Partial Content'. We need to do a stat (e.g. HEAD in HTTP terms) operation here, and the proceed accordingly based off the values we get back from it. 3. The mtime stuff was rather ugly, and relied on the called function to write back to a passed in reference, which isn't the greatest. Instead, use the power of the filesystem to contain this info. Every file downloaded internally is now carefully timestamped with the remote file time. This should allow the resume logic to work. In order to guarantee this, we need to implement a signal handler that catches interrupts, notifies the running code, and causes it to set the mtimes on the file. It then rethrows the signal so the pacman signal handler (or any frontend) works as expected. 4. We did a lot of funky stuff in trying to track the DB last modified time. It is a lot easier to just keep the downloaded DB file around and track the time on that rather than in a funky dot file. It also kills a lot of code. 5. For GPG verification of the databases down the road, we are going to need the DB file around for at least a short bit of time anyway, so this gets us closer to that. Signed-off-by: Dan McGee <dan(a)archlinux.org> --- lib/libalpm/alpm.h | 8 +-- lib/libalpm/be_files.c | 96 ++------------------------- lib/libalpm/dload.c | 177 +++++++++++++++++++++++++++++++++--------------- lib/libalpm/dload.h | 4 +- src/pacman/pacman.c | 11 +++- 5 files changed, 141 insertions(+), 155 deletions(-) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index c433b73..05ad82e 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -86,14 +86,12 @@ typedef void (*alpm_cb_totaldl)(off_t total); /** A callback for downloading files * @param url the URL of the file to be downloaded * @param localpath the directory to which the file should be downloaded - * @param mtimeold the modification time of the file previously downloaded - * @param mtimenew the modification time of the newly downloaded file. - * This should be set by the callback. - * @return 0 on success, 1 if the modification times are identical, -1 on + * @param force whether to force an update, even if the file is the same + * @return 0 on success, 1 if the file exists and is identical, -1 on * error. */ typedef int (*alpm_cb_fetch)(const char *url, const char *localpath, - time_t mtimeold, time_t *mtimenew); + int force); /* * Options diff --git a/lib/libalpm/be_files.c b/lib/libalpm/be_files.c index ffbaa8d..90e97a5 100644 --- a/lib/libalpm/be_files.c +++ b/lib/libalpm/be_files.c @@ -47,76 +47,6 @@ #include "dload.h" -/* - * Return the last update time as number of seconds from the epoch. - * Returns 0 if the value is unknown or can't be read. - */ -static time_t getlastupdate(pmdb_t *db) -{ - FILE *fp; - char *file; - const char *dbpath; - time_t ret = 0; - - ALPM_LOG_FUNC; - - if(db == NULL) { - return(ret); - } - - dbpath = _alpm_db_path(db); - /* dbpath + '.lastupdate' + NULL */ - MALLOC(file, strlen(dbpath) + 12, RET_ERR(PM_ERR_MEMORY, ret)); - sprintf(file, "%s.lastupdate", dbpath); - - /* get the last update time, if it's there */ - if((fp = fopen(file, "r")) == NULL) { - free(file); - return(ret); - } else { - char line[64]; - if(fgets(line, sizeof(line), fp)) { - ret = atol(line); - } - } - fclose(fp); - free(file); - return(ret); -} - -/* - * writes the dbpath/.lastupdate file with the value in time - */ -static int setlastupdate(pmdb_t *db, time_t time) -{ - FILE *fp; - char *file; - const char *dbpath; - int ret = 0; - - ALPM_LOG_FUNC; - - if(db == NULL || time == 0) { - return(-1); - } - - dbpath = _alpm_db_path(db); - /* dbpath + '.lastupdate' + NULL */ - MALLOC(file, strlen(dbpath) + 12, RET_ERR(PM_ERR_MEMORY, ret)); - sprintf(file, "%s.lastupdate", dbpath); - - if((fp = fopen(file, "w")) == NULL) { - free(file); - return(-1); - } - if(fprintf(fp, "%ju", (uintmax_t)time) <= 0) { - ret = -1; - } - fclose(fp); - free(file); - return(ret); -} - static int checkdbdir(pmdb_t *db) { struct stat buf; @@ -178,7 +108,6 @@ static int checkdbdir(pmdb_t *db) int SYMEXPORT alpm_db_update(int force, pmdb_t *db) { char *dbfile, *dbfilepath; - time_t newmtime = 0, lastupdate = 0; const char *dbpath; size_t len; @@ -200,27 +129,17 @@ int SYMEXPORT alpm_db_update(int force, pmdb_t *db) RET_ERR(PM_ERR_DB_NOT_FOUND, -1); } - if(!force) { - /* get the lastupdate time */ - lastupdate = getlastupdate(db); - if(lastupdate == 0) { - _alpm_log(PM_LOG_DEBUG, "failed to get lastupdate time for %s\n", - db->treename); - } - } - len = strlen(db->treename) + strlen(DBEXT) + 1; MALLOC(dbfile, len, RET_ERR(PM_ERR_MEMORY, -1)); sprintf(dbfile, "%s" DBEXT, db->treename); dbpath = alpm_option_get_dbpath(); - ret = _alpm_download_single_file(dbfile, db->servers, dbpath, - lastupdate, &newmtime); + ret = _alpm_download_single_file(dbfile, db->servers, dbpath, force); free(dbfile); if(ret == 1) { - /* mtimes match, do nothing */ + /* files match, do nothing */ pm_errno = 0; return(1); } else if(ret == -1) { @@ -233,8 +152,11 @@ int SYMEXPORT alpm_db_update(int force, pmdb_t *db) if(_alpm_rmrf(syncdbpath) != 0) { _alpm_log(PM_LOG_ERROR, _("could not remove database %s\n"), db->treename); RET_ERR(PM_ERR_DB_REMOVE, -1); + } else { + _alpm_log(PM_LOG_DEBUG, "database dir %s removed\n", _alpm_db_path(db)); } + /* Cache needs to be rebuilt */ _alpm_db_free_pkgcache(db); @@ -250,15 +172,7 @@ int SYMEXPORT alpm_db_update(int force, pmdb_t *db) free(dbfilepath); RET_ERR(PM_ERR_SYSTEM, -1); } - unlink(dbfilepath); free(dbfilepath); - - /* if we have a new mtime, set the DB last update value */ - if(newmtime) { - _alpm_log(PM_LOG_DEBUG, "sync: new mtime for %s: %ju\n", - db->treename, (uintmax_t)newmtime); - setlastupdate(db, newmtime); - } } return(0); diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index a4c9f1f..683f514 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -25,6 +25,9 @@ #include <errno.h> #include <string.h> #include <unistd.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/stat.h> #include <signal.h> #include <limits.h> /* the following two are needed on BSD for libfetch */ @@ -85,18 +88,32 @@ static const char *gethost(struct url *fileurl) return(host); } +int dload_interrupted; +static RETSIGTYPE inthandler(int signum) +{ + dload_interrupted = 1; +} + +#define check_stop() if(dload_interrupted) { ret = -1; goto cleanup; } +enum sighandlers { OLD = 0, NEW = 1 }; + static int download_internal(const char *url, const char *localpath, - time_t mtimeold, time_t *mtimenew) { - fetchIO *dlf = NULL; + int force) { FILE *localf = NULL; - struct url_stat ust; struct stat st; - int chk_resume = 0, ret = 0; + int ret = 0; off_t dl_thisfile = 0; ssize_t nread = 0; char *tempfile, *destfile, *filename; - struct sigaction new_action, old_action; + struct sigaction sig_pipe[2], sig_int[2]; + + off_t local_size = 0; + time_t local_time = 0; + struct url *fileurl; + struct url_stat ust; + fetchIO *dlf = NULL; + char buffer[PM_DLBUF_LEN]; filename = get_filename(url); @@ -113,51 +130,80 @@ static int download_internal(const char *url, const char *localpath, destfile = get_destfile(localpath, filename); tempfile = get_tempfile(localpath, filename); - if(mtimeold) { - fileurl->last_modified = mtimeold; - } - - /* pass the raw filename for passing to the callback function */ - _alpm_log(PM_LOG_DEBUG, "using '%s' for download progress\n", filename); - if(stat(tempfile, &st) == 0 && st.st_size > 0) { - _alpm_log(PM_LOG_DEBUG, "existing file found, using it\n"); - fileurl->offset = (off_t)st.st_size; + _alpm_log(PM_LOG_DEBUG, "tempfile found, attempting continuation\n"); + local_time = fileurl->last_modified = st.st_mtime; + local_size = fileurl->offset = (off_t)st.st_size; dl_thisfile = st.st_size; localf = fopen(tempfile, "ab"); - chk_resume = 1; + } else if(!force && stat(destfile, &st) == 0 && st.st_size > 0) { + _alpm_log(PM_LOG_DEBUG, "destfile found, using mtime only\n"); + local_time = fileurl->last_modified = st.st_mtime; + local_size = /* no fu->off here */ (off_t)st.st_size; } else { - fileurl->offset = (off_t)0; - dl_thisfile = 0; + _alpm_log(PM_LOG_DEBUG, "no file found matching criteria, starting from scratch\n"); } + /* pass the raw filename for passing to the callback function */ + _alpm_log(PM_LOG_DEBUG, "using '%s' for download progress\n", filename); + /* print proxy info for debug purposes */ _alpm_log(PM_LOG_DEBUG, "HTTP_PROXY: %s\n", getenv("HTTP_PROXY")); _alpm_log(PM_LOG_DEBUG, "http_proxy: %s\n", getenv("http_proxy")); _alpm_log(PM_LOG_DEBUG, "FTP_PROXY: %s\n", getenv("FTP_PROXY")); _alpm_log(PM_LOG_DEBUG, "ftp_proxy: %s\n", getenv("ftp_proxy")); - /* libfetch does not reset the error code */ - fetchLastErrCode = 0; - /* 10s timeout */ fetchTimeout = 10; /* ignore any SIGPIPE signals- these may occur if our FTP socket dies or * something along those lines. Store the old signal handler first. */ - new_action.sa_handler = SIG_IGN; - sigemptyset(&new_action.sa_mask); - new_action.sa_flags = 0; - sigaction(SIGPIPE, NULL, &old_action); - sigaction(SIGPIPE, &new_action, NULL); - - dlf = fetchXGet(fileurl, &ust, "i"); - - if(fetchLastErrCode == FETCH_UNCHANGED) { - _alpm_log(PM_LOG_DEBUG, "mtimes are identical, skipping %s\n", filename); + sig_pipe[NEW].sa_handler = SIG_IGN; + sigemptyset(&sig_pipe[NEW].sa_mask); + sig_pipe[NEW].sa_flags = 0; + sigaction(SIGPIPE, NULL, &sig_pipe[OLD]); + sigaction(SIGPIPE, &sig_pipe[NEW], NULL); + + dload_interrupted = 0; + sig_int[NEW].sa_handler = &inthandler; + sigemptyset(&sig_int[NEW].sa_mask); + sig_int[NEW].sa_flags = 0; + sigaction(SIGINT, NULL, &sig_int[OLD]); + sigaction(SIGINT, &sig_int[NEW], NULL); + + /* NOTE: libfetch does not reset the error code, be sure to do it before + * calls into the library */ + + /* find out the remote size *and* mtime in one go. there is a lot of + * trouble in trying to do both size and "if-modified-since" logic in a + * non-stat request, so avoid it. */ + fetchLastErrCode = 0; + if(fetchStat(fileurl, &ust, "") == -1) { + ret = -1; + goto cleanup; + } + check_stop(); + + _alpm_log(PM_LOG_DEBUG, "ust.mtime: %ld local_time: %ld compare: %ld\n", + ust.mtime, local_time, local_time - ust.mtime); + _alpm_log(PM_LOG_DEBUG, "ust.size: %ld local_size: %ld compare: %ld\n", + ust.size, local_size, local_size - ust.size); + if(!force && ust.mtime && ust.mtime == local_time + && ust.size && ust.size == local_size) { + /* the remote time and size values agreed with what we have, so move on + * because there is nothing more to do. */ + _alpm_log(PM_LOG_DEBUG, "files are identical, skipping %s\n", filename); ret = 1; goto cleanup; } + if(!ust.mtime || ust.mtime != local_time) { + _alpm_log(PM_LOG_DEBUG, "mtimes were different or unavailable, downloading %s from beginning\n", filename); + fileurl->offset = 0; + } + + fetchLastErrCode = 0; + dlf = fetchGet(fileurl, ""); + check_stop(); if(fetchLastErrCode != 0 || dlf == NULL) { pm_errno = PM_ERR_LIBFETCH; @@ -169,17 +215,14 @@ static int download_internal(const char *url, const char *localpath, _alpm_log(PM_LOG_DEBUG, "connected to %s successfully\n", fileurl->host); } - if(ust.mtime && mtimenew) { - *mtimenew = ust.mtime; + if(localf && fileurl->offset == 0) { + _alpm_log(PM_LOG_WARNING, _("resuming download of %s not possible; starting over\n"), filename); + fclose(localf); + localf = NULL; + } else if(fileurl->offset) { + _alpm_log(PM_LOG_DEBUG, "resuming download at position %ld\n", fileurl->offset); } - if(chk_resume && fileurl->offset == 0) { - _alpm_log(PM_LOG_WARNING, _("cannot resume download, starting over\n")); - if(localf != NULL) { - fclose(localf); - localf = NULL; - } - } if(localf == NULL) { _alpm_rmrf(tempfile); @@ -187,7 +230,8 @@ static int download_internal(const char *url, const char *localpath, dl_thisfile = 0; localf = fopen(tempfile, "wb"); if(localf == NULL) { /* still null? */ - _alpm_log(PM_LOG_ERROR, _("cannot write to file '%s'\n"), tempfile); + _alpm_log(PM_LOG_ERROR, _("error writing to file '%s': %s\n"), + tempfile, strerror(errno)); ret = -1; goto cleanup; } @@ -199,11 +243,12 @@ static int download_internal(const char *url, const char *localpath, } while((nread = fetchIO_read(dlf, buffer, PM_DLBUF_LEN)) > 0) { + check_stop(); size_t nwritten = 0; nwritten = fwrite(buffer, 1, nread, localf); if((nwritten != nread) || ferror(localf)) { _alpm_log(PM_LOG_ERROR, _("error writing to file '%s': %s\n"), - destfile, strerror(errno)); + tempfile, strerror(errno)); ret = -1; goto cleanup; } @@ -240,36 +285,60 @@ static int download_internal(const char *url, const char *localpath, fetchIO_close(dlf); dlf = NULL; + /* set the times on the file to the same as that of the remote file */ + if(ust.mtime) { + struct timeval tv[2]; + memset(&tv, 0, sizeof(tv)); + tv[0].tv_sec = ust.atime; + tv[1].tv_sec = ust.mtime; + utimes(tempfile, tv); + } rename(tempfile, destfile); ret = 0; cleanup: - /* restore any existing SIGPIPE signal handler */ - sigaction(SIGPIPE, &old_action, NULL); - FREE(tempfile); FREE(destfile); if(localf != NULL) { + /* if we still had a local file open, we got interrupted. set the mtimes on + * the file accordingly. */ + fflush(localf); + if(ust.mtime) { + struct timeval tv[2]; + memset(&tv, 0, sizeof(tv)); + tv[0].tv_sec = ust.atime; + tv[1].tv_sec = ust.mtime; + futimes(fileno(localf), tv); + } fclose(localf); } if(dlf != NULL) { fetchIO_close(dlf); } fetchFreeURL(fileurl); + + /* restore the old signal handlers */ + sigaction(SIGINT, &sig_int[OLD], NULL); + sigaction(SIGPIPE, &sig_pipe[OLD], NULL); + /* if we were interrupted, trip the old handler */ + if(dload_interrupted) { + raise(SIGINT); + } + return(ret); } #endif static int download(const char *url, const char *localpath, - time_t mtimeold, time_t *mtimenew) { + int force) { if(handle->fetchcb == NULL) { #if defined(INTERNAL_DOWNLOAD) - return(download_internal(url, localpath, mtimeold, mtimenew)); + return(download_internal(url, localpath, force)); #else RET_ERR(PM_ERR_EXTERNAL_DOWNLOAD, -1); #endif } else { - int ret = handle->fetchcb(url, localpath, mtimeold, mtimenew); + int ret = handle->fetchcb(url, localpath, force); if(ret == -1) { RET_ERR(PM_ERR_EXTERNAL_DOWNLOAD, -1); } @@ -279,19 +348,15 @@ static int download(const char *url, const char *localpath, /* * Download a single file - * - if mtimeold is non-NULL, then only download the file if it's different - * than mtimeold. - * - if *mtimenew is non-NULL, it will be filled with the mtime of the remote - * file. * - servers must be a list of urls WITHOUT trailing slashes. * * RETURN: 0 for successful download - * 1 if the mtimes are identical + * 1 if the files are identical * -1 on error */ int _alpm_download_single_file(const char *filename, alpm_list_t *servers, const char *localpath, - time_t mtimeold, time_t *mtimenew) + int force) { alpm_list_t *i; int ret = -1; @@ -308,7 +373,7 @@ int _alpm_download_single_file(const char *filename, CALLOC(fileurl, len, sizeof(char), RET_ERR(PM_ERR_MEMORY, -1)); snprintf(fileurl, len, "%s/%s", server, filename); - ret = download(fileurl, localpath, mtimeold, mtimenew); + ret = download(fileurl, localpath, force); FREE(fileurl); if(ret != -1) { break; @@ -327,7 +392,7 @@ int _alpm_download_files(alpm_list_t *files, for(lp = files; lp; lp = lp->next) { char *filename = lp->data; if(_alpm_download_single_file(filename, servers, - localpath, 0, NULL) == -1) { + localpath, 0) == -1) { ret++; } } @@ -354,7 +419,7 @@ char SYMEXPORT *alpm_fetch_pkgurl(const char *url) cachedir = _alpm_filecache_setup(); /* download the file */ - ret = download(url, cachedir, 0, NULL); + ret = download(url, cachedir, 0); if(ret == -1) { _alpm_log(PM_LOG_WARNING, _("failed to download %s\n"), url); return(NULL); diff --git a/lib/libalpm/dload.h b/lib/libalpm/dload.h index 64b57d2..ee80024 100644 --- a/lib/libalpm/dload.h +++ b/lib/libalpm/dload.h @@ -25,11 +25,11 @@ #include <time.h> -#define PM_DLBUF_LEN (1024 * 10) +#define PM_DLBUF_LEN (1024 * 16) int _alpm_download_single_file(const char *filename, alpm_list_t *servers, const char *localpath, - time_t mtimeold, time_t *mtimenew); + int force); int _alpm_download_files(alpm_list_t *files, alpm_list_t *servers, const char *localpath); diff --git a/src/pacman/pacman.c b/src/pacman/pacman.c index f4f8044..3fe9093 100644 --- a/src/pacman/pacman.c +++ b/src/pacman/pacman.c @@ -34,6 +34,7 @@ #include <signal.h> #include <unistd.h> #include <sys/types.h> +#include <sys/stat.h> #include <sys/utsname.h> /* uname */ #include <locale.h> /* setlocale */ #include <time.h> /* time_t */ @@ -631,10 +632,11 @@ static char *get_tempfile(const char *path, const char *filename) { /** External fetch callback */ int download_with_xfercommand(const char *url, const char *localpath, - time_t mtimeold, time_t *mtimenew) { + int force) { int ret = 0; int retval; int usepart = 0; + struct stat st; char *parsedcmd,*tempcmd; char cwd[PATH_MAX]; char *destfile, *tempfile, *filename; @@ -650,6 +652,13 @@ int download_with_xfercommand(const char *url, const char *localpath, destfile = get_destfile(localpath, filename); tempfile = get_tempfile(localpath, filename); + if(force && stat(tempfile, &st) == 0) { + unlink(tempfile); + } + if(force && stat(destfile, &st) == 0) { + unlink(destfile); + } + tempcmd = strdup(config->xfercommand); /* replace all occurrences of %o with fn.part */ if(strstr(tempcmd, "%o")) { -- 1.6.5.2

4 8

[pacman-dev] [PATCH] Refactor do-while cycle in the api, util.c
by Laszlo Papp 16 Nov '09

16 Nov '09

* It makes the code clearer to read/understand * Cppcheck tool doesn't show this anymore: [./util.c:215]: (error) Resource leak: fd --- lib/libalpm/util.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/libalpm/util.c b/lib/libalpm/util.c index cf2d623..f54b2bf 100644 --- a/lib/libalpm/util.c +++ b/lib/libalpm/util.c @@ -212,8 +212,9 @@ int _alpm_lckmk() _alpm_makepath(dir); FREE(dir); - while((fd = open(file, O_WRONLY | O_CREAT | O_EXCL, 0000)) == -1 - && errno == EINTR); + do { + fd = open(file, O_WRONLY | O_CREAT | O_EXCL, 0000); + } while ( fd == -1 && errno == EINTR); if(fd > 0) { FILE *f = fdopen(fd, "w"); fprintf(f, "%ld\n", (long)getpid()); @@ -315,7 +316,7 @@ int _alpm_unpack(const char *archive, const char *prefix, alpm_list_t *list, int st = archive_entry_stat(entry); entryname = archive_entry_pathname(entry); - + if(S_ISREG(st->st_mode)) { archive_entry_set_perm(entry, 0644); } else if(S_ISDIR(st->st_mode)) { -- 1.6.5.1

4 6

[pacman-dev] [GIT] The official pacman repository branch, master, updated. v3.3.3-98-g96b3430
by dan＠archlinux.org 16 Nov '09

16 Nov '09

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "The official pacman repository". The branch, master has been updated via 96b34308fe6aed5ef002c3769639ad33dac923c7 (commit) from 8a8dfc9d55c6045d59bed6cde4ae6e2307b335ea (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 96b34308fe6aed5ef002c3769639ad33dac923c7 Author: Dan McGee <dan(a)archlinux.org> Date: Sun Nov 15 21:22:01 2009 -0600 Fix syntax error in new translation Signed-off-by: Dan McGee <dan(a)archlinux.org> ----------------------------------------------------------------------- Summary of changes: po/ca.po | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) hooks/post-receive -- The official pacman repository

1 0

[pacman-dev] [GIT] The official pacman repository branch, master, updated. v3.3.3-97-g8a8dfc9
by dan＠archlinux.org 16 Nov '09

16 Nov '09

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "The official pacman repository". The branch, master has been updated via 8a8dfc9d55c6045d59bed6cde4ae6e2307b335ea (commit) via d2dbb04a9af7a18da217fb21b9152626c43461da (commit) via be266b43647ef57632d7bcfd07a4441f737b5aed (commit) from 120cd312e49b9fbb844dd36c71fe1f2e2910b707 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 8a8dfc9d55c6045d59bed6cde4ae6e2307b335ea Author: Dan McGee <dan(a)archlinux.org> Date: Sun Nov 15 20:21:50 2009 -0600 HACKING: add some notes about valgrind/gdb usage Signed-off-by: Dan McGee <dan(a)archlinux.org> commit d2dbb04a9af7a18da217fb21b9152626c43461da Author: Dan McGee <dan(a)archlinux.org> Date: Wed Nov 11 23:39:26 2009 -0600 download: major refactor to address lingering issues Sorry for this being such a huge patch, but I believe it is necessary for quite a few reasons which I will attempt to explain herein. I've been mulling this over for a while, but wasn't super happy with making the download interface more complex. Instead, if we carefully order things in the internal download code, we can actually make the interface simpler. 1. FS#15657 - This involves `name.db.tar.gz.part` files being left around the filesystem, and then causing all sorts of issues when someone attempts to rerun the operation they canceled. We need to ensure that if we resume a download, we are resuming it on exactly the same file; if we cannot be almost postive of that then we need to start over. 2. http://www.mail-archive.com/pacman-dev@archlinux.org/msg03536.html - Here we have a lighttpd bug to ruin the day. If we send both a Range: header and If-Modified-Since: header across the wire in a GET request, lighttpd doesn't do what we want in several cases. If the file hadn't been modified, it returns a '304 Not Modified' instead of a '206 Partial Content'. We need to do a stat (e.g. HEAD in HTTP terms) operation here, and the proceed accordingly based off the values we get back from it. 3. The mtime stuff was rather ugly, and relied on the called function to write back to a passed in reference, which isn't the greatest. Instead, use the power of the filesystem to contain this info. Every file downloaded internally is now carefully timestamped with the remote file time. This should allow the resume logic to work. In order to guarantee this, we need to implement a signal handler that catches interrupts, notifies the running code, and causes it to set the mtimes on the file. It then rethrows the signal so the pacman signal handler (or any frontend) works as expected. 4. We did a lot of funky stuff in trying to track the DB last modified time. It is a lot easier to just keep the downloaded DB file around and track the time on that rather than in a funky dot file. It also kills a lot of code. 5. For GPG verification of the databases down the road, we are going to need the DB file around for at least a short bit of time anyway, so this gets us closer to that. Signed-off-by: Dan McGee <dan(a)archlinux.org> [Xav: fixed printf with off_t] Signed-off-by: Xavier Chantry <shiningxc(a)gmail.com> commit be266b43647ef57632d7bcfd07a4441f737b5aed Author: Laszlo Papp <djszapi2(a)gmail.com> Date: Fri Nov 13 00:59:34 2009 +0100 Refactor do/while cycle and multiple while cycles * It makes the code clearer to read/understand * Cppcheck tool doesn't show this anymore: [./util.c:215]: (error) Resource leak: fd [Dan: don't change the coding style] Signed-off-by: Dan McGee <dan(a)archlinux.org> ----------------------------------------------------------------------- Summary of changes: HACKING | 13 ++++ lib/libalpm/alpm.h | 8 +-- lib/libalpm/be_files.c | 96 ++------------------------- lib/libalpm/dload.c | 177 +++++++++++++++++++++++++++++++++--------------- lib/libalpm/dload.h | 4 +- lib/libalpm/trans.c | 5 +- lib/libalpm/util.c | 11 ++- src/pacman/pacman.c | 15 ++++- 8 files changed, 168 insertions(+), 161 deletions(-) hooks/post-receive -- The official pacman repository

1 0

[pacman-dev] [GIT] The official pacman repository branch, master, updated. v3.3.3-94-g120cd31
by dan＠archlinux.org 16 Nov '09

16 Nov '09

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "The official pacman repository". The branch, master has been updated via 120cd312e49b9fbb844dd36c71fe1f2e2910b707 (commit) via 4d2ec3751c6e7ee47cdb784ed5cab0992f03c9dc (commit) via 564352c4a2d12050d7b4c386e972976a24f1f993 (commit) via 9c34dfd908480baed06bb2e618d8899fb8728dbd (commit) via c2999619d2af0c4fa3cbee13a4f12d95c6ab059e (commit) via 966c815881350e90628a6825c4c51bc77428c0d5 (commit) via 5d5070f47d6542a530c8cbe954bb331389b7393f (commit) via fb310fc01ed752b4e046138c3695f5482ca54e16 (commit) via 77023978c568dbb912f9c103cbe14d32153c6f89 (commit) via 77e84bea71e3dfe87a8009e7e7a913d1a12b7585 (commit) via 0199a7ee717f0848e336fda8dc1f5b9364ce3478 (commit) via 88706168f2fb9ccd91f97b4647b715cc9076d5ed (commit) from 7ae15768e5558def8bdf88fe6c5c49fa6c9b08ad (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 120cd312e49b9fbb844dd36c71fe1f2e2910b707 Author: Eric Bélanger <snowmaniscool(a)gmail.com> Date: Tue Nov 3 08:47:18 2009 -0500 makepkg: Fixed logging for split packages Signed-off-by: Eric Bélanger <snowmaniscool(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 4d2ec3751c6e7ee47cdb784ed5cab0992f03c9dc Author: Cedric Staniewski <cedric(a)gmx.ca> Date: Tue Oct 27 12:56:13 2009 +0100 makepkg: allow the use of only a package() function For some packages, generally the 'any' arch ones, a build step is not required and therefore can be skipped. In these cases, a package() function without a build() one is sufficient. As a side effect, this commit makes meta packages without any function at all in the PKGBUILD possible. Fixes FS#15147. Signed-off-by: Cedric Staniewski <cedric(a)gmx.ca> Signed-off-by: Allan McRae <allan(a)archlinux.org> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 564352c4a2d12050d7b4c386e972976a24f1f993 Author: Cedric Staniewski <cedric(a)gmx.ca> Date: Mon Oct 26 13:21:43 2009 +0100 makepkg: extend test for hyphen prefixes to pkgbase and all pkgnames Since commit fb97d32, which brought in this test, support for split PKGBUILDs was added, and therefore, all values of pkgname and also pkgbase have to be checked now. Signed-off-by: Cedric Staniewski <cedric(a)gmx.ca> Signed-off-by: Allan McRae <allan(a)archlinux.org> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 9c34dfd908480baed06bb2e618d8899fb8728dbd Author: Allan McRae <allan(a)archlinux.org> Date: Sun Oct 25 14:33:47 2009 +1000 makepkg: Add fallback to package function makepkg looks for a package() function when building a single package but package_$pkgname() style package functions when building a split package. This patch allows the use of a package_$pkgname() function when building a single package for consistency. This is achieved by having makepkg consider a non-split package with a package_$pkgname() function as a split package (creating just the one package). Fixes FS#16622. Signed-off-by: Allan McRae <allan(a)archlinux.org> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit c2999619d2af0c4fa3cbee13a4f12d95c6ab059e Author: Isaac Good <pacman(a)isaac.otherinbox.com> Date: Thu Nov 12 15:09:05 2009 -0500 makepkg: use bash test operators, part two * FS#16623, second half of makepkg * Includes stuff like -o to ||, -a to &&, etc. * if [ $(type ... preserved due to a bash bug with [[ and set -e and ERR traps Signed-off-by: Isaac Good <pacman(a)isaac.otherinbox.com> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 966c815881350e90628a6825c4c51bc77428c0d5 Author: Isaac Good <pacman(a)isaac.otherinbox.com> Date: Thu Nov 12 15:07:34 2009 -0500 makepkg: use bash test operators, part one * FS#16623, first half of makepkg * Includes stuff like -o to ||, -a to &&, etc. * if [ $(type ... preserved due to a bash bug with [[ and set -e and ERR traps Signed-off-by: Isaac Good <pacman(a)isaac.otherinbox.com> [Dan: made commit message useful] Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 5d5070f47d6542a530c8cbe954bb331389b7393f Author: Cedric Staniewski <cedric(a)gmx.ca> Date: Thu Nov 5 16:55:48 2009 +0100 scripts: replace test builtin [ with shell keywords [[ and (( FS#16623 suggested this change for makepkg; this patch applies it to the remaining files in the scripts directory. Signed-off-by: Cedric Staniewski <cedric(a)gmx.ca> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit fb310fc01ed752b4e046138c3695f5482ca54e16 Author: Xavier Chantry <shiningxc(a)gmail.com> Date: Sat Nov 14 17:12:06 2009 +0100 pacman.conf : enable resuming for curl Signed-off-by: Xavier Chantry <shiningxc(a)gmail.com> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 77023978c568dbb912f9c103cbe14d32153c6f89 Author: Eric Bélanger <snowmaniscool(a)gmail.com> Date: Fri Nov 6 10:16:49 2009 -0500 makepkg: Clarified error message when a sourceball exist already Signed-off-by: Eric Bélanger <snowmaniscool(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 77e84bea71e3dfe87a8009e7e7a913d1a12b7585 Author: Cedric Staniewski <cedric(a)gmx.ca> Date: Tue Nov 10 20:47:09 2009 +0100 makepkg: fix abortion after sourcing /etc/profile The source command triggers / might trigger the ERR trap which makes makepkg abort right after a successful installation of missing dependencies. Thanks to Xavier Chantry <shiningxc(a)gmail.com> for finding this solution. Signed-off-by: Cedric Staniewski <cedric(a)gmx.ca> Signed-off-by: Xavier Chantry <shiningxc(a)gmail.com> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 0199a7ee717f0848e336fda8dc1f5b9364ce3478 Author: Allan McRae <allan(a)archlinux.org> Date: Sun Nov 15 20:24:24 2009 +1000 contrib/bacman: fix checking if file has been added Fixes FS#17140. Signed-off-by: Allan McRae <allan(a)archlinux.org> Signed-off-by: Dan McGee <dan(a)archlinux.org> commit 88706168f2fb9ccd91f97b4647b715cc9076d5ed Author: Manuel Tortosa <manutortosa(a)gmail.com> Date: Mon Nov 16 00:29:16 2009 +0100 Add Catalan translation Signed-off-by: Dan McGee <dan(a)archlinux.org> ----------------------------------------------------------------------- Summary of changes: contrib/bacman | 4 +- doc/PKGBUILD.5.txt | 12 +- etc/pacman.conf.in | 2 +- lib/libalpm/po/LINGUAS | 1 + lib/libalpm/po/{es.po => ca.po} | 306 ++++---- po/LINGUAS | 1 + po/ca.po | 1755 +++++++++++++++++++++++++++++++++++++++ scripts/makepkg.sh.in | 465 ++++++----- scripts/pacman-optimize.sh.in | 22 +- scripts/pkgdelta.sh.in | 22 +- scripts/rankmirrors.sh.in | 12 +- scripts/repo-add.sh.in | 74 +- 12 files changed, 2225 insertions(+), 451 deletions(-) copy lib/libalpm/po/{es.po => ca.po} (51%) create mode 100644 po/ca.po hooks/post-receive -- The official pacman repository

1 0

[pacman-dev] [translation] Catalan translation patch for Pacman
by Manuel Tortosa 16 Nov '09

16 Nov '09

Hi, here's the patch to add the Catalan language to Pacman. it includes both modofications to LINGUAS files + both ca.po files (pacman+libalpm) Tanks. Manuel Tortosa manutortosa(a)gmail.com

2 1

[pacman-dev] Another proposal for the "-Sy libjpeg" problem
by Nagy Gabor 15 Nov '09

15 Nov '09

Hi! I think the problem is known for everyone, if a user upgrades a package (without -Su), he can easily get "error while loading shared libraries". The most perfect (but maybe overkill) solution is Florian's one: http://www.archlinux.org/pipermail/pacman-dev/2009-August/009158.html (that patch needs some work on optdepends side, but that is not so complicated.) I mention a simpler one: Add a new switch to upgrade reversedeps too (recursively), ie. "pacman -S --reversedeps libjpeg" would upgrade all packages depend on (the local) libjpeg. This should be straightforward to implement and imho this is a bit more sophisticated than the "pacman -Syu libjpeg" request (see bugs.archlinux.org :-P). Opinions? Bye

3 2