March 2021 - Pacman-dev - lists.archlinux.org

[pacman-dev] [PATCH] makepkg: --source should download repos with PGP signatures
by Eli Schwartz 08 Apr '21

08 Apr '21

We optimize this out for sourceballs since VCS sources don't get their checksums verified. But this logic is broken ever since we implemented PGP signature checking for git sources -- if the git source is signed, we still check it, but we don't make sure to download it first. makepkg then fails to generate a sourceball unless you previously ran --verifysource or attempted to build. Signed-off-by: Eli Schwartz <eschwartz(a)archlinux.org> --- scripts/libmakepkg/source.sh.in | 5 ++++- scripts/libmakepkg/source/git.sh.in | 9 ++++++--- scripts/makepkg.sh.in | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/libmakepkg/source.sh.in b/scripts/libmakepkg/source.sh.in index a0c6b662..b95e6be8 100644 --- a/scripts/libmakepkg/source.sh.in +++ b/scripts/libmakepkg/source.sh.in @@ -35,7 +35,7 @@ done download_sources() { local netfile all_sources - local get_source_fn=get_all_sources_for_arch get_vcs=1 + local get_source_fn=get_all_sources_for_arch get_vcs=1 get_pgp=0 msg "$(gettext "Retrieving sources...")" @@ -47,6 +47,9 @@ download_sources() { novcs) get_vcs=0 ;; + getpgp) + (( SKIPPGPCHECK )) || get_pgp=1 + ;; *) break ;; diff --git a/scripts/libmakepkg/source/git.sh.in b/scripts/libmakepkg/source/git.sh.in index 7d191b8d..d090f14e 100644 --- a/scripts/libmakepkg/source/git.sh.in +++ b/scripts/libmakepkg/source/git.sh.in @@ -29,13 +29,16 @@ source "$LIBRARY/util/pkgbuild.sh" download_git() { + local netfile=$1 + local query=$(get_uri_query "$netfile") + # abort early if parent says not to fetch if declare -p get_vcs > /dev/null 2>&1; then - (( get_vcs )) || return + if (( ! get_pgp )) || [[ $query != signed ]]; then + (( get_vcs )) || return + fi fi - local netfile=$1 - local dir=$(get_filepath "$netfile") [[ -z "$dir" ]] && dir="$SRCDEST/$(get_filename "$netfile")" diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in index 7e8d6805..c9940f0a 100644 --- a/scripts/makepkg.sh.in +++ b/scripts/makepkg.sh.in @@ -1416,7 +1416,7 @@ if (( SOURCEONLY )); then download_sources allarch elif ( (( ! SKIPCHECKSUMS )) || \ ( (( ! SKIPPGPCHECK )) && source_has_signatures ) ); then - download_sources allarch novcs + download_sources allarch novcs getpgp fi check_source_integrity all cd_safe "$startdir" -- 2.28.0

2 1

[pacman-dev] [PATCH 1/2] skip servers with too many errors
by Andrew Gregory 07 Apr '21

07 Apr '21

Keep track of errors from servers so that bad ones can be skipped once a threshold is reached. Key the error tracking off the hostname because hosts may serve multiple repos under different url's and errors are likely to be host-wide. Implements: FS#29293. Signed-off-by: Andrew Gregory <andrew.gregory.8(a)gmail.com> --- Changes from 1st draft: * error list is now tracked on the handle, i.e. servers blacklisted during -y will still be ignored during -u * a warning message is printed when a server is ignored * error tracking keys off the hostname I have not removed the individual error messages. I agree with Emil that those messages provide useful information to the user about what, if anything, they need to do about their mirrorlist. I think a small number of similar error messages is worth the utility they add. lib/libalpm/alpm.c | 1 + lib/libalpm/dload.c | 89 ++++++++++++++++++++++++++++++++++++++++++-- lib/libalpm/handle.h | 1 + 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/lib/libalpm/alpm.c b/lib/libalpm/alpm.c index f8298454..6708e202 100644 --- a/lib/libalpm/alpm.c +++ b/lib/libalpm/alpm.c @@ -114,6 +114,7 @@ int SYMEXPORT alpm_release(alpm_handle_t *myhandle) #ifdef HAVE_LIBCURL curl_multi_cleanup(myhandle->curlm); curl_global_cleanup(); + FREELIST(myhandle->server_errors); #endif _alpm_handle_unlock(myhandle); diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 0d0936c7..1318bde2 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -51,8 +51,83 @@ #ifdef HAVE_LIBCURL +/* RFC1123 states applications should support this length */ +#define HOSTNAME_SIZE 256 + static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, struct dload_payload *payload, const char *localpath); +static int curl_gethost(const char *url, char *buffer, size_t buf_len); + +/* number of "soft" errors required to blacklist a server, set to 0 to disable + * server blacklisting */ +const unsigned int server_error_limit = 3; + +struct server_error_count { + char server[HOSTNAME_SIZE]; + unsigned int errors; +}; + +static struct server_error_count *find_server_errors(alpm_handle_t *handle, const char *server) +{ + alpm_list_t *i; + struct server_error_count *h; + char hostname[HOSTNAME_SIZE]; + /* key off the hostname because a host may serve multiple repos under + * different url's and errors are likely to be host-wide */ + if(curl_gethost(server, hostname, sizeof(hostname)) != 0) { + return NULL; + } + for(i = handle->server_errors; i; i = i->next) { + h = i->data; + if(strcmp(hostname, h->server) == 0) { + return h; + } + } + if((h = calloc(sizeof(struct server_error_count), 1)) + && alpm_list_append(&handle->server_errors, h)) { + strcpy(h->server, hostname); + h->errors = 0; + return h; + } else { + free(h); + return NULL; + } +} + +static int should_skip_server(alpm_handle_t *handle, const char *server) +{ + struct server_error_count *h; + if(server_error_limit && (h = find_server_errors(handle, server)) ) { + return h->errors >= server_error_limit; + } + return 0; +} + +static void server_increment_error(alpm_handle_t *handle, const char *server, int count) +{ + struct server_error_count *h; + if(server_error_limit + && (h = find_server_errors(handle, server)) + && !should_skip_server(handle, server) ) { + h->errors += count; + + if(should_skip_server(handle, server)) { + _alpm_log(handle, ALPM_LOG_WARNING, + _("too many errors from %s, skipping for the remainder of this transaction\n"), + h->server); + } + } +} + +static void server_soft_error(alpm_handle_t *handle, const char *server) +{ + server_increment_error(handle, server, 1); +} + +static void server_hard_error(alpm_handle_t *handle, const char *server) +{ + server_increment_error(handle, server, server_error_limit); +} static const char *get_filename(const char *url) { @@ -332,9 +407,6 @@ static FILE *create_tempfile(struct dload_payload *payload, const char *localpat return fp; } -/* RFC1123 states applications should support this length */ -#define HOSTNAME_SIZE 256 - /* Return 0 if retry was successful, -1 otherwise */ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload *payload) { @@ -342,7 +414,7 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload size_t len; alpm_handle_t *handle = payload->handle; - if(payload->servers) { + while(payload->servers && should_skip_server(handle, payload->servers->data)) { payload->servers = payload->servers->next; } if(!payload->servers) { @@ -351,6 +423,7 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload return -1; } server = payload->servers->data; + payload->servers = payload->servers->next; /* regenerate a new fileurl */ FREE(payload->fileurl); @@ -423,6 +496,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, _alpm_log(handle, ALPM_LOG_ERROR, _("failed retrieving file '%s' from %s : %s\n"), payload->remote_name, hostname, payload->error_buffer); + server_soft_error(handle, payload->fileurl); } if(curl_retry_next_server(curlm, curl, payload) == 0) { return 2; @@ -440,6 +514,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, _alpm_log(handle, ALPM_LOG_ERROR, _("failed retrieving file '%s' from %s : expected download size exceeded\n"), payload->remote_name, hostname); + server_soft_error(handle, payload->fileurl); } goto cleanup; case CURLE_COULDNT_RESOLVE_HOST: @@ -448,6 +523,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, _alpm_log(handle, ALPM_LOG_ERROR, _("failed retrieving file '%s' from %s : %s\n"), payload->remote_name, hostname, payload->error_buffer); + server_hard_error(handle, payload->fileurl); if(curl_retry_next_server(curlm, curl, payload) == 0) { return 2; } else { @@ -463,6 +539,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, _alpm_log(handle, ALPM_LOG_ERROR, _("failed retrieving file '%s' from %s : %s\n"), payload->remote_name, hostname, payload->error_buffer); + server_soft_error(handle, payload->fileurl); } else { _alpm_log(handle, ALPM_LOG_DEBUG, "failed retrieving file '%s' from %s : %s\n", @@ -632,11 +709,15 @@ static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, ASSERT(!payload->filepath, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup)); } else { const char *server; + while(payload->servers && should_skip_server(handle, payload->servers->data)) { + payload->servers = payload->servers->next; + } ASSERT(payload->servers, GOTO_ERR(handle, ALPM_ERR_SERVER_NONE, cleanup)); ASSERT(payload->filepath, GOTO_ERR(handle, ALPM_ERR_WRONG_ARGS, cleanup)); server = payload->servers->data; + payload->servers = payload->servers->next; len = strlen(server) + strlen(payload->filepath) + 2; MALLOC(payload->fileurl, len, GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h index 349ef2c7..fdd8e6e7 100644 --- a/lib/libalpm/handle.h +++ b/lib/libalpm/handle.h @@ -60,6 +60,7 @@ struct __alpm_handle_t { #ifdef HAVE_LIBCURL /* libcurl handle */ CURLM *curlm; + alpm_list_t *server_errors; #endif unsigned short disable_dl_timeout; -- 2.31.1

2 1

[pacman-dev] [PATCH] makepkg: add PACMAN_AUTH configurable setting for sudo elevation
by Eli Schwartz 06 Apr '21

06 Apr '21

If specified, this will be used no matter what. If not, then we check if sudo exists and use that, or else fall back on su. Implements FS#32621 Signed-off-by: Eli Schwartz <eschwartz(a)archlinux.org> --- doc/makepkg.conf.5.asciidoc | 8 ++++++++ etc/makepkg.conf.in | 7 +++++++ scripts/makepkg.sh.in | 13 ++++++++++--- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/doc/makepkg.conf.5.asciidoc b/doc/makepkg.conf.5.asciidoc index 2c7a54dbf..398529158 100644 --- a/doc/makepkg.conf.5.asciidoc +++ b/doc/makepkg.conf.5.asciidoc @@ -278,6 +278,14 @@ Options `.tar.lzo`, `.tar.lrz`, `.tar.lz4`, `.tar.lz` and `.tar.Z`, or simply `.tar` to disable compression entirely. +**PACMAN_AUTH=()**:: + Specify a command prefix for running pacman as root. If unset, makepkg will + check for the presence of sudo(8) and su(1) in turn, and try the first one + it finds. + + + If present, `%q` will be replaced with the shell-quoted form of the command + to run. Otherwise, the command to run is appended to the auth command. + See Also diff --git a/etc/makepkg.conf.in b/etc/makepkg.conf.in index 43a69df66..fff5b8eb2 100644 --- a/etc/makepkg.conf.in +++ b/etc/makepkg.conf.in @@ -147,3 +147,10 @@ COMPRESSLZ=(lzip -c -f) # PKGEXT='@PKGEXT@' SRCEXT='@SRCEXT@' + +######################################################################### +# OTHER +######################################################################### +# +#-- Command used to run pacman as root, instead of trying sudo and su +PACMAN_AUTH=() diff --git a/scripts/makepkg.sh.in b/scripts/makepkg.sh.in index f4a2de7d4..a0cd1a4fb 100644 --- a/scripts/makepkg.sh.in +++ b/scripts/makepkg.sh.in @@ -225,15 +225,22 @@ missing_source_file() { } run_pacman() { - local cmd + local cmd cmdescape if [[ $1 = -@(T|Q)*([[:alpha:]]) ]]; then cmd=("$PACMAN_PATH" "$@") else cmd=("$PACMAN_PATH" "${PACMAN_OPTS[@]}" "$@") - if type -p sudo >/dev/null; then + cmdescape="$(printf '%q ' "${cmd[@]}")" + if (( ${#PACMAN_AUTH[@]} )); then + if in_array '%q' "${PACMAN_AUTH[@]}"; then + cmd=("${PACMAN_AUTH[@]/\%q/$cmdescape}") + else + cmd=("${PACMAN_AUTH[@]}" "${cmd[@]}") + fi + elif type -p sudo >/dev/null; then cmd=(sudo "${cmd[@]}") else - cmd=(su root -c "$(printf '%q ' "${cmd[@]}")") + cmd=(su root -c "$cmdescape") fi local lockfile="$(pacman-conf DBPath)/db.lck" while [[ -f $lockfile ]]; do -- 2.31.0

3 4

[pacman-dev] [Proposal] make "makepkg --holdver" preserve the $pkgver, too
by Erich Eckner 30 Mar '21

30 Mar '21

-----BEGIN PGP SIGNED MESSAGE----- Hash: SHA256 Hi, I'd like to propose to let makepkg not only hold the version of the VCS sources, but also the $pkgver itself, when --holdver is given (that is: call pkgver() only, whenn --holdver is not given). In case, there is genuine need for keeping VCS versions, but still updating $pkgver via pkgver(), a different option name could be chosen (--holdpkgver ?). Motivation: This would be useful for building older packages (e.g. from git checkouts) which offer a pkgver() function. ("older" can simply mean, that the source is updated faster than the package is being checked into git) Is anyone else finding this useful? Opinions? If we settle on something, I can look into how to implement it, myself (but I don't want to line up at the front :-D). regards, Erich -----BEGIN PGP SIGNATURE----- iQIzBAEBCAAdFiEE3p92iMrPBP64GmxZCu7JB1Xae1oFAmBi1JgACgkQCu7JB1Xa e1qIkxAAnLoodUckvSYOaA9fouDO2ZjfhztixUfRAvELT5Refpn8Frh0PAEQNAt1 4JUUEPqL89nSd0CQe03IOEr1SQ6+8xYPaRhJLTG37rxcP+uZbE3gokz/Jin0jUn5 3L6uXQzqnaJWstg7Wz17/SSjhwIpxSDj5iYAV34IoasFSg+sLSFyRYiJ3399aFyc MvDrTqI1GjRvCoa6a/PYG4h+9YmQJfIr7vTZ19ZPpZKrz/HilwowPRNcvqFyj6si A4yuMLjxWFOyo3ry+FngTTzE0tIYpp1FoQJKg1iR7Ll8rbz4SZnModCmojIsmkqs eog3PKil1V25OYjingIge4Fz8B+qPJcA8Fne/JNGNjEo8vqI/JnD4FsZ/JwS1ajg A4BJs/3xWJPQ7eg2LLfD8PKQ6wrmExhaG96iv9Q/zonvuFBU7sen8aVuFWsC30vp 8kJcWvBTTQoB+L6Q8VUng8zFf42UGxSPugtCzyrvKHA3onHfnwH06mEOpI4P4gCs NpI9cpbGoSAny24qU8pxknKOSSvdGX1xBojMLeHgVhaTEY3Oonht16QrP+t4IOVc vDTI97A/qOqQK/kmCBEFPvk0+Tfk1W2e6/BEE9lDiMC7EKEWGyFmalGJIIUIEKeH tsywoa8VNDR2j1jr6X2wT1G7uAko8kJW6cK+WkZ/wS9EEQ9DnEA= =6sJj -----END PGP SIGNATURE-----

1 1

[pacman-dev] [PATCH 2/2] include retries and signatures in download count
by Andrew Gregory 28 Mar '21

28 Mar '21

Fixes: FS#69881 Signed-off-by: Andrew Gregory <andrew.gregory.8(a)gmail.com> --- lib/libalpm/dload.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 1318bde2..2183a3e2 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -456,7 +456,7 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload * Returns -2 if an error happened for an optional file */ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, - const char *localpath) + const char *localpath, int *active_downloads_num) { alpm_handle_t *handle = NULL; struct dload_payload *payload = NULL; @@ -499,6 +499,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, server_soft_error(handle, payload->fileurl); } if(curl_retry_next_server(curlm, curl, payload) == 0) { + (*active_downloads_num)++; return 2; } else { goto cleanup; @@ -525,6 +526,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, payload->remote_name, hostname, payload->error_buffer); server_hard_error(handle, payload->fileurl); if(curl_retry_next_server(curlm, curl, payload) == 0) { + (*active_downloads_num)++; return 2; } else { goto cleanup; @@ -546,6 +548,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, payload->remote_name, hostname, payload->error_buffer); } if(curl_retry_next_server(curlm, curl, payload) == 0) { + (*active_downloads_num)++; return 2; } else { goto cleanup; @@ -613,6 +616,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, sig->max_size = 16 * 1024; curl_add_payload(handle, curlm, sig, localpath); + (*active_downloads_num)++; } /* time condition was met and we didn't download anything. we need to @@ -789,12 +793,11 @@ static int curl_download_internal(alpm_handle_t *handle, const char *localpath) { int active_downloads_num = 0; - bool recheck_downloads = false; int err = 0; int max_streams = handle->parallel_downloads; CURLM *curlm = handle->curlm; - while(active_downloads_num > 0 || payloads || recheck_downloads) { + while(active_downloads_num > 0 || payloads) { CURLMcode mc; for(; active_downloads_num < max_streams && payloads; active_downloads_num++) { @@ -828,7 +831,6 @@ static int curl_download_internal(alpm_handle_t *handle, err = -1; } - recheck_downloads = false; while(true) { int msgs_left = 0; CURLMsg *msg = curl_multi_info_read(curlm, &msgs_left); @@ -836,7 +838,8 @@ static int curl_download_internal(alpm_handle_t *handle, break; } if(msg->msg == CURLMSG_DONE) { - int ret = curl_check_finished_download(curlm, msg, localpath); + int ret = curl_check_finished_download(curlm, msg, + localpath, &active_downloads_num); if(ret == -1) { /* if current payload failed to download then stop adding new payloads but wait for the * current ones @@ -844,11 +847,6 @@ static int curl_download_internal(alpm_handle_t *handle, payloads = NULL; err = -1; } - /* curl_multi_check_finished_download() might add more payloads e.g. in case of a retry - * from the next mirror. We need to execute curl_multi_perform() at least one more time - * to make sure new payload requests are processed. - */ - recheck_downloads = true; } else { _alpm_log(handle, ALPM_LOG_ERROR, _("curl transfer error: %d\n"), msg->msg); } -- 2.31.1

1 0

[pacman-dev] [PATCH] libmakepkg: fix detection of file names with future gcc versions
by Allan McRae 27 Mar '21

27 Mar '21

The current gcc build from git master give different output from readelf: gcc-10.2.0 $ readelf "hello" --debug-dump | grep hello <11> DW_AT_name : (indirect string, offset: 0xbfc): hello.cpp gcc-git $ readelf "hello" --debug-dump | grep hello <12> DW_AT_name : (indirect line string, offset: 0x0): hello.cpp This causes the awk statement extracting the file name to fail as it relied on the information being in the 8th field. Instead, extract the information from the final field. Fixes FS#70168 Signed-off-by: Allan McRae <allan(a)archlinux.org> --- scripts/libmakepkg/tidy/strip.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/libmakepkg/tidy/strip.sh.in b/scripts/libmakepkg/tidy/strip.sh.in index ceb2a108..737c5115 100644 --- a/scripts/libmakepkg/tidy/strip.sh.in +++ b/scripts/libmakepkg/tidy/strip.sh.in @@ -37,7 +37,7 @@ build_id() { source_files() { LANG=C readelf "$1" --debug-dump | \ - awk '/DW_AT_name +:/{name=$8}/DW_AT_comp_dir +:/{{if (name == "<artificial>") next}{if (name !~ /^[<\/]/) {printf "%s/", $8}}{print name}}' + awk '/DW_AT_name +:/{name=$NF}/DW_AT_comp_dir +:/{{if (name == "<artificial>") next}{if (name !~ /^[<\/]/) {printf "%s/", $NF}}{print name}}' } strip_file() { -- 2.31.0

1 0

[pacman-dev] [GIT] The official pacman repository branch, master, updated. v6.0.0alpha1-78-g1e60a5f0
by Allan McRae 25 Mar '21

25 Mar '21

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "The official pacman repository". The branch, master has been updated via 1e60a5f00674a9a9b0c530c741bd6701d0e1e834 (commit) via 9bf3d6a7603bf48771a20d1f4bd789670e7446f7 (commit) from ede73f4d898946a7e54634169b3605a024553315 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 1e60a5f00674a9a9b0c530c741bd6701d0e1e834 Author: Anatol Pomozov <anatol.pomozov(a)gmail.com> Date: Mon Mar 15 16:33:08 2021 -0700 Remove "total download" callback in favor of generic event callback Total download callback called right before packages start downloaded. But we already have an event for such event (ALPM_EVENT_PKG_RETRIEVE_START) and it is naturally to use the event to pass information about expected download size. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 9bf3d6a7603bf48771a20d1f4bd789670e7446f7 Author: Eli Schwartz <eschwartz(a)archlinux.org> Date: Sat Mar 13 22:06:00 2021 -0500 remove sed command lookup and hardcoding in edit-script.sh We should not need to hardcode the path to sed as we simply don't care. We don't check what kind of sed we found, and we're using the same one we initially found on the PATH, which is surely still on the PATH. At one point we did care to find the system copy of sed and hardcode it in makepkg, because we also passed non-portable -i options to it and makepkg needed to continue working on macOS even if some incompatible GNU sed got installed afterward, elsewhere on the PATH. But this was never relevant to the in-tree buildsystem script running sed. In commit 3a814ee6bca9ee24a868c0dc032b321048a53e08 we removed even that, so we don't need to look it up at all. Signed-off-by: Eli Schwartz <eschwartz(a)archlinux.org> Signed-off-by: Allan McRae <allan(a)archlinux.org> ----------------------------------------------------------------------- Summary of changes: README | 1 - build-aux/edit-script.sh.in | 2 +- lib/libalpm/alpm.h | 32 ++++++++++++-------------------- lib/libalpm/dload.c | 3 ++- lib/libalpm/handle.c | 13 ------------- lib/libalpm/handle.h | 1 - lib/libalpm/sync.c | 25 +++++++++---------------- meson.build | 2 -- src/pacman/callback.c | 9 ++------- src/pacman/callback.h | 2 -- src/pacman/conf.c | 4 ---- 11 files changed, 26 insertions(+), 68 deletions(-) hooks/post-receive -- The official pacman repository

2 1

[pacman-dev] [Proposal] Hold the first working mirror when downloading
by Marco Rubin 22 Mar '21

22 Mar '21

Hello, I'd like to propose a change to pacman, but I don't know if someone is already working on it for the next release. I think it would be useful if pacman could hold the first working mirror when downloading updates. Yesterday I was upgrading all my packages and, for some reason, the first five mirrors timed out after a few seconds, so every time pacman was downloading a package, it checked all the mirrors from the beginning until it found one where to grab the updated package. I had many old packages so this process quickly became a little frustrating, and to me it would have been faster if it remembered the first working mirror instead of checking all of them every time.

3 3

[pacman-dev] [PATCH] RFC: Remove "duplicate" download error messages
by Allan McRae 20 Mar '21

20 Mar '21

This is a very incomplete and hacky patch... But it does do this: allan@mando ~/arch/code/pacman (patchqueue) > pacman -Syy :: Synchronizing package databases... core 131.1 KiB 47.0 KiB/s 00:03 [######################] 100% extra 1645.7 KiB 490 KiB/s 00:03 [######################] 100% community 5.4 MiB 161 KiB/s 00:34 [######################] 100% error: failed retrieving file 'extra.db' from allanmcrae.com : The requested URL returned error: 404 error: failed retrieving file 'community.db' from allanmcrae.com : The requested URL returned error: 404 error: failed retrieving file 'core.db' from allanmcrae.com : The requested URL returned error: 404 allan@mando ~/arch/code/pacman (patchqueue) > sudo ./build/pacman -Syy [sudo] password for allan: :: Synchronizing package databases... core 131.1 KiB 51.2 KiB/s 00:03 [######################] 100% extra 1645.7 KiB 494 KiB/s 00:03 [######################] 100% community 5.4 MiB 172 KiB/s 00:32 [######################] 100% error: failed retrieving file(s) from allanmcrae.com : The requested URL returned error: 404 There are a few things to note. 1) we lose the file names for the failed downloads. And we lose information of how many downloads failed. 2) I only changed the messages for download failures so far. So (e.g.) having a file exceed its expected download size will still report the file name. 3) debug output still gives file names 4) note the "error" in the messages above, yet pacman was still successful as I have an actual mirror configured. I guess this (and others?) should be changed to warning. We get an overall error message if there is complete failure anyway. 5) I have been super lazy for this PoC, switching alpm_list_remove_dupes() to use string comparisons, rather than pointer comparisons. This will need fixed. Things to discuss: - do we need to report the number of failures from a server. Currently 1 and 200 failures are treated the same. This involves doing more than a dedup of the error list. - converting errors to warnings. They are clearly not errors until they are... Is the final line enough: error: failed to synchronize all databases (failed to retrieve some files) --- lib/libalpm/alpm_list.c | 2 +- lib/libalpm/dload.c | 12 ++++++------ src/pacman/callback.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/libalpm/alpm_list.c b/lib/libalpm/alpm_list.c index 5a684181..bc243898 100644 --- a/lib/libalpm/alpm_list.c +++ b/lib/libalpm/alpm_list.c @@ -338,7 +338,7 @@ alpm_list_t SYMEXPORT *alpm_list_remove_dupes(const alpm_list_t *list) const alpm_list_t *lp = list; alpm_list_t *newlist = NULL; while(lp) { - if(!alpm_list_find_ptr(newlist, lp->data)) { + if(!alpm_list_find_str(newlist, lp->data)) { if(alpm_list_append(&newlist, lp->data) == NULL) { alpm_list_free(newlist); return NULL; diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 0d0936c7..702bd059 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -421,8 +421,8 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, snprintf(payload->error_buffer, sizeof(payload->error_buffer), "The requested URL returned error: %ld", payload->respcode); _alpm_log(handle, ALPM_LOG_ERROR, - _("failed retrieving file '%s' from %s : %s\n"), - payload->remote_name, hostname, payload->error_buffer); + _("failed retrieving file(s) from %s : %s\n"), + hostname, payload->error_buffer); } if(curl_retry_next_server(curlm, curl, payload) == 0) { return 2; @@ -446,8 +446,8 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, payload->unlink_on_fail = 1; handle->pm_errno = ALPM_ERR_SERVER_BAD_URL; _alpm_log(handle, ALPM_LOG_ERROR, - _("failed retrieving file '%s' from %s : %s\n"), - payload->remote_name, hostname, payload->error_buffer); + _("failed retrieving file(s) from %s : %s\n"), + hostname, payload->error_buffer); if(curl_retry_next_server(curlm, curl, payload) == 0) { return 2; } else { @@ -461,8 +461,8 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, if(!payload->errors_ok) { handle->pm_errno = ALPM_ERR_LIBCURL; _alpm_log(handle, ALPM_LOG_ERROR, - _("failed retrieving file '%s' from %s : %s\n"), - payload->remote_name, hostname, payload->error_buffer); + _("failed retrieving file(s) from %s : %s\n"), + hostname, payload->error_buffer); } else { _alpm_log(handle, ALPM_LOG_DEBUG, "failed retrieving file '%s' from %s : %s\n", diff --git a/src/pacman/callback.c b/src/pacman/callback.c index 40973340..d07df816 100644 --- a/src/pacman/callback.c +++ b/src/pacman/callback.c @@ -212,9 +212,12 @@ static void fill_progress(const int bar_percent, const int disp_percent, } static void flush_output_list(void) { - alpm_list_t *i = NULL; + alpm_list_t *i = NULL, *rmdup; fflush(stdout); - for(i = output; i; i = i->next) { + + rmdup = alpm_list_remove_dupes(output); + + for(i = rmdup; i; i = i->next) { fputs((const char *)i->data, stderr); } fflush(stderr); -- 2.30.1

3 4

[pacman-dev] [PATCH v2] Remove "total download" callback in favor of generic event callback
by Anatol Pomozov 16 Mar '21

16 Mar '21

Total download callback called right before packages start downloaded. But we already have an event for such event (ALPM_EVENT_PKG_RETRIEVE_START) and it is naturally to use the event to pass information about expected download size. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- README | 1 - lib/libalpm/alpm.h | 32 ++++++++++++-------------------- lib/libalpm/dload.c | 2 +- lib/libalpm/handle.c | 13 ------------- lib/libalpm/handle.h | 1 - lib/libalpm/sync.c | 25 +++++++++---------------- src/pacman/callback.c | 9 ++------- src/pacman/callback.h | 2 -- src/pacman/conf.c | 4 ---- 9 files changed, 24 insertions(+), 65 deletions(-) diff --git a/README b/README index 6aa68374..470ccf3c 100644 --- a/README +++ b/README @@ -52,7 +52,6 @@ library is initialized. * logcb: The callback function for "log" operations. * dlcb: The callback function for download progress of each package. * fetchcb: Callback for custom download function. -* totaldlcb: The callback function for overall download progress. * eventcb: Callback for transaction messages. * questioncb: Callback for selecting amongst choices. * progresscb: Callback to handle display of transaction progress. diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 0909fe11..833df829 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -926,6 +926,16 @@ typedef struct _alpm_event_hook_run_t { size_t total; } alpm_event_hook_run_t; +/** Packages downloading about to start. */ +typedef struct _alpm_event_pkg_retrieve_t { + /** Type of event */ + alpm_event_type_t type; + /** Number of packages to download */ + size_t num; + /** Total size of packages to download */ + off_t total_size; +} alpm_event_pkg_retrieve_t; + /** Events. * This is a union passed to the callback that allows the frontend to know * which type of event was triggered (via type). It is then possible to @@ -954,6 +964,8 @@ typedef union _alpm_event_t { alpm_event_hook_t hook; /** A hook was ran */ alpm_event_hook_run_t hook_run; + /** Download packages */ + alpm_event_pkg_retrieve_t pkg_retrieve; } alpm_event_t; /** Event callback. @@ -1197,12 +1209,6 @@ typedef void (*alpm_cb_download)(const char *filename, alpm_download_event_type_t event, void *data); -/** Total Download callback. - * @param howmany the number of packages that will be downloaded during \link alpm_trans_commit \endlink. - * @param total amount that will be downloaded during \link alpm_trans_commit \endlink. - */ -typedef void (*alpm_cb_totaldl)(size_t howmany, off_t total); - /** A callback for downloading files * @param url the URL of the file to be downloaded * @param localpath the directory to which the file should be downloaded @@ -1525,20 +1531,6 @@ alpm_cb_fetch alpm_option_get_fetchcb(alpm_handle_t *handle); */ int alpm_option_set_fetchcb(alpm_handle_t *handle, alpm_cb_fetch cb); -/** Returns the callback used to report total download size. - * @param handle the context handle - * @return the currently set total download callback - */ -alpm_cb_totaldl alpm_option_get_totaldlcb(alpm_handle_t *handle); - -/** Sets the callback used to report total download size. - * @param handle the context handle - * @param cb the cb to use - * @return 0 on success, -1 on error (pm_errno is set accordingly) - */ -int alpm_option_set_totaldlcb(alpm_handle_t *handle, alpm_cb_totaldl cb); - - /** Returns the callback used for events. * @param handle the context handle * @return the currently set event callback diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 0d0936c7..0423a139 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -851,7 +851,7 @@ int SYMEXPORT alpm_fetch_pkgurl(alpm_handle_t *handle, const alpm_list_t *urls, const char *cachedir; alpm_list_t *payloads = NULL; const alpm_list_t *i; - alpm_event_t event; + alpm_event_t event = {0}; CHECK_HANDLE(handle, return -1); ASSERT(*fetched == NULL, RET_ERR(handle, ALPM_ERR_WRONG_ARGS, -1)); diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c index d1ea5886..7b8cb1da 100644 --- a/lib/libalpm/handle.c +++ b/lib/libalpm/handle.c @@ -174,12 +174,6 @@ alpm_cb_fetch SYMEXPORT alpm_option_get_fetchcb(alpm_handle_t *handle) return handle->fetchcb; } -alpm_cb_totaldl SYMEXPORT alpm_option_get_totaldlcb(alpm_handle_t *handle) -{ - CHECK_HANDLE(handle, return NULL); - return handle->totaldlcb; -} - alpm_cb_event SYMEXPORT alpm_option_get_eventcb(alpm_handle_t *handle) { CHECK_HANDLE(handle, return NULL); @@ -327,13 +321,6 @@ int SYMEXPORT alpm_option_set_fetchcb(alpm_handle_t *handle, alpm_cb_fetch cb) return 0; } -int SYMEXPORT alpm_option_set_totaldlcb(alpm_handle_t *handle, alpm_cb_totaldl cb) -{ - CHECK_HANDLE(handle, return -1); - handle->totaldlcb = cb; - return 0; -} - int SYMEXPORT alpm_option_set_eventcb(alpm_handle_t *handle, alpm_cb_event cb) { CHECK_HANDLE(handle, return -1); diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h index 349ef2c7..2ebaafb2 100644 --- a/lib/libalpm/handle.h +++ b/lib/libalpm/handle.h @@ -72,7 +72,6 @@ struct __alpm_handle_t { /* callback functions */ alpm_cb_log logcb; /* Log callback function */ alpm_cb_download dlcb; /* Download callback function */ - alpm_cb_totaldl totaldlcb; /* Total download callback function */ alpm_cb_fetch fetchcb; /* Download file callback function */ alpm_cb_event eventcb; alpm_cb_question questioncb; diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index d258b8d1..7fa50a13 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -749,7 +749,7 @@ static int download_files(alpm_handle_t *handle) const char *cachedir; alpm_list_t *i, *files = NULL; int ret = 0; - alpm_event_t event; + alpm_event_t event = {0}; alpm_list_t *payloads = NULL; cachedir = _alpm_filecache_setup(handle); @@ -760,21 +760,6 @@ static int download_files(alpm_handle_t *handle) goto finish; } - /* Total progress - figure out the total download size if required to - * pass to the callback. This function is called once, and it is up to the - * frontend to compute incremental progress. */ - if(handle->totaldlcb) { - off_t total_size = (off_t)0; - size_t howmany = 0; - /* sum up the download size for each package and store total */ - for(i = files; i; i = i->next) { - alpm_pkg_t *spkg = i->data; - total_size += spkg->download_size; - howmany++; - } - handle->totaldlcb(howmany, total_size); - } - if(files) { /* check for necessary disk space for download */ if(handle->checkspace) { @@ -800,6 +785,14 @@ static int download_files(alpm_handle_t *handle) } event.type = ALPM_EVENT_PKG_RETRIEVE_START; + + /* sum up the number of packages to download and its total size */ + for(i = files; i; i = i->next) { + alpm_pkg_t *spkg = i->data; + event.pkg_retrieve.total_size += spkg->download_size; + event.pkg_retrieve.num++; + } + EVENT(handle, &event); for(i = files; i; i = i->next) { alpm_pkg_t *pkg = i->data; diff --git a/src/pacman/callback.c b/src/pacman/callback.c index 40973340..a28a79a9 100644 --- a/src/pacman/callback.c +++ b/src/pacman/callback.c @@ -338,6 +338,8 @@ void cb_event(alpm_event_t *event) case ALPM_EVENT_PKG_RETRIEVE_START: colon_printf(_("Retrieving packages...\n")); on_progress = 1; + list_total_pkgs = event->pkg_retrieve.num; + list_total = event->pkg_retrieve.total_size; total_enabled = config->totaldownload && list_total; if(total_enabled) { init_total_progressbar(); @@ -696,13 +698,6 @@ void cb_progress(alpm_progress_t event, const char *pkgname, int percent, } } -/* callback to handle receipt of total download value */ -void cb_dl_total(size_t howmany, off_t total) -{ - list_total_pkgs = howmany; - list_total = total; -} - static int dload_progressbar_enabled(void) { return !config->noprogressbar && (getcols() != 0); diff --git a/src/pacman/callback.h b/src/pacman/callback.h index 9b5deb6f..8ac9d960 100644 --- a/src/pacman/callback.h +++ b/src/pacman/callback.h @@ -35,8 +35,6 @@ void cb_question(alpm_question_t *question); void cb_progress(alpm_progress_t event, const char *pkgname, int percent, size_t howmany, size_t remain); -/* callback to handle receipt of total download value */ -void cb_dl_total(size_t howmany, off_t total); /* callback to handle display of download progress */ void cb_download(const char *filename, alpm_download_event_type_t event, void *data); diff --git a/src/pacman/conf.c b/src/pacman/conf.c index 96e5790f..a4f2ba35 100644 --- a/src/pacman/conf.c +++ b/src/pacman/conf.c @@ -892,10 +892,6 @@ static int setup_libalpm(void) pm_printf(ALPM_LOG_WARNING, _("no '%s' configured\n"), "XferCommand"); } - if(config->totaldownload) { - alpm_option_set_totaldlcb(handle, cb_dl_total); - } - alpm_option_set_arch(handle, config->arch); alpm_option_set_checkspace(handle, config->checkspace); alpm_option_set_usesyslog(handle, config->usesyslog); -- 2.30.1

2 2