- Pacman-dev - lists.archlinux.org

[pacman-dev] [PATCH] doc/makepkg.8: Added punctuations.
by foxboron＠archlinux.org 14 Jul '20

14 Jul '20

From: Morten Linderud <morten(a)linderud.pw> Signed-off-by: Morten Linderud <morten(a)linderud.pw> --- doc/makepkg.8.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/makepkg.8.asciidoc b/doc/makepkg.8.asciidoc index 544659fc..3b5e61b3 100644 --- a/doc/makepkg.8.asciidoc +++ b/doc/makepkg.8.asciidoc @@ -274,7 +274,7 @@ Environment Variables **GPGKEY=**"keyid":: Specify a key to use when signing packages, overriding the GPGKEY setting - in linkman:makepkg.conf[5] + in linkman:makepkg.conf[5]. **SOURCE_DATE_EPOCH=**"<date>":: Used for link:https://reproducible-builds.org/docs/[Reproducible Builds]. @@ -299,7 +299,7 @@ On exit, makepkg will return one of the following error codes. Error in configuration file. 3:: - User specified an invalid option + User specified an invalid option. 4:: Error in user-supplied function in PKGBUILD. -- 2.27.0

2 1

[pacman-dev] [PATCH] Check that destfile_name exists before using it
by Anatol Pomozov 14 Jul '20

14 Jul '20

In some cases (when trust_remote_name is used for a URL without a filename and no Content-Disposition is provided by the server) destfile_name will be NULL. In this case payload data will be stored in tempfile_name and no destfile_name is set. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/dload.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 343f5c78..673e769f 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -897,15 +897,18 @@ int SYMEXPORT alpm_fetch_pkgurl(alpm_handle_t *handle, const alpm_list_t *urls, for(i = payloads; i; i = i->next) { struct dload_payload *payload = i->data; - const char *filename; char *filepath; if(payload->signature) { continue; } - filename = mbasename(payload->destfile_name); - filepath = _alpm_filecache_find(handle, filename); + if(payload->destfile_name) { + const char *filename = mbasename(payload->destfile_name); + filepath = _alpm_filecache_find(handle, filename); + } else { + STRDUP(filepath, payload->tempfile_name, GOTO_ERR(handle, ALPM_ERR_MEMORY, err)); + } if(filepath) { alpm_list_append(fetched, filepath); } else { -- 2.27.0

2 2

[pacman-dev] [PATCH] Build signature remote name based on the main payload name
by Anatol Pomozov 14 Jul '20

14 Jul '20

The main payload final name might be affected by url redirects or Content-Disposition HTTP header value. We want to make sure that accompanion *.sig filename always matches the package filename. So ignore finalname/Content-Disposition for the *.sig file. It also helps to fix a corner case when the download URL does not contain a filename and server provides Content-Disposition for the main payload but not for the signature payload. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/dload.c | 61 +++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 78808eb3..1785dd6a 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -481,6 +481,31 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &timecond); curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); + if(payload->trust_remote_name) { + if(payload->content_disp_name) { + /* content-disposition header has a better name for our file */ + free(payload->destfile_name); + payload->destfile_name = get_fullpath(localpath, + get_filename(payload->content_disp_name), ""); + } else { + const char *effective_filename = strrchr(effective_url, '/'); + + if(effective_filename && strlen(effective_filename) > 2) { + effective_filename++; + + /* if destfile was never set, we wrote to a tempfile. even if destfile is + * set, we may have followed some redirects and the effective url may + * have a better suggestion as to what to name our file. in either case, + * refactor destfile to this newly derived name. */ + if(!payload->destfile_name || strcmp(effective_filename, + strrchr(payload->destfile_name, '/') + 1) != 0) { + free(payload->destfile_name); + payload->destfile_name = get_fullpath(localpath, effective_filename, ""); + } + } + } + } + /* Let's check if client requested downloading accompanion *.sig file */ if(!payload->signature && payload->download_signature && curlerr == CURLE_OK && payload->respcode < 400) { struct dload_payload *sig = NULL; @@ -489,6 +514,18 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, CALLOC(sig, 1, sizeof(*sig), GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); MALLOC(sig->fileurl, len, FREE(sig); GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); snprintf(sig->fileurl, len, "%s.sig", effective_url); + + if(payload->trust_remote_name) { + /* In this case server might provide a new name for the main payload. + * Choose *.sig filename based on this new name. + */ + const char* realname = payload->destfile_name ? payload->destfile_name : payload->tempfile_name; + const char *final_file = get_filename(realname); + int remote_name_len = strlen(final_file) + 5; + MALLOC(sig->remote_name, remote_name_len, FREE(sig->fileurl); FREE(sig); GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); + snprintf(sig->remote_name, remote_name_len, "%s.sig", final_file); + } + sig->signature = 1; sig->handle = handle; sig->force = payload->force; @@ -520,30 +557,6 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, GOTO_ERR(handle, ALPM_ERR_RETRIEVE, cleanup); } - if(payload->trust_remote_name) { - if(payload->content_disp_name) { - /* content-disposition header has a better name for our file */ - free(payload->destfile_name); - payload->destfile_name = get_fullpath(localpath, - get_filename(payload->content_disp_name), ""); - } else { - const char *effective_filename = strrchr(effective_url, '/'); - if(effective_filename && strlen(effective_filename) > 2) { - effective_filename++; - - /* if destfile was never set, we wrote to a tempfile. even if destfile is - * set, we may have followed some redirects and the effective url may - * have a better suggestion as to what to name our file. in either case, - * refactor destfile to this newly derived name. */ - if(!payload->destfile_name || strcmp(effective_filename, - strrchr(payload->destfile_name, '/') + 1) != 0) { - free(payload->destfile_name); - payload->destfile_name = get_fullpath(localpath, effective_filename, ""); - } - } - } - } - ret = 0; cleanup: -- 2.27.0

2 1

[pacman-dev] [PATCH] Do not free payload fields in the middle of this structure use
by Anatol Pomozov 13 Jul '20

13 Jul '20

At the end of payload use it calls _alpm_dload_payload_reset() that will free() these and other fields anyway. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/dload.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 1785dd6a..343f5c78 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -698,10 +698,6 @@ static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, return 0; cleanup: - FREE(payload->fileurl); - FREE(payload->tempfile_name); - FREE(payload->destfile_name); - FREE(payload->content_disp_name); curl_easy_cleanup(curl); return ret; } -- 2.27.0

1 0

[pacman-dev] Errors in pacman man pages
by Helge Kreutzmann 09 Jul '20

09 Jul '20

Dear pacman maintainer, the manpage-l10n project maintains a large number of translations of man pages both from a large variety of sources (including packman) as well for a large variety of target languages. During their work translators notice different possible issues in the original (english) man pages. Sometimes this is a straightforward typo, sometimes a hard to read sentence, sometimes this is a convention not held up and sometimes we simply do not understand the original. We use several distributions as sources and update regularly (at least every 2 month). This means we are fairly recent (some distributions like archlinux also update frequently) but might miss the latest upstream version once in a while, so the error might be already fixed. We apologize and ask you to close the issue immediately if this should be the case, but given the huge volume of projects and the very limited number of volunteers we are not able to double check each and every issue. Secondly we translators see the manpages in the neutral po format, i.e. converted and harmonized, but not the original source (be it man, groff, xml or other). So we cannot provide a true patch (where possible), but only an approximation which you need to convert into your source format. Finally the issues I'm reporting have accumulated over time and are not always discovered by me, so sometimes my description of the problem my be a bit limited - do not hesitate to ask so we can clarify them. I'm now reporting the errors for your project. If future reports should use another channel, please let me know. Man page: makepkg-template.1 Issue: "name" and "version" should be I<> as they are options of the program "Currently used keys are: name (mandatory) and version\\&. Template names are " "limited to alphanumerics, \"@\", \"+\", \"\\&.\", \"-\", and \"_\"\\&. " "Versions are limited to numbers and \"\\&.\"\\&." -- Man page: pacman.8 Issue: This and other options' parameters could be wrapped in 'I<' and '>' "I<pacman> E<lt>operationE<gt> [options] [targets]" -- Man page: pacman.8 Issue: Curly double quotes in some strings, others regular quotes. Uniformize. "Since version 3\\&.0\\&.0, pacman has been the front-end to B<libalpm>(3), " "the \\(lqArch Linux Package Management\\(rq library\\&. This library allows " "alternative front-ends to be written (for instance, a GUI front-end)\\&." -- Man page: pacman.8 Issue: 'Search' 'owns' 'group' 'list' 'check' = option-related words, wrap around with "I<--" and ">" (or "B<--" and ">"?) "Show less information for certain query operations\\&. This is useful when " "pacman\\(cqs output is processed in a script\\&. Search will only show " "package names and not version, group, and description information; owns will " "only show package names instead of \"file is owned by pkg\" messages; group " "will only show package names and omit group names; list will only show files " "and omit package names; check will only show pairs of package names and " "missing files; a bare query will only show package names rather than names " -- Man page: pacman.8 Issue: 'Search' 'group' 'list' = option-related words, wrap around with "I<--" and ">" "Show less information for certain sync operations\\&. This is useful when " "pacman\\(cqs output is processed in a script\\&. Search will only show " "package names and not repository, version, group, and description " "information; list will only show package names and omit databases and " "versions; group will only show package names and omit group names\\&." -- Man page: pacman.conf.5 Issue: wget → B<wget>(1) "If set, an external program will be used to download all remote files\\&. " "All instances of %u will be replaced with the download URL\\&. If present, " "instances of %o will be replaced with the local filename, plus a \\(lq\\&." "part\\(rq extension, which allows programs like wget to do file resumes " "properly\\&." "This option is useful for users who experience problems with built-in HTTP/" "FTP support, or need the more advanced proxy support that comes with " "utilities like wget\\&." -- Man page: pacman.conf.5 Issue: Incorrect markup of -U: "" → B<> "Set the signature verification level for installing packages using the \"-U" "\" operation on a local file\\&. Uses the value from SigLevel as the default" "\\&." -- Man page: pacman.conf.5 Issue: Missing markup of syslog(): syslog() → B<syslog>() "Log action messages through syslog()\\&. This will insert log entries into /" "var/log/messages or equivalent\\&." -- Man page: pacman.conf.5 Issue: Where did "%o" come from? The XferCommand command usage does not have an %o "If set, an external program will be used to download all remote files\\&. " "All instances of %u will be replaced with the download URL\\&. If present, " "instances of %o will be replaced with the local filename, plus a \\(lq\\&." "part\\(rq extension, which allows programs like wget to do file resumes " "properly\\&." -- Man page: pacman.conf.5 Issue: [options] should not be translated, wrap it with B<> "The I<SigLevel> directive is valid in both the [options] and repository " "sections\\&. If used in [options], it sets a default value for any " "repository that does not provide the setting\\&." -- Man page: pacman-conf.8 Issue: remote → repository "B<-r, --repo> E<lt>remoteE<gt>" -- Dr. Helge Kreutzmann debian(a)helgefjell.de Dipl.-Phys. http://www.helgefjell.de/debian.php 64bit GNU powered gpg signed mail preferred Help keep free software "libre": http://www.ffii.de/

2 2

[pacman-dev] [GIT] The official pacman repository branch, master, updated. v5.2.1-105-gb01bcc7d
by Allan McRae 07 Jul '20

07 Jul '20

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "The official pacman repository". The branch, master has been updated via b01bcc7d3d680856bd60c4ae03e4ba3f6d889cb2 (commit) via f3dfba73d22b7eca3810a8114f2aab63da488b4c (commit) via 78d6dcec6c49bd2fa830237a46fd14337bc9fd4c (commit) via 34ba8d984d89393ab85ca67724b87af67ff004c3 (commit) via f078c2d3bcb72bafda0dce5fe2c9418ca462bb1a (commit) from 6b9c1b4d54225b4c2808b5fadc2b6e779ae1916a (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit b01bcc7d3d680856bd60c4ae03e4ba3f6d889cb2 Author: Anatol Pomozov <anatol.pomozov(a)gmail.com> Date: Tue May 26 19:12:08 2020 -0700 Fallback to detached signatures during keyring check Pacman has a 'key in keyring' verification step that makes sure the signatures have a valid keyid. Currently pacman parses embedded package signatures only. Add a fallback to detached signatures. If embedded signature is missing then it tries to read corresponding *.sig file and get keyid from there. Verification: debug: found cached pkg: /var/cache/pacman/pkg/glib-networking-2.64.3-1-x86_64.pkg.tar.zst debug: found detached signature /var/cache/pacman/pkg/glib-networking-2.64.3-1-x86_64.pkg.tar.zst.sig with size 310 debug: found signature key: A5E9288C4FA415FA debug: looking up key A5E9288C4FA415FA locally debug: key lookup success, key exists Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit f3dfba73d22b7eca3810a8114f2aab63da488b4c Author: Anatol Pomozov <anatol.pomozov(a)gmail.com> Date: Mon May 18 14:30:04 2020 -0700 FS#33992: force download *.sig file if it does not exist in the cache In case if *.pkg exists but *.sig file does not we still have to pass the pkg to multi_download API. To avoid redownloading *.pkg file we use CURLOPT_TIMECONDITION curl option. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 78d6dcec6c49bd2fa830237a46fd14337bc9fd4c Author: Anatol Pomozov <anatol.pomozov(a)gmail.com> Date: Thu Apr 23 17:28:43 2020 -0700 Add a utility function to check whether a file exists in the cache It is similar to _alpm_filecache_find() but does not return a dynamically allocated memory to user. Thus the user does not need to free this resource. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 34ba8d984d89393ab85ca67724b87af67ff004c3 Author: Anatol Pomozov <anatol.pomozov(a)gmail.com> Date: Mon May 18 14:13:11 2020 -0700 Do not use counter for error tracking Current code uses an incrementing counter to check whether a function returned error: errors += some_function(); if(errors) { goto finish } Replace with a more standard variable errors = some_function(); if(errors) { goto finish } Rename 'errors' variable to a more typical 'ret'. Avoid reporting both ALPM_EVENT_PKG_RETRIEVE_FAILED and ALPM_EVENT_PKG_RETRIEVE_DONE in the error path. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit f078c2d3bcb72bafda0dce5fe2c9418ca462bb1a Author: Anatol Pomozov <anatol.pomozov(a)gmail.com> Date: Tue May 12 15:26:38 2020 -0700 Move signature payload creation to download engine Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> ----------------------------------------------------------------------- Summary of changes: lib/libalpm/alpm.h | 11 ++++++ lib/libalpm/be_sync.c | 34 ++++-------------- lib/libalpm/dload.c | 95 ++++++++++++++++++++++++++++++++------------------- lib/libalpm/dload.h | 4 ++- lib/libalpm/package.c | 40 ++++++++++++++++++++++ lib/libalpm/sync.c | 67 ++++++++++++++++++++++-------------- lib/libalpm/util.c | 51 +++++++++++++++++++++++++++ lib/libalpm/util.h | 3 ++ 8 files changed, 215 insertions(+), 90 deletions(-) hooks/post-receive -- The official pacman repository

1 0

[pacman-dev] [PATCH] FS#33992: force download *.sig file if it does not exist in the cache
by Anatol Pomozov 29 Jun '20

29 Jun '20

In case if *.pkg exists but *.sig file does not we still have to pass the pkg to multi_download API. To avoid redownloading *.pkg file we use CURLOPT_TIMECONDITION curl option. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/dload.c | 2 +- lib/libalpm/sync.c | 31 +++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index f8ee6f85..78808eb3 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -282,7 +282,7 @@ static void curl_set_handle_opts(CURL *curl, struct dload_payload *payload) curl_easy_setopt(curl, CURLOPT_USERAGENT, useragent); } - if(!payload->allow_resume && !payload->force && payload->destfile_name && + if(!payload->force && payload->destfile_name && stat(payload->destfile_name, &st) == 0) { /* start from scratch, but only download if our local is out of date. */ curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index ea45767a..8c01ad95 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -22,6 +22,7 @@ */ #include <sys/types.h> /* off_t */ +#include <stdbool.h> #include <stdlib.h> #include <stdio.h> #include <string.h> @@ -698,8 +699,9 @@ static int find_dl_candidates(alpm_handle_t *handle, alpm_list_t **files) alpm_pkg_t *spkg = i->data; if(spkg->origin != ALPM_PKG_FROM_FILE) { - char *fpath = NULL; alpm_db_t *repo = spkg->origin_data.db; + bool need_download; + int siglevel = alpm_db_get_siglevel(alpm_pkg_get_db(spkg)); if(!repo->servers) { handle->pm_errno = ALPM_ERR_SERVER_NONE; @@ -710,16 +712,26 @@ static int find_dl_candidates(alpm_handle_t *handle, alpm_list_t **files) ASSERT(spkg->filename != NULL, RET_ERR(handle, ALPM_ERR_PKG_INVALID_NAME, -1)); - if(spkg->download_size == 0) { - /* check for file in cache - allows us to handle complete .part files */ - fpath = _alpm_filecache_find(handle, spkg->filename); + need_download = spkg->download_size != 0 || !_alpm_filecache_exists(handle, spkg->filename); + /* even if the package file in the cache we need to check for + * accompanion *.sig file as well. + * If *.sig is not cached then force download the package + its signature file. + */ + if(!need_download && (siglevel & ALPM_SIG_PACKAGE)) { + char *sig_filename = NULL; + int len = strlen(spkg->filename) + 5; + + MALLOC(sig_filename, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); + snprintf(sig_filename, len, "%s.sig", spkg->filename); + + need_download = !_alpm_filecache_exists(handle, sig_filename); + + FREE(sig_filename); } - if(spkg->download_size != 0 || !fpath) { + if(need_download) { *files = alpm_list_add(*files, spkg); } - - FREE(fpath); } } @@ -782,7 +794,8 @@ static int download_files(alpm_handle_t *handle) event.type = ALPM_EVENT_PKG_RETRIEVE_START; EVENT(handle, &event); for(i = files; i; i = i->next) { - const alpm_pkg_t *pkg = i->data; + alpm_pkg_t *pkg = i->data; + int siglevel = alpm_db_get_siglevel(alpm_pkg_get_db(pkg)); struct dload_payload *payload = NULL; CALLOC(payload, 1, sizeof(*payload), GOTO_ERR(handle, ALPM_ERR_MEMORY, finish)); @@ -794,6 +807,8 @@ static int download_files(alpm_handle_t *handle) payload->servers = pkg->origin_data.db->servers; payload->handle = handle; payload->allow_resume = 1; + payload->download_signature = (siglevel & ALPM_SIG_PACKAGE); + payload->signature_optional = (siglevel & ALPM_SIG_PACKAGE_OPTIONAL); payloads = alpm_list_add(payloads, payload); } -- 2.27.0

1 0

[pacman-dev] [PATCH] Add a utility function to check whether a file exists in the cache
by Anatol Pomozov 29 Jun '20

29 Jun '20

It is similar to _alpm_filecache_find() but does not return a dynamically allocated memory to user. Thus the user does not need to free this resource. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/util.c | 14 ++++++++++++++ lib/libalpm/util.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/lib/libalpm/util.c b/lib/libalpm/util.c index ead03004..76728eb4 100644 --- a/lib/libalpm/util.c +++ b/lib/libalpm/util.c @@ -834,6 +834,20 @@ char *_alpm_filecache_find(alpm_handle_t *handle, const char *filename) return NULL; } +/** Check whether a filename exists in a registered alpm cachedir. + * @param handle the context handle + * @param filename name of file to find + * @return 0 if the filename was not found, 1 otherwise + */ +int _alpm_filecache_exists(alpm_handle_t *handle, const char *filename) +{ + int res; + char *fpath = _alpm_filecache_find(handle, filename); + res = (fpath != NULL); + FREE(fpath); + return res; +} + /** Check the alpm cachedirs for existence and find a writable one. * If no valid cache directory can be found, use /tmp. * @param handle the context handle diff --git a/lib/libalpm/util.h b/lib/libalpm/util.h index 3306a022..4fc6e718 100644 --- a/lib/libalpm/util.h +++ b/lib/libalpm/util.h @@ -136,6 +136,8 @@ int _alpm_run_chroot(alpm_handle_t *handle, const char *cmd, char *const argv[], int _alpm_ldconfig(alpm_handle_t *handle); int _alpm_str_cmp(const void *s1, const void *s2); char *_alpm_filecache_find(alpm_handle_t *handle, const char *filename); +/* Checks whether a file exists in cache */ +int _alpm_filecache_exists(alpm_handle_t *handle, const char *filename); const char *_alpm_filecache_setup(alpm_handle_t *handle); /* Unlike many uses of alpm_pkgvalidation_t, _alpm_test_checksum expects * an enum value rather than a bitfield. */ -- 2.27.0

1 0

[pacman-dev] [PATCH] Do not use counter for error tracking
by Anatol Pomozov 26 Jun '20

26 Jun '20

Current code uses an incrementing counter to check whether a function returned error: errors += some_function(); if(errors) { goto finish } Replace with a more standard variable errors = some_function(); if(errors) { goto finish } Rename 'errors' variable to a more typical 'ret'. Avoid reporting both ALPM_EVENT_PKG_RETRIEVE_FAILED and ALPM_EVENT_PKG_RETRIEVE_DONE in the error path. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/sync.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 0c2341ae..ea45767a 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -730,7 +730,7 @@ static int download_files(alpm_handle_t *handle) { const char *cachedir; alpm_list_t *i, *files = NULL; - int errors = 0; + int ret = 0; alpm_event_t event; alpm_list_t *payloads = NULL; @@ -750,14 +750,16 @@ static int download_files(alpm_handle_t *handle) handle->totaldlcb(total_size); } - errors += find_dl_candidates(handle, &files); + ret = find_dl_candidates(handle, &files); + if(ret) { + goto finish; + } - if(files && !errors) { + if(files) { /* check for necessary disk space for download */ if(handle->checkspace) { off_t *file_sizes; size_t idx, num_files; - int ret; _alpm_log(handle, ALPM_LOG_DEBUG, "checking available disk space for download\n"); @@ -773,7 +775,6 @@ static int download_files(alpm_handle_t *handle) free(file_sizes); if(ret != 0) { - errors++; goto finish; } } @@ -796,12 +797,13 @@ static int download_files(alpm_handle_t *handle) payloads = alpm_list_add(payloads, payload); } - event.type = ALPM_EVENT_PKG_RETRIEVE_DONE; if(_alpm_download(handle, payloads, cachedir) == -1) { - errors++; event.type = ALPM_EVENT_PKG_RETRIEVE_FAILED; + EVENT(handle, &event); _alpm_log(handle, ALPM_LOG_WARNING, _("failed to retrieve some files\n")); + goto finish; } + event.type = ALPM_EVENT_PKG_RETRIEVE_DONE; EVENT(handle, &event); } @@ -826,7 +828,7 @@ finish: handle->totaldlcb(0); } - return errors; + return ret; } #ifdef HAVE_LIBGPGME -- 2.27.0

1 0

[pacman-dev] [PATCH v2] Move signature payload creation to download engine
by Anatol Pomozov 26 Jun '20

26 Jun '20

Until now callee of ALPM download functionality has been in charge of payload creation both for the main file (e.g. *.pkg) and for the accompanied *.sig file. One advantage of such solution is that all payloads are independent and can be fetched in parallel thus exploiting the maximum level of download parallelism. To build *.sig file url we've been using a simple string concatenation: $requested_url + ".sig". Unfortunately there are cases when it does not work. For example an archlinux.org "Download From Mirror" link looks like this https://www.archlinux.org/packages/core/x86_64/bash/download/ and it gets redirected to some mirror. But if we append ".sig" to the end of the link url and try to download it then archlinux.org returns 404 error. To overcome this issue we need to follow redirects for the main payload first, find the final url and only then append '.sig' suffix. This implies 2 things: - the signature payload initialization need to be moved to dload.c as it is the place where we have access to the resolved url - *.sig is downloaded serially with the main payload and this reduces level of parallelism Move *.sig payload creation to dload.c. Once the main payload is fetched successfully we check if the callee asked to download the accompanied signature. If yes - create a new payload and add it to mcurl. *.sig payload does not use server list of the main payload and thus does not support mirror failover. *.sig file comes from the same server as the main payload. Refactor event loop in curl_multi_download_internal() a bit. Instead of relying on curl_multi_check_finished_download() to return number of new payloads we simply rerun the loop iteration one more time to check if there are any active downloads left. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/be_sync.c | 34 +++------------- lib/libalpm/dload.c | 93 +++++++++++++++++++++++++++---------------- lib/libalpm/dload.h | 4 +- 3 files changed, 67 insertions(+), 64 deletions(-) diff --git a/lib/libalpm/be_sync.c b/lib/libalpm/be_sync.c index a0736320..225df76d 100644 --- a/lib/libalpm/be_sync.c +++ b/lib/libalpm/be_sync.c @@ -180,12 +180,10 @@ int SYMEXPORT alpm_db_update(alpm_handle_t *handle, alpm_list_t *dbs, int force) dbforce = 1; } - CALLOC(payload, 1, sizeof(*payload), GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); + siglevel = alpm_db_get_siglevel(db); - /* set hard upper limit of 128 MiB */ - payload->max_size = 128 * 1024 * 1024; + CALLOC(payload, 1, sizeof(*payload), GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); payload->servers = db->servers; - /* print server + filename into a buffer */ len = strlen(db->treename) + strlen(dbext) + 1; MALLOC(payload->filepath, len, @@ -194,31 +192,11 @@ int SYMEXPORT alpm_db_update(alpm_handle_t *handle, alpm_list_t *dbs, int force) payload->handle = handle; payload->force = dbforce; payload->unlink_on_fail = 1; - + payload->download_signature = (siglevel & ALPM_SIG_DATABASE); + payload->signature_optional = (siglevel & ALPM_SIG_DATABASE_OPTIONAL); + /* set hard upper limit of 128 MiB */ + payload->max_size = 128 * 1024 * 1024; payloads = alpm_list_add(payloads, payload); - - siglevel = alpm_db_get_siglevel(db); - if(siglevel & ALPM_SIG_DATABASE) { - struct dload_payload *sig_payload; - CALLOC(sig_payload, 1, sizeof(*sig_payload), GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); - sig_payload->signature = 1; - - /* print filename into a buffer (leave space for separator and .sig) */ - len = strlen(db->treename) + strlen(dbext) + 5; - MALLOC(sig_payload->filepath, len, - FREE(sig_payload); GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); - snprintf(sig_payload->filepath, len, "%s%s.sig", db->treename, dbext); - - sig_payload->handle = handle; - sig_payload->force = dbforce; - sig_payload->errors_ok = (siglevel & ALPM_SIG_DATABASE_OPTIONAL); - - /* set hard upper limit of 16 KiB */ - sig_payload->max_size = 16 * 1024; - sig_payload->servers = db->servers; - - payloads = alpm_list_add(payloads, sig_payload); - } } event.type = ALPM_EVENT_DB_RETRIEVE_START; diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index a91bb1ac..f8ee6f85 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -18,6 +18,7 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <stdbool.h> #include <stdlib.h> #include <stdio.h> #include <errno.h> @@ -49,6 +50,10 @@ #include "handle.h" #ifdef HAVE_LIBCURL + +static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, + struct dload_payload *payload, const char *localpath); + static const char *get_filename(const char *url) { char *filename = strrchr(url, '/'); @@ -476,6 +481,25 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &timecond); curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); + /* Let's check if client requested downloading accompanion *.sig file */ + if(!payload->signature && payload->download_signature && curlerr == CURLE_OK && payload->respcode < 400) { + struct dload_payload *sig = NULL; + + int len = strlen(effective_url) + 5; + CALLOC(sig, 1, sizeof(*sig), GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); + MALLOC(sig->fileurl, len, FREE(sig); GOTO_ERR(handle, ALPM_ERR_MEMORY, cleanup)); + snprintf(sig->fileurl, len, "%s.sig", effective_url); + sig->signature = 1; + sig->handle = handle; + sig->force = payload->force; + sig->unlink_on_fail = payload->unlink_on_fail; + sig->errors_ok = payload->signature_optional; + /* set hard upper limit of 16KiB */ + sig->max_size = 16 * 1024; + + curl_add_payload(handle, curlm, sig, localpath); + } + /* time condition was met and we didn't download anything. we need to * clean up the 0 byte .part file that's left behind. */ if(timecond == 1 && DOUBLE_EQ(bytes_dl, 0)) { @@ -565,6 +589,13 @@ cleanup: if(ret == -1 && payload->errors_ok) { ret = -2; } + + if(payload->signature) { + /* free signature payload memory that was allocated earlier in dload.c */ + _alpm_dload_payload_reset(payload); + FREE(payload); + } + return ret; } @@ -666,17 +697,16 @@ static int curl_download_internal(alpm_handle_t *handle, alpm_list_t *payloads /* struct dload_payload */, const char *localpath) { - int still_running = 0; + int active_downloads_num = 0; + bool recheck_downloads = false; int err = 0; - int parallel_downloads = handle->parallel_downloads; - + int max_streams = handle->parallel_downloads; CURLM *curlm = handle->curlm; - CURLMsg *msg; - while(still_running || payloads) { - int msgs_left = -1; + while(active_downloads_num > 0 || payloads || recheck_downloads) { + CURLMcode mc; - for(; still_running < parallel_downloads && payloads; still_running++) { + for(; active_downloads_num < max_streams && payloads; active_downloads_num++) { struct dload_payload *payload = payloads->data; if(curl_add_payload(handle, curlm, payload, localpath) == 0) { @@ -687,15 +717,19 @@ static int curl_download_internal(alpm_handle_t *handle, payloads = payloads->next; } else { - // the payload failed to start, do not start any new downloads just wait until - // active one complete. + /* The payload failed to start. Do not start any new downloads. + * Wait until all active downloads complete. + */ _alpm_log(handle, ALPM_LOG_ERROR, _("failed to setup a download payload for %s\n"), payload->remote_name); payloads = NULL; err = -1; } } - CURLMcode mc = curl_multi_perform(curlm, &still_running); + mc = curl_multi_perform(curlm, &active_downloads_num); + if(mc == CURLM_OK) { + mc = curl_multi_wait(curlm, NULL, 0, 1000, NULL); + } if(mc != CURLM_OK) { _alpm_log(handle, ALPM_LOG_ERROR, _("curl returned error %d from transfer\n"), mc); @@ -703,7 +737,13 @@ static int curl_download_internal(alpm_handle_t *handle, err = -1; } - while((msg = curl_multi_info_read(curlm, &msgs_left))) { + recheck_downloads = false; + while(true) { + int msgs_left = 0; + CURLMsg *msg = curl_multi_info_read(curlm, &msgs_left); + if(!msg) { + break; + } if(msg->msg == CURLMSG_DONE) { int ret = curl_check_finished_download(curlm, msg, localpath); if(ret == -1) { @@ -712,19 +752,16 @@ static int curl_download_internal(alpm_handle_t *handle, */ payloads = NULL; err = -1; - } else if(ret == 2) { - /* in case of a retry increase the counter of active requests - * to avoid exiting the loop early - */ - still_running++; } + /* curl_multi_check_finished_download() might add more payloads e.g. in case of a retry + * from the next mirror. We need to execute curl_multi_perform() at least one more time + * to make sure new payload requests are processed. + */ + recheck_downloads = true; } else { _alpm_log(handle, ALPM_LOG_ERROR, _("curl transfer error: %d\n"), msg->msg); } } - if(still_running) { - curl_multi_wait(curlm, NULL, 0, 1000, NULL); - } } _alpm_log(handle, ALPM_LOG_DEBUG, "curl_download_internal return code is %d\n", err); @@ -803,13 +840,11 @@ int SYMEXPORT alpm_fetch_pkgurl(alpm_handle_t *handle, const alpm_list_t *urls, { const char *cachedir; alpm_list_t *payloads = NULL; - int download_sigs; const alpm_list_t *i; alpm_event_t event; CHECK_HANDLE(handle, return -1); ASSERT(*fetched == NULL, RET_ERR(handle, ALPM_ERR_WRONG_ARGS, -1)); - download_sigs = handle->siglevel & ALPM_SIG_PACKAGE; /* find a valid cache dir to download to */ cachedir = _alpm_filecache_setup(handle); @@ -831,21 +866,9 @@ int SYMEXPORT alpm_fetch_pkgurl(alpm_handle_t *handle, const alpm_list_t *urls, payload->allow_resume = 1; payload->handle = handle; payload->trust_remote_name = 1; + payload->download_signature = (handle->siglevel & ALPM_SIG_PACKAGE); + payload->signature_optional = (handle->siglevel & ALPM_SIG_PACKAGE_OPTIONAL); payloads = alpm_list_add(payloads, payload); - - if(download_sigs) { - int len = strlen(url) + 5; - CALLOC(payload, 1, sizeof(*payload), GOTO_ERR(handle, ALPM_ERR_MEMORY, err)); - payload->signature = 1; - MALLOC(payload->fileurl, len, FREE(payload); GOTO_ERR(handle, ALPM_ERR_MEMORY, err)); - snprintf(payload->fileurl, len, "%s.sig", url); - payload->handle = handle; - payload->trust_remote_name = 1; - payload->errors_ok = (handle->siglevel & ALPM_SIG_PACKAGE_OPTIONAL); - /* set hard upper limit of 16KiB */ - payload->max_size = 16 * 1024; - payloads = alpm_list_add(payloads, payload); - } } } diff --git a/lib/libalpm/dload.h b/lib/libalpm/dload.h index 9103c1c6..9491dd8c 100644 --- a/lib/libalpm/dload.h +++ b/lib/libalpm/dload.h @@ -47,11 +47,13 @@ struct dload_payload { int errors_ok; int unlink_on_fail; int trust_remote_name; - int signature; /* specifies if the payload is a signature file */ + int download_signature; /* specifies if an accompanion *.sig file need to be downloaded*/ + int signature_optional; /* *.sig file is optional */ #ifdef HAVE_LIBCURL CURL *curl; char error_buffer[CURL_ERROR_SIZE]; FILE *localf; /* temp download file */ + int signature; /* specifies if this payload is for a signature file */ #endif }; -- 2.27.0

1 0