April 2021 - Pacman-dev - lists.archlinux.org

[pacman-dev] [PATCH] libalpm: fix alpm_fetch_pkgurl with a fetch callback
by Guillaume Benoit 17 May '21

17 May '21

After download, alpm_fetch_pkgurl uses payload->destfile_name and payload->tempfile_name in order to find the downloaded file path. Those fields are not set if a custom fetch callback is defined. Use filecache_find_url instead, like in the beginning of the function. --- lib/libalpm/dload.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 6f33451a..d45c7707 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -1007,16 +1007,11 @@ int SYMEXPORT alpm_fetch_pkgurl(alpm_handle_t *handle, const alpm_list_t *urls, EVENT(handle, &event); } - for(i = payloads; i; i = i->next) { - struct dload_payload *payload = i->data; - char *filepath; + for(i = urls; i; i = i->next) { + char *url = i->data; - if(payload->destfile_name) { - const char *filename = mbasename(payload->destfile_name); - filepath = _alpm_filecache_find(handle, filename); - } else { - STRDUP(filepath, payload->tempfile_name, GOTO_ERR(handle, ALPM_ERR_MEMORY, err)); - } + /* attempt again to find the file in our pkgcache */ + char *filepath = filecache_find_url(handle, url); if(filepath) { alpm_list_append(fetched, filepath); } else { -- 2.31.1

3 9

[pacman-dev] [PATCH] libalpm: check for duplicate filenames
by morganamilo 09 May '21

09 May '21

This partially fixes FS#67850 It fixes the case for -S'ing packages but not -U'ing urls. morganamilo@Octavia dup ~git/pacman % sudo ./build/pacman -S a/a b/b resolving dependencies... error: a and b have the same filename: a-1-1-any.pkg.tar.zst error: failed to prepare transaction (duplicate filename) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 833df829..ccf2edc2 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -257,6 +257,8 @@ typedef enum _alpm_errno_t { ALPM_ERR_TRANS_NULL, /** Duplicate target in transaction */ ALPM_ERR_TRANS_DUP_TARGET, + /** Duplicate filename in transaction */ + ALPM_ERR_TRANS_DUP_FILENAME, /** A transaction has not been initialized */ ALPM_ERR_TRANS_NOT_INITIALIZED, /** Transaction has not been prepared */ diff --git a/lib/libalpm/error.c b/lib/libalpm/error.c index c6f19599..c06d091a 100644 --- a/lib/libalpm/error.c +++ b/lib/libalpm/error.c @@ -90,6 +90,8 @@ const char SYMEXPORT *alpm_strerror(alpm_errno_t err) return _("transaction not initialized"); case ALPM_ERR_TRANS_DUP_TARGET: return _("duplicate target"); + case ALPM_ERR_TRANS_DUP_FILENAME: + return _("duplicate filename"); case ALPM_ERR_TRANS_NOT_INITIALIZED: return _("transaction not initialized"); case ALPM_ERR_TRANS_NOT_PREPARED: diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 7fa50a13..73e9e77d 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -464,6 +464,25 @@ int _alpm_sync_prepare(alpm_handle_t *handle, alpm_list_t **data) } } + /* Ensure two packages don't have the same filename */ + for(i = resolved; i; i = i->next) { + alpm_pkg_t *pkg1 = i->data; + for(j = i->next; j; j = j->next) { + alpm_pkg_t *pkg2 = j->data; + if(strcmp(pkg1->filename, pkg2->filename) == 0) { + alpm_list_free(resolved); + ret = -1; + handle->pm_errno = ALPM_ERR_TRANS_DUP_FILENAME; + _alpm_log(handle, ALPM_LOG_ERROR, _("%s and %s have the same filename: %s\n"), + pkg1->name, pkg2->name, pkg1->filename); + } + } + } + + if(ret != 0) { + goto cleanup; + } + /* Set DEPEND reason for pulled packages */ for(i = resolved; i; i = i->next) { alpm_pkg_t *pkg = i->data; diff --git a/lib/libalpm/trans.c b/lib/libalpm/trans.c index c6ec7eea..4cee603c 100644 --- a/lib/libalpm/trans.c +++ b/lib/libalpm/trans.c @@ -122,7 +122,7 @@ int SYMEXPORT alpm_trans_prepare(alpm_handle_t *handle, alpm_list_t **data) /* pm_errno is set by _alpm_remove_prepare() */ return -1; } - } else { + } else { if(_alpm_sync_prepare(handle, data) == -1) { /* pm_errno is set by _alpm_sync_prepare() */ return -1; -- 2.31.1

2 1

[pacman-dev] [PATCH] libalpm: download rates becoming negative
by morganamilo 09 May '21

09 May '21

When a download fails on one mirror a new download is started on the next mirror. This new download will have the initial size of whatever has been downloaded so far, as well as the ammount downloaded reset to 0. To account for this, when a download changes mirror, save how much has been downloaded so far and add that to dlcb calls. --- lib/libalpm/dload.c | 14 ++++++++++++-- lib/libalpm/dload.h | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index a4c42f8d..27a7748a 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -207,8 +207,8 @@ static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow, /* do NOT include initial_size since it wasn't part of the package's * download_size (nor included in the total download size callback) */ - cb_data.total = dltotal; - cb_data.downloaded = dlnow; + cb_data.total = dltotal + payload->resetprogress; + cb_data.downloaded = dlnow + payload->resetprogress; payload->handle->dlcb(payload->remote_name, ALPM_DOWNLOAD_PROGRESS, &cb_data); payload->prevprogress = current_size; @@ -440,6 +440,16 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload fseek(payload->localf, 0, SEEK_SET); } + + /* reset progress for next server */ + payload->resetprogress += payload->prevprogress - payload->initial_size; + payload->unlink_on_fail = 0; + + if(payload->allow_resume) { + payload->initial_size = payload->prevprogress; + } + + /* Set curl with the new URL */ curl_easy_setopt(curl, CURLOPT_URL, payload->fileurl); diff --git a/lib/libalpm/dload.h b/lib/libalpm/dload.h index 8f3d17b4..abfd9466 100644 --- a/lib/libalpm/dload.h +++ b/lib/libalpm/dload.h @@ -42,6 +42,7 @@ struct dload_payload { off_t initial_size; off_t max_size; off_t prevprogress; + off_t resetprogress; /* progress made before we try a new server */ int force; int allow_resume; int errors_ok; -- 2.31.1

5 15

[pacman-dev] [PATCH 0/1] signhandler add SIGQUIT soft_interrupt_handler
by ZheNing Hu 09 May '21

09 May '21

Hello, eveyone! I'am a newbie, because of the experience of using pacman downloading with the `Ctrl +\` exit was not very good,it leave the database lock, so I am thinking can we just deal with SIGQUIT rather than doing nothing. Hopefully this will improve the experience of users. Thanks. ZheNing Hu (1): signhandler: add SIGQUIT soft_interrupt_handler src/pacman/sighandler.c | 3 +++ 1 file changed, 3 insertions(+) -- 2.31.0

3 3

[pacman-dev] [PATCH] libalpm: clone data on alpm_db_set_servers
by morganamilo 09 May '21

09 May '21

Every alpm_option_set function clones the input so lets be more consistent. Also this fixes servers not being sanatized. --- lib/libalpm/db.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/libalpm/db.c b/lib/libalpm/db.c index 8511bb83..6ddf5461 100644 --- a/lib/libalpm/db.c +++ b/lib/libalpm/db.c @@ -139,9 +139,16 @@ alpm_list_t SYMEXPORT *alpm_db_get_servers(const alpm_db_t *db) int SYMEXPORT alpm_db_set_servers(alpm_db_t *db, alpm_list_t *servers) { + alpm_list_t *i; ASSERT(db != NULL, return -1); FREELIST(db->servers); - db->servers = servers; + for(i = servers; i; i = i->next) { + char *url = i->data; + if(alpm_db_add_server(db, url) != 0) { + return -1; + } + servers = servers->next; + } return 0; } -- 2.31.1

3 3

[pacman-dev] [PATCH] libalpm: download signatures with a fetch callback
by Guillaume Benoit 30 Apr '21

30 Apr '21

Remove an unwanted break and download signatures when using a custom fetch callback. --- lib/libalpm/dload.c | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index a4c42f8d..6f33451a 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -877,9 +877,26 @@ int _alpm_download(alpm_handle_t *handle, int success = 0; if(payload->fileurl) { - if (handle->fetchcb(payload->fileurl, localpath, payload->force) != -1) { - success = 1; - break; + int ret; + + ret = handle->fetchcb(payload->fileurl, localpath, payload->force); + + if (ret != -1) { + /* Let's check if client requested downloading accompanion *.sig file */ + if(payload->download_signature) { + char *sig_fileurl; + + size_t sig_len = strlen(payload->fileurl) + 5; + MALLOC(sig_fileurl, sig_len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); + snprintf(sig_fileurl, sig_len, "%s.sig", payload->fileurl); + + ret = handle->fetchcb(sig_fileurl, localpath, payload->force); + + free(sig_fileurl); + } + if (ret != -1) { + success = 1; + } } } else { for(s = payload->servers; s; s = s->next) { @@ -892,12 +909,26 @@ int _alpm_download(alpm_handle_t *handle, snprintf(fileurl, len, "%s/%s", server, payload->filepath); ret = handle->fetchcb(fileurl, localpath, payload->force); - free(fileurl); if (ret != -1) { - success = 1; - break; + /* Let's check if client requested downloading accompanion *.sig file */ + if(payload->download_signature) { + char *sig_fileurl; + + size_t sig_len = strlen(fileurl) + 5; + MALLOC(sig_fileurl, sig_len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); + snprintf(sig_fileurl, sig_len, "%s.sig", fileurl); + + ret = handle->fetchcb(sig_fileurl, localpath, payload->force); + + free(sig_fileurl); + } + if (ret != -1) { + success = 1; + break; + } } + free(fileurl); } } if(!success && !payload->errors_ok) { -- 2.31.1

1 0

[pacman-dev] [PATCH] call download progress callback for signatures
by Andrew Gregory 30 Apr '21

30 Apr '21

pacman may not care about them, but other front-ends do. Signed-off-by: Andrew Gregory <andrew.gregory.8(a)gmail.com> --- lib/libalpm/dload.c | 18 +++++++----------- src/pacman/callback.c | 12 ++++++++++++ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 935f5418..8fa46bb2 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -166,11 +166,6 @@ static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow, off_t current_size, total_size; alpm_download_event_progress_t cb_data = {0}; - /* do not print signature files progress bar */ - if(payload->signature) { - return 0; - } - /* avoid displaying progress bar for redirects with a body */ if(payload->respcode >= 300) { return 0; @@ -669,7 +664,7 @@ cleanup: unlink(payload->tempfile_name); } - if(handle->dlcb && !payload->signature) { + if(handle->dlcb) { alpm_download_event_completed_t cb_data = {0}; cb_data.total = bytes_dl; cb_data.result = ret; @@ -782,6 +777,12 @@ static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, curl_easy_setopt(curl, CURLOPT_WRITEDATA, payload->localf); curl_multi_add_handle(curlm, curl); + + if(handle->dlcb) { + alpm_download_event_init_t cb_data = {.optional = payload->errors_ok}; + handle->dlcb(handle->dlcb_ctx, payload->remote_name, ALPM_DOWNLOAD_INIT, &cb_data); + } + return 0; cleanup: @@ -810,11 +811,6 @@ static int curl_download_internal(alpm_handle_t *handle, struct dload_payload *payload = payloads->data; if(curl_add_payload(handle, curlm, payload, localpath) == 0) { - if(handle->dlcb && !payload->signature) { - alpm_download_event_init_t cb_data = {.optional = payload->errors_ok}; - handle->dlcb(handle->dlcb_ctx, payload->remote_name, ALPM_DOWNLOAD_INIT, &cb_data); - } - payloads = payloads->next; } else { /* The payload failed to start. Do not start any new downloads. diff --git a/src/pacman/callback.c b/src/pacman/callback.c index c3563af3..ae7e1bed 100644 --- a/src/pacman/callback.c +++ b/src/pacman/callback.c @@ -1070,10 +1070,22 @@ static void dload_complete_event(const char *filename, alpm_download_event_compl } } +static int strendswith(const char *haystack, const char *needle) +{ + size_t hlen = strlen(haystack), nlen = strlen(needle); + return hlen >= nlen && strcmp(haystack + hlen - nlen, needle) == 0; +} + /* Callback to handle display of download progress */ void cb_download(void *ctx, const char *filename, alpm_download_event_type_t event, void *data) { (void)ctx; + + /* do not print signature files progress bar */ + if(strendswith(filename, ".sig")) { + return; + } + if(event == ALPM_DOWNLOAD_INIT) { dload_init_event(filename, data); } else if(event == ALPM_DOWNLOAD_PROGRESS) { -- 2.31.1

1 0

[pacman-dev] Patch Policy Inquiry
by Sijie Bu 28 Apr '21

28 Apr '21

Hello, Not sure if I'm posting it to the right place, but this is a developer-related question so hopefully it's in the right place. I will be starting an internship project with CIP United, a company specialized in MIPS processors, and will be porting Arch to MIPS64 R6. And my project supervisor would like to know pacman's patch policies for architecture-specific modifications. I have already asked on arch-ports list and was told by people from Arch Linux 32 that the pacman codebase is generally portable, but I see them maintaining a fork for certain i486-specific optimizations. I have also looked at the readme files (HACKING etc.) but was unable to find a clear statement of your patch policies. Therefore, I would like to know some details about your patch policies, especially the ones regarding non-amd64 architectures. Thank you for your time and have a nice day. Sincerely Tom

3 2

[pacman-dev] [PATCH] alpm_db_update: indicate if dbs were up to date
by Andrew Gregory 26 Apr '21

26 Apr '21

Restore the prior indicator whether or not databases were up to date. 0 is used to indicate if *any* db was actually updated as callers are more likely to care about that than if *all* dbs were updated. Signed-off-by: Andrew Gregory <andrew.gregory.8(a)gmail.com> --- Please include this for v6.0. Pacutils uses the up-to-date indicator. lib/libalpm/alpm.h | 3 ++- lib/libalpm/dload.c | 35 +++++++++++++++++++++-------------- lib/libalpm/sync.c | 2 +- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 2e99d4d8..f47ddc1d 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -1356,7 +1356,8 @@ int alpm_db_remove_server(alpm_db_t *db, const char *url); * @param dbs list of package databases to update * @param force if true, then forces the update, otherwise update only in case * the databases aren't up to date - * @return 0 on success, -1 on error (pm_errno is set accordingly) + * @return 0 on success, -1 on error (pm_errno is set accordingly), + * 1 if all databases are up to to date */ int alpm_db_update(alpm_handle_t *handle, alpm_list_t *dbs, int force); diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 57d25383..00a729ec 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -788,6 +788,10 @@ cleanup: return ret; } +/* Returns -1 if an error happened for a required file + * Returns 0 if a payload was actually downloaded + * Returns 1 if no files were downloaded and all errors were non-fatal + */ static int curl_download_internal(alpm_handle_t *handle, alpm_list_t *payloads /* struct dload_payload */, const char *localpath) @@ -795,6 +799,7 @@ static int curl_download_internal(alpm_handle_t *handle, int active_downloads_num = 0; int err = 0; int max_streams = handle->parallel_downloads; + int updated = 0; /* was a file actually updated */ CURLM *curlm = handle->curlm; while(active_downloads_num > 0 || payloads) { @@ -846,6 +851,8 @@ static int curl_download_internal(alpm_handle_t *handle, */ payloads = NULL; err = -1; + } else if(ret == 0) { + updated = 1; } } else { _alpm_log(handle, ALPM_LOG_ERROR, _("curl transfer error: %d\n"), msg->msg); @@ -854,11 +861,15 @@ static int curl_download_internal(alpm_handle_t *handle, } _alpm_log(handle, ALPM_LOG_DEBUG, "curl_download_internal return code is %d\n", err); - return err; + return err ? -1 : updated ? 0 : 1; } #endif +/* Returns -1 if an error happened for a required file + * Returns 0 if a payload was actually downloaded + * Returns 1 if no files were downloaded and all errors were non-fatal + */ int _alpm_download(alpm_handle_t *handle, alpm_list_t *payloads /* struct dload_payload */, const char *localpath) @@ -871,20 +882,18 @@ int _alpm_download(alpm_handle_t *handle, #endif } else { alpm_list_t *p; + int updated = 0; for(p = payloads; p; p = p->next) { struct dload_payload *payload = p->data; alpm_list_t *s; - int success = 0; + int ret = -1; if(payload->fileurl) { - if (handle->fetchcb(payload->fileurl, localpath, payload->force) != -1) { - success = 1; - } + ret = handle->fetchcb(payload->fileurl, localpath, payload->force); } else { - for(s = payload->servers; s; s = s->next) { + for(s = payload->servers; s && ret == -1; s = s->next) { const char *server = s->data; char *fileurl; - int ret; size_t len = strlen(server) + strlen(payload->filepath) + 2; MALLOC(fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); @@ -892,18 +901,16 @@ int _alpm_download(alpm_handle_t *handle, ret = handle->fetchcb(fileurl, localpath, payload->force); free(fileurl); - - if (ret != -1) { - success = 1; - break; - } } } - if(!success && !payload->errors_ok) { + + if(ret == -1 && !payload->errors_ok) { RET_ERR(handle, ALPM_ERR_EXTERNAL_DOWNLOAD, -1); + } else if(ret == 0) { + updated = 1; } } - return 0; + return updated ? 0 : 1; } } diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 7fa50a13..2419cc69 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -815,7 +815,7 @@ static int download_files(alpm_handle_t *handle) } ret = _alpm_download(handle, payloads, cachedir); - if(ret != 0) { + if(ret == -1) { event.type = ALPM_EVENT_PKG_RETRIEVE_FAILED; EVENT(handle, &event); _alpm_log(handle, ALPM_LOG_WARNING, _("failed to retrieve some files\n")); -- 2.31.1

1 0

[pacman-dev] [PATCH 1/3] pmpkg: add makepkg_bytes
by Andrew Gregory 26 Apr '21

26 Apr '21

Builds the package file in memory. Useful with the built-in server. Signed-off-by: Andrew Gregory <andrew.gregory.8(a)gmail.com> --- test/pacman/pmpkg.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/test/pacman/pmpkg.py b/test/pacman/pmpkg.py index 8ff1ec1e..01ed4e0a 100644 --- a/test/pacman/pmpkg.py +++ b/test/pacman/pmpkg.py @@ -98,7 +98,12 @@ def parse_filename(name): filename, extra = filename.split("|") return filename - def makepkg(self, path): + def makepkg_bytes(self): + buf = BytesIO(); + self.makepkg(fileobj=buf) + return buf.getvalue() + + def makepkg(self, path=None, fileobj=None): """Creates an Arch Linux package archive. A package archive is generated in the location 'path', based on the data @@ -138,11 +143,12 @@ def makepkg(self, path): if any(self.install.values()): archive_files.append((".INSTALL", self.installfile())) - self.path = os.path.join(path, self.filename()) - util.mkdir(os.path.dirname(self.path)) + if path: + self.path = os.path.join(path, self.filename()) + util.mkdir(os.path.dirname(self.path)) # Generate package metadata - tar = tarfile.open(self.path, "w:gz", format=tarfile.GNU_FORMAT) + tar = tarfile.open(name=self.path, fileobj=fileobj, mode="w:gz", format=tarfile.GNU_FORMAT) for name, data in archive_files: info = tarfile.TarInfo(name) info.size = len(data) -- 2.31.1

1 2