- Pacman-dev - lists.archlinux.org

[pacman-dev] [PATCH v3] libalpm: fix download rates becoming negative
by morganamilo 09 May '21

09 May '21

When a download fails on one mirror a new download is started on the next mirror. This causes the ammount downloaded to reset, confusing the rate math and making it display a negative rate. This is further complicated by the fact that a download may be resumed from where it is or started over. To account for this we alert the frontend that the download was restarted. Pacman then starts the progress bar over. --- v2: stat tempfile to get initial_size v3: review fixes --- lib/libalpm/alpm.h | 7 +++++++ lib/libalpm/dload.c | 33 +++++++++++++++++++++++++-------- src/pacman/callback.c | 30 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 101d686b..226fd953 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -1172,6 +1172,8 @@ typedef enum _alpm_download_event_type_t { ALPM_DOWNLOAD_INIT, /** A download made progress */ ALPM_DOWNLOAD_PROGRESS, + /** Download will be retried */ + ALPM_DOWNLOAD_RETRY, /** A download completed */ ALPM_DOWNLOAD_COMPLETED } alpm_download_event_type_t; @@ -1190,6 +1192,11 @@ typedef struct _alpm_download_event_progress_t { off_t total; } alpm_download_event_progress_t; +/** Context struct for when a download retries. */ +typedef struct _alpm_download_event_retry_t { + /** If the download will resume or start over */ + int resume; +} alpm_download_event_retry_t; /** Context struct for when a download completes. */ typedef struct _alpm_download_event_completed_t { diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 8fa46bb2..2d8b4d6d 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -408,6 +408,7 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload { const char *server; size_t len; + struct stat st; alpm_handle_t *handle = payload->handle; while(payload->servers && should_skip_server(handle, payload->servers->data)) { @@ -427,15 +428,31 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload MALLOC(payload->fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); snprintf(payload->fileurl, len, "%s/%s", server, payload->filepath); - if(payload->unlink_on_fail) { + + fflush(payload->localf); + + if(payload->allow_resume && stat(payload->tempfile_name, &st) == 0) { + /* a previous partial download exists, resume from end of file. */ + payload->tempfile_openmode = "ab"; + curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE, (curl_off_t)st.st_size); + _alpm_log(handle, ALPM_LOG_DEBUG, + "%s: tempfile found, attempting continuation from %jd bytes\n", + payload->remote_name, (intmax_t)st.st_size); + payload->initial_size = st.st_size; + } else { /* we keep the file for a new retry but remove its data if any */ - fflush(payload->localf); if(ftruncate(fileno(payload->localf), 0)) { RET_ERR(handle, ALPM_ERR_SYSTEM, -1); } fseek(payload->localf, 0, SEEK_SET); } + if(handle->dlcb && !payload->signature) { + alpm_download_event_retry_t cb_data; + cb_data.resume = payload->allow_resume; + handle->dlcb(handle->dlcb_ctx, payload->remote_name, ALPM_DOWNLOAD_RETRY, &cb_data); + } + /* Set curl with the new URL */ curl_easy_setopt(curl, CURLOPT_URL, payload->fileurl); @@ -483,7 +500,6 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, _alpm_log(handle, ALPM_LOG_DEBUG, "%s: response code %ld\n", payload->remote_name, payload->respcode); if(payload->respcode >= 400) { - payload->unlink_on_fail = 1; if(!payload->errors_ok) { handle->pm_errno = ALPM_ERR_RETRIEVE; /* non-translated message is same as libcurl */ @@ -498,6 +514,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, (*active_downloads_num)++; return 2; } else { + payload->unlink_on_fail = 1; goto cleanup; } } @@ -515,7 +532,6 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, } goto cleanup; case CURLE_COULDNT_RESOLVE_HOST: - payload->unlink_on_fail = 1; handle->pm_errno = ALPM_ERR_SERVER_BAD_URL; _alpm_log(handle, ALPM_LOG_ERROR, _("failed retrieving file '%s' from %s : %s\n"), @@ -525,13 +541,10 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, (*active_downloads_num)++; return 2; } else { + payload->unlink_on_fail = 1; goto cleanup; } default: - /* delete zero length downloads */ - if(fstat(fileno(payload->localf), &st) == 0 && st.st_size == 0) { - payload->unlink_on_fail = 1; - } if(!payload->errors_ok) { handle->pm_errno = ALPM_ERR_LIBCURL; _alpm_log(handle, ALPM_LOG_ERROR, @@ -547,6 +560,10 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, (*active_downloads_num)++; return 2; } else { + /* delete zero length downloads */ + if(fstat(fileno(payload->localf), &st) == 0 && st.st_size == 0) { + payload->unlink_on_fail = 1; + } goto cleanup; } } diff --git a/src/pacman/callback.c b/src/pacman/callback.c index ae7e1bed..1881f8ac 100644 --- a/src/pacman/callback.c +++ b/src/pacman/callback.c @@ -994,6 +994,34 @@ static void dload_progress_event(const char *filename, alpm_download_event_progr fflush(stdout); } +/* download retried */ +static void dload_retry_event(const char *filename, alpm_download_event_retry_t *data) { + if(!dload_progressbar_enabled()) { + return; + } + + int index; + struct pacman_progress_bar *bar; + bool ok = find_bar_for_filename(filename, &index, &bar); + assert(ok); + + if(!data->resume) { + if(total_enabled) { + /* note total download does not reflect partial downloads that are restarted */ + totalbar->xfered -= bar->xfered; + } + } + + bar->xfered = 0; + bar->total_size = 0; + bar->init_time = get_time_ms(); + bar->sync_time = 0; + bar->sync_xfered = 0; + bar->rate = 0.0; + bar->eta = 0.0; +} + + /* download completed */ static void dload_complete_event(const char *filename, alpm_download_event_completed_t *data) { @@ -1090,6 +1118,8 @@ void cb_download(void *ctx, const char *filename, alpm_download_event_type_t eve dload_init_event(filename, data); } else if(event == ALPM_DOWNLOAD_PROGRESS) { dload_progress_event(filename, data); + } else if(event == ALPM_DOWNLOAD_RETRY) { + dload_retry_event(filename, data); } else if(event == ALPM_DOWNLOAD_COMPLETED) { dload_complete_event(filename, data); } else { -- 2.31.1

1 0

[pacman-dev] [PATCH] libalpm: check for duplicate filenames
by morganamilo 09 May '21

09 May '21

This partially fixes FS#67850 It fixes the case for -S'ing packages but not -U'ing urls. morganamilo@Octavia dup ~git/pacman % sudo ./build/pacman -S a/a b/b resolving dependencies... error: a and b have the same filename: a-1-1-any.pkg.tar.zst error: failed to prepare transaction (duplicate filename) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 833df829..ccf2edc2 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -257,6 +257,8 @@ typedef enum _alpm_errno_t { ALPM_ERR_TRANS_NULL, /** Duplicate target in transaction */ ALPM_ERR_TRANS_DUP_TARGET, + /** Duplicate filename in transaction */ + ALPM_ERR_TRANS_DUP_FILENAME, /** A transaction has not been initialized */ ALPM_ERR_TRANS_NOT_INITIALIZED, /** Transaction has not been prepared */ diff --git a/lib/libalpm/error.c b/lib/libalpm/error.c index c6f19599..c06d091a 100644 --- a/lib/libalpm/error.c +++ b/lib/libalpm/error.c @@ -90,6 +90,8 @@ const char SYMEXPORT *alpm_strerror(alpm_errno_t err) return _("transaction not initialized"); case ALPM_ERR_TRANS_DUP_TARGET: return _("duplicate target"); + case ALPM_ERR_TRANS_DUP_FILENAME: + return _("duplicate filename"); case ALPM_ERR_TRANS_NOT_INITIALIZED: return _("transaction not initialized"); case ALPM_ERR_TRANS_NOT_PREPARED: diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 7fa50a13..73e9e77d 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -464,6 +464,25 @@ int _alpm_sync_prepare(alpm_handle_t *handle, alpm_list_t **data) } } + /* Ensure two packages don't have the same filename */ + for(i = resolved; i; i = i->next) { + alpm_pkg_t *pkg1 = i->data; + for(j = i->next; j; j = j->next) { + alpm_pkg_t *pkg2 = j->data; + if(strcmp(pkg1->filename, pkg2->filename) == 0) { + alpm_list_free(resolved); + ret = -1; + handle->pm_errno = ALPM_ERR_TRANS_DUP_FILENAME; + _alpm_log(handle, ALPM_LOG_ERROR, _("%s and %s have the same filename: %s\n"), + pkg1->name, pkg2->name, pkg1->filename); + } + } + } + + if(ret != 0) { + goto cleanup; + } + /* Set DEPEND reason for pulled packages */ for(i = resolved; i; i = i->next) { alpm_pkg_t *pkg = i->data; diff --git a/lib/libalpm/trans.c b/lib/libalpm/trans.c index c6ec7eea..4cee603c 100644 --- a/lib/libalpm/trans.c +++ b/lib/libalpm/trans.c @@ -122,7 +122,7 @@ int SYMEXPORT alpm_trans_prepare(alpm_handle_t *handle, alpm_list_t **data) /* pm_errno is set by _alpm_remove_prepare() */ return -1; } - } else { + } else { if(_alpm_sync_prepare(handle, data) == -1) { /* pm_errno is set by _alpm_sync_prepare() */ return -1; -- 2.31.1

2 1

[pacman-dev] [PATCH] libalpm: download rates becoming negative
by morganamilo 09 May '21

09 May '21

When a download fails on one mirror a new download is started on the next mirror. This new download will have the initial size of whatever has been downloaded so far, as well as the ammount downloaded reset to 0. To account for this, when a download changes mirror, save how much has been downloaded so far and add that to dlcb calls. --- lib/libalpm/dload.c | 14 ++++++++++++-- lib/libalpm/dload.h | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index a4c42f8d..27a7748a 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -207,8 +207,8 @@ static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow, /* do NOT include initial_size since it wasn't part of the package's * download_size (nor included in the total download size callback) */ - cb_data.total = dltotal; - cb_data.downloaded = dlnow; + cb_data.total = dltotal + payload->resetprogress; + cb_data.downloaded = dlnow + payload->resetprogress; payload->handle->dlcb(payload->remote_name, ALPM_DOWNLOAD_PROGRESS, &cb_data); payload->prevprogress = current_size; @@ -440,6 +440,16 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload fseek(payload->localf, 0, SEEK_SET); } + + /* reset progress for next server */ + payload->resetprogress += payload->prevprogress - payload->initial_size; + payload->unlink_on_fail = 0; + + if(payload->allow_resume) { + payload->initial_size = payload->prevprogress; + } + + /* Set curl with the new URL */ curl_easy_setopt(curl, CURLOPT_URL, payload->fileurl); diff --git a/lib/libalpm/dload.h b/lib/libalpm/dload.h index 8f3d17b4..abfd9466 100644 --- a/lib/libalpm/dload.h +++ b/lib/libalpm/dload.h @@ -42,6 +42,7 @@ struct dload_payload { off_t initial_size; off_t max_size; off_t prevprogress; + off_t resetprogress; /* progress made before we try a new server */ int force; int allow_resume; int errors_ok; -- 2.31.1

5 15

[pacman-dev] [PATCH v3] libalpm: clone data on alpm_db_set_servers
by morganamilo 09 May '21

09 May '21

Every alpm_option_set function clones the input so lets be more consistent. Also this fixes servers not being sanatized. --- v2: fix copy paste error --- lib/libalpm/alpm.h | 4 +--- lib/libalpm/db.c | 8 +++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 65b8e2fb..e6aefd4c 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -1312,9 +1312,7 @@ alpm_list_t *alpm_db_get_servers(const alpm_db_t *db); /** Sets the list of servers for the database to use. * @param db the database to set the servers - * @param servers a char* list of servers. Note: the database will - * take ownership of the list and it should no longer be - * freed by the caller + * @param servers a char* list of servers. */ int alpm_db_set_servers(alpm_db_t *db, alpm_list_t *servers); diff --git a/lib/libalpm/db.c b/lib/libalpm/db.c index 8511bb83..b8d1b157 100644 --- a/lib/libalpm/db.c +++ b/lib/libalpm/db.c @@ -139,9 +139,15 @@ alpm_list_t SYMEXPORT *alpm_db_get_servers(const alpm_db_t *db) int SYMEXPORT alpm_db_set_servers(alpm_db_t *db, alpm_list_t *servers) { + alpm_list_t *i; ASSERT(db != NULL, return -1); FREELIST(db->servers); - db->servers = servers; + for(i = servers; i; i = i->next) { + char *url = i->data; + if(alpm_db_add_server(db, url) != 0) { + return -1; + } + } return 0; } -- 2.31.1

1 0

[pacman-dev] [PATCH 0/1] signhandler add SIGQUIT soft_interrupt_handler
by ZheNing Hu 09 May '21

09 May '21

Hello, eveyone! I'am a newbie, because of the experience of using pacman downloading with the `Ctrl +\` exit was not very good,it leave the database lock, so I am thinking can we just deal with SIGQUIT rather than doing nothing. Hopefully this will improve the experience of users. Thanks. ZheNing Hu (1): signhandler: add SIGQUIT soft_interrupt_handler src/pacman/sighandler.c | 3 +++ 1 file changed, 3 insertions(+) -- 2.31.0

3 3

[pacman-dev] [PATCH] libalpm: clone data on alpm_db_set_servers
by morganamilo 09 May '21

09 May '21

Every alpm_option_set function clones the input so lets be more consistent. Also this fixes servers not being sanatized. --- lib/libalpm/db.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/libalpm/db.c b/lib/libalpm/db.c index 8511bb83..6ddf5461 100644 --- a/lib/libalpm/db.c +++ b/lib/libalpm/db.c @@ -139,9 +139,16 @@ alpm_list_t SYMEXPORT *alpm_db_get_servers(const alpm_db_t *db) int SYMEXPORT alpm_db_set_servers(alpm_db_t *db, alpm_list_t *servers) { + alpm_list_t *i; ASSERT(db != NULL, return -1); FREELIST(db->servers); - db->servers = servers; + for(i = servers; i; i = i->next) { + char *url = i->data; + if(alpm_db_add_server(db, url) != 0) { + return -1; + } + servers = servers->next; + } return 0; } -- 2.31.1

3 3

[pacman-dev] [PATCH] remove unused variable in meson.build
by morganamilo 07 May '21

07 May '21

--- meson.build | 1 - 1 file changed, 1 deletion(-) diff --git a/meson.build b/meson.build index c8835ab6..46f38de6 100644 --- a/meson.build +++ b/meson.build @@ -28,7 +28,6 @@ LIBMAKEPKGDIR = join_paths(PREFIX, DATAROOTDIR, 'makepkg') PKGDATADIR = join_paths(PREFIX, DATAROOTDIR, meson.project_name()) PYTHON = import('python').find_installation('python3') -DU = find_program('du') LDCONFIG = get_option('ldconfig') MESON_MAKE_SYMLINK = join_paths(meson.source_root(), 'build-aux/meson-make-symlink.sh') -- 2.31.1

1 0

[pacman-dev] [GIT] The official pacman repository branch, master, updated. v6.0.0alpha1-107-gef14c338
by Allan McRae 03 May '21

03 May '21

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "The official pacman repository". The branch, master has been updated via ef14c3380da0ed0e3eeba7e8b6c66294db316883 (commit) from 72238aa046b8a8de1c7b9110726986c7538023b5 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit ef14c3380da0ed0e3eeba7e8b6c66294db316883 Author: morganamilo <morganamilo(a)archlinux.org> Date: Sat May 1 21:25:35 2021 +0100 libalpm: fix error when installing cached packages without signatures With a repo using "SigLevel = Optional" and a package already downloaded into the cache, download_files() returns 1 (via _alpm_download) to indicate no files were downloaded. This causes installation of the package to fail. Explicitly check that download_files() returns -1 (error) rather than non-zero. Signed-off-by: Allan McRae <allan(a)archlinux.org> ----------------------------------------------------------------------- Summary of changes: lib/libalpm/sync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) hooks/post-receive -- The official pacman repository

1 0

[pacman-dev] (no subject)
by Jazmine Breaux 03 May '21

03 May '21

1 0

[pacman-dev] [PATCH v2] libalpm: fix error when downloading up to date files
by morganamilo 03 May '21

03 May '21

--- lib/libalpm/sync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 2419cc69..bdfab635 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -1128,7 +1128,7 @@ int _alpm_sync_load(alpm_handle_t *handle, alpm_list_t **data) uint64_t total_bytes = 0; alpm_trans_t *trans = handle->trans; - if(download_files(handle)) { + if(download_files(handle) == -1) { return -1; } -- 2.31.1

2 1