April 2020 - Pacman-dev - lists.archlinux.org

[pacman-dev] [GIT] The official pacman repository branch, master, updated. v5.2.1-61-g969e1ab9
by Allan McRae 28 Apr '20

28 Apr '20

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "The official pacman repository". The branch, master has been updated via 969e1ab996e79f3e121d0fb3bfd4e91ac7d6f488 (commit) via 24392223752480b9088321ad0f914118d0346e55 (commit) from 3ea7fdd96c874e111588a6e64ee30b75b55db785 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 969e1ab996e79f3e121d0fb3bfd4e91ac7d6f488 Author: Ivy Foster <escondida(a)iff.ink> Date: Thu Apr 23 13:12:03 2020 -0500 Add NoProgressbar to pacman.conf options This is useful for dumb terminals that do not support escape sequences. Signed-off-by: Ivy Foster <escondida(a)iff.ink> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 24392223752480b9088321ad0f914118d0346e55 Author: Rikard Falkeborn <rikard.falkeborn(a)gmail.com> Date: Fri Apr 24 23:43:58 2020 +0200 Constify some input pointers Signed-off-by: Rikard Falkeborn <rikard.falkeborn(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> ----------------------------------------------------------------------- Summary of changes: doc/pacman.conf.5.asciidoc | 4 ++++ etc/pacman.conf.in | 1 + lib/libalpm/diskspace.c | 2 +- lib/libalpm/diskspace.h | 2 +- lib/libalpm/util.c | 2 +- src/pacman/conf.c | 2 ++ src/pacman/pacman-conf.c | 3 +++ src/pacman/util.c | 2 +- 8 files changed, 14 insertions(+), 4 deletions(-) hooks/post-receive -- The official pacman repository

1 0

[pacman-dev] (no subject)
by Carson Black 28 Apr '20

28 Apr '20

This asciicast demonstrates the output of this patch: https://asciinema.org/a/321160

4 13

[pacman-dev] [PATCH v2] Add NoProgressbar to pacman.conf options
by escondida＠iff.ink 28 Apr '20

28 Apr '20

From: Ivy Foster <escondida(a)iff.ink> This is useful for dumb terminals that do not support escape sequences. Signed-off-by: Ivy Foster <escondida(a)iff.ink> --- doc/pacman.conf.5.asciidoc | 4 ++++ etc/pacman.conf.in | 1 + src/pacman/conf.c | 2 ++ src/pacman/pacman-conf.c | 3 +++ 4 files changed, 10 insertions(+) diff --git a/doc/pacman.conf.5.asciidoc b/doc/pacman.conf.5.asciidoc index b297e332..ecbbd8b3 100644 --- a/doc/pacman.conf.5.asciidoc +++ b/doc/pacman.conf.5.asciidoc @@ -186,6 +186,10 @@ Options *Color*:: Automatically enable colors only when pacman's output is on a tty. +*NoProgressBar*:: + Disables progressbars. This is useful for terminals which do + not support escape characters. + *TotalDownload*:: When downloading, display the amount downloaded, download rate, ETA, and completed percentage of the entire download list rather diff --git a/etc/pacman.conf.in b/etc/pacman.conf.in index 7446944f..e3c73e13 100644 --- a/etc/pacman.conf.in +++ b/etc/pacman.conf.in @@ -31,6 +31,7 @@ Architecture = auto # Misc options #UseSyslog #Color +#NoProgressBar #TotalDownload CheckSpace #VerbosePkgLists diff --git a/src/pacman/conf.c b/src/pacman/conf.c index f9de386f..e5f37b1a 100644 --- a/src/pacman/conf.c +++ b/src/pacman/conf.c @@ -579,6 +579,8 @@ static int _parse_options(const char *key, char *value, config->color = isatty(fileno(stdout)) ? PM_COLOR_ON : PM_COLOR_OFF; enable_colors(config->color); } + } else if(strcmp(key, "NoProgressBar") == 0) { + config->noprogressbar = 1; } else if(strcmp(key, "DisableDownloadTimeout") == 0) { config->disable_dl_timeout = 1; } else { diff --git a/src/pacman/pacman-conf.c b/src/pacman/pacman-conf.c index 6e25d243..a6f7362b 100644 --- a/src/pacman/pacman-conf.c +++ b/src/pacman/pacman-conf.c @@ -259,6 +259,7 @@ static void dump_config(void) show_bool("VerbosePkgLists", config->verbosepkglists); show_bool("DisableDownloadTimeout", config->disable_dl_timeout); show_bool("ILoveCandy", config->chomp); + show_bool("NoProgressBar", config->noprogressbar); show_cleanmethod("CleanMethod", config->cleanmethod); @@ -368,6 +369,8 @@ static int list_directives(void) show_bool("VerbosePkgLists", config->verbosepkglists); } else if(strcasecmp(i->data, "DisableDownloadTimeout") == 0) { show_bool("DisableDownloadTimeout", config->disable_dl_timeout); + } else if(strcasecmp(i->data, "NoProgressBar") == 0) { + show_bool("NoProgressBar", config->noprogressbar); } else if(strcasecmp(i->data, "CleanMethod") == 0) { show_cleanmethod("CleanMethod", config->cleanmethod); -- 2.26.2

2 1

[pacman-dev] [PATCH] Adds a "retry?" prompt for when transaction creation fails for user-friendlier fail support.
by Pedro 27 Apr '20

27 Apr '20

Signed-off-by: Pedro <pedrocga1(a)gmail.com> --- src/pacman/remove.c | 13 ++++++++++--- src/pacman/sync.c | 13 ++++++++++--- src/pacman/upgrade.c | 13 ++++++++++--- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/pacman/remove.c b/src/pacman/remove.c index 9920f502..f61c8652 100644 --- a/src/pacman/remove.c +++ b/src/pacman/remove.c @@ -87,9 +87,16 @@ int pacman_remove(alpm_list_t *targets) } /* Step 0: create a new transaction */ - if(trans_init(config->flags, 0) == -1) { - return 1; - } + while(1) { + if(trans_init(config->flags, 1) == -1) { + if(yesno(_("Retry?")) == 0) { + return 1; + } + /* Try again */ + continue; + } + break; + } /* Step 1: add targets to the created transaction */ for(i = targets; i; i = alpm_list_next(i)) { diff --git a/src/pacman/sync.c b/src/pacman/sync.c index f7dcb958..89f01061 100644 --- a/src/pacman/sync.c +++ b/src/pacman/sync.c @@ -694,9 +694,16 @@ static int sync_trans(alpm_list_t *targets) alpm_list_t *i; /* Step 1: create a new transaction... */ - if(trans_init(config->flags, 1) == -1) { - return 1; - } + while(1) { + if(trans_init(config->flags, 1) == -1) { + if(yesno(_("Retry?")) == 0) { + return 1; + } + /* Try again */ + continue; + } + break; + } /* process targets */ for(i = targets; i; i = alpm_list_next(i)) { diff --git a/src/pacman/upgrade.c b/src/pacman/upgrade.c index 5f984e44..406fec24 100644 --- a/src/pacman/upgrade.c +++ b/src/pacman/upgrade.c @@ -79,9 +79,16 @@ int pacman_upgrade(alpm_list_t *targets) } /* Step 1: create a new transaction */ - if(trans_init(config->flags, 1) == -1) { - retval = 1; - goto fail_free; + while(1) { + if(trans_init(config->flags, 1) == -1) { + if(yesno(_("Retry?")) == 0) { + retval = 1; + goto fail_free; + } + /* Try again */ + continue; + } + break; } printf(_("loading packages...\n")); -- 2.26.2

3 2

[pacman-dev] [PATCH] Add a utility function to check whether a file exists in the cache
by Anatol Pomozov 26 Apr '20

26 Apr '20

It is similar to _alpm_filecache_find() but does not return a dynamically allocated memory to user. Thus the user does not need to free this resource. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/util.c | 14 ++++++++++++++ lib/libalpm/util.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/lib/libalpm/util.c b/lib/libalpm/util.c index cb838e43..6307dc14 100644 --- a/lib/libalpm/util.c +++ b/lib/libalpm/util.c @@ -834,6 +834,20 @@ char *_alpm_filecache_find(alpm_handle_t *handle, const char *filename) return NULL; } +/** Check whether a filename exists in a registered alpm cachedir. + * @param handle the context handle + * @param filename name of file to find + * @return 0 if the filename was not found, 1 otherwise + */ +int _alpm_filecache_exists(alpm_handle_t *handle, const char *filename) +{ + int res; + char *fpath = _alpm_filecache_find(handle, filename); + res = !!fpath; + FREE(fpath); + return res; +} + /** Check the alpm cachedirs for existence and find a writable one. * If no valid cache directory can be found, use /tmp. * @param handle the context handle diff --git a/lib/libalpm/util.h b/lib/libalpm/util.h index 3306a022..4fc6e718 100644 --- a/lib/libalpm/util.h +++ b/lib/libalpm/util.h @@ -136,6 +136,8 @@ int _alpm_run_chroot(alpm_handle_t *handle, const char *cmd, char *const argv[], int _alpm_ldconfig(alpm_handle_t *handle); int _alpm_str_cmp(const void *s1, const void *s2); char *_alpm_filecache_find(alpm_handle_t *handle, const char *filename); +/* Checks whether a file exists in cache */ +int _alpm_filecache_exists(alpm_handle_t *handle, const char *filename); const char *_alpm_filecache_setup(alpm_handle_t *handle); /* Unlike many uses of alpm_pkgvalidation_t, _alpm_test_checksum expects * an enum value rather than a bitfield. */ -- 2.26.2

2 2

[pacman-dev] [PATCH v2] Add a utility function to check whether a file exists in the cache
by Anatol Pomozov 26 Apr '20

26 Apr '20

It is similar to _alpm_filecache_find() but does not return a dynamically allocated memory to user. Thus the user does not need to free this resource. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/util.c | 14 ++++++++++++++ lib/libalpm/util.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/lib/libalpm/util.c b/lib/libalpm/util.c index cb838e43..5c8c5ed6 100644 --- a/lib/libalpm/util.c +++ b/lib/libalpm/util.c @@ -834,6 +834,20 @@ char *_alpm_filecache_find(alpm_handle_t *handle, const char *filename) return NULL; } +/** Check whether a filename exists in a registered alpm cachedir. + * @param handle the context handle + * @param filename name of file to find + * @return 0 if the filename was not found, 1 otherwise + */ +int _alpm_filecache_exists(alpm_handle_t *handle, const char *filename) +{ + int res; + char *fpath = _alpm_filecache_find(handle, filename); + res = (fpath != NULL); + FREE(fpath); + return res; +} + /** Check the alpm cachedirs for existence and find a writable one. * If no valid cache directory can be found, use /tmp. * @param handle the context handle diff --git a/lib/libalpm/util.h b/lib/libalpm/util.h index 3306a022..4fc6e718 100644 --- a/lib/libalpm/util.h +++ b/lib/libalpm/util.h @@ -136,6 +136,8 @@ int _alpm_run_chroot(alpm_handle_t *handle, const char *cmd, char *const argv[], int _alpm_ldconfig(alpm_handle_t *handle); int _alpm_str_cmp(const void *s1, const void *s2); char *_alpm_filecache_find(alpm_handle_t *handle, const char *filename); +/* Checks whether a file exists in cache */ +int _alpm_filecache_exists(alpm_handle_t *handle, const char *filename); const char *_alpm_filecache_setup(alpm_handle_t *handle); /* Unlike many uses of alpm_pkgvalidation_t, _alpm_test_checksum expects * an enum value rather than a bitfield. */ -- 2.26.2

1 0

[pacman-dev] [PATCH] Constify some input pointers
by Rikard Falkeborn 26 Apr '20

26 Apr '20

Signed-off-by: Rikard Falkeborn <rikard.falkeborn(a)gmail.com> --- lib/libalpm/diskspace.c | 2 +- lib/libalpm/diskspace.h | 2 +- lib/libalpm/util.c | 2 +- src/pacman/util.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/libalpm/diskspace.c b/lib/libalpm/diskspace.c index 4cc17a12..2be98055 100644 --- a/lib/libalpm/diskspace.c +++ b/lib/libalpm/diskspace.c @@ -359,7 +359,7 @@ static int check_mountpoint(alpm_handle_t *handle, alpm_mountpoint_t *mp) } int _alpm_check_downloadspace(alpm_handle_t *handle, const char *cachedir, - size_t num_files, off_t *file_sizes) + size_t num_files, const off_t *file_sizes) { alpm_list_t *mount_points; alpm_mountpoint_t *cachedir_mp; diff --git a/lib/libalpm/diskspace.h b/lib/libalpm/diskspace.h index 4f7d00b9..5f17a878 100644 --- a/lib/libalpm/diskspace.h +++ b/lib/libalpm/diskspace.h @@ -58,6 +58,6 @@ typedef struct __alpm_mountpoint_t { int _alpm_check_diskspace(alpm_handle_t *handle); int _alpm_check_downloadspace(alpm_handle_t *handle, const char *cachedir, - size_t num_files, off_t *file_sizes); + size_t num_files, const off_t *file_sizes); #endif /* ALPM_DISKSPACE_H */ diff --git a/lib/libalpm/util.c b/lib/libalpm/util.c index cb838e43..ead03004 100644 --- a/lib/libalpm/util.c +++ b/lib/libalpm/util.c @@ -1006,7 +1006,7 @@ static int sha256_file(const char *path, unsigned char output[32]) * @return a NULL terminated string with the hexadecimal representation of * bytes or NULL on error. This string must be freed. */ -static char *hex_representation(unsigned char *bytes, size_t size) +static char *hex_representation(const unsigned char *bytes, size_t size) { static const char *hex_digits = "0123456789abcdef"; char *str; diff --git a/src/pacman/util.c b/src/pacman/util.c index 6693ec75..97b8e06d 100644 --- a/src/pacman/util.c +++ b/src/pacman/util.c @@ -515,7 +515,7 @@ void string_display(const char *title, const char *string, unsigned short cols) } static void table_print_line(const alpm_list_t *line, short col_padding, - size_t colcount, size_t *widths, int *has_data) + size_t colcount, const size_t *widths, const int *has_data) { size_t i; int need_padding = 0; -- 2.26.2

2 1

[pacman-dev] Dropping SIGPIPE workaround from dload.c
by Anatol Pomozov 20 Apr '20

20 Apr '20

Hi folks I am looking at the lib/libalpm/dload.c download codepath and found that it has a following snippet: /* Ignore any SIGPIPE signals. With libcurl, these shouldn't be happening, * but better safe than sorry. Store the old signal handler first. */ mask_signal(SIGPIPE, SIG_IGN, &orig_sig_pipe); dload_interrupted = 0; mask_signal(SIGINT, &inthandler, &orig_sig_int); My understanding that this code tries to handle DNS timeouts that are signaled as a SIGPIPE. And to avoid killing the whole app this code installs this sighandler. But my experiment (at my dev branch that removes the codesnippet above) shows that curl handles DNS timeout correctly: sudo pacman -U http://blackhole.webpagetest.org/foo.zst :: Retrieving packages... foo.zst failed to download foo.zst.sig failed to download error: failed retrieving file 'foo.zst' from blackhole.webpagetest.org : Connection timed out after 10000 milliseconds error: failed retrieving file 'foo.zst.sig' from blackhole.webpagetest.org : Connection timed out after 10000 milliseconds warning: failed to retrieve some files There is also a test that checks for SIGPIPE handling (test/pacman/tests/scriptlet-signal-reset.py) and it passes as well: 157/332 scriptlet-signal-reset.py OK 0.11832118034362793 s Which makes me believe that SIGPIPE handler trick is not needed and mCURL handles timeouts correctly. I've been using a branch without this handler for two months and never seen any signal related issues with my parallel-download pacman. I am thinking of *not* carrying this workaround to the new multiplexed function and want to check with you if you are OK with it.

2 1

[pacman-dev] [PATCH] Implement multibar UI
by Anatol Pomozov 17 Apr '20

17 Apr '20

Multiplexed download requires ability to draw UI for multiple active progress bars. To implement it we use ANSI codes to move cursor up/down. `active_dls` variable represents the list of active downloads. `struct dl_progress_bar` is a data structure for an active progress bar. In some cases we want to keep progress bars in order (e.g. db downloads). In some other cases (package downloads) we want to move completed items to the top of the screen. Global `multibar_move_complete_top` allows to configure such behavior. Optional downloads (such as optional signature files) do not have a visible progress bar. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/dload.c | 31 +--- lib/libalpm/dload.h | 1 - src/pacman/callback.c | 401 +++++++++++++++++++++++++++++------------- src/pacman/callback.h | 3 + src/pacman/sync.c | 2 + 5 files changed, 294 insertions(+), 144 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index c09abde4..cf467ff0 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -100,6 +100,11 @@ static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow, return 1; } + if(dlnow < 0 || dltotal <= 0 || dlnow > dltotal) { + /* bogus values : stop here */ + return 0; + } + current_size = payload->initial_size + dlnow; /* is our filesize still under any set limit? */ @@ -115,34 +120,15 @@ static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow, total_size = payload->initial_size + dltotal; - if(dltotal == 0 || payload->prevprogress == total_size) { + if(payload->prevprogress == total_size) { return 0; } - /* initialize the progress bar here to avoid displaying it when - * a repo is up to date and nothing gets downloaded. - * payload->handle->dlcb will receive the remote_name - * and the following arguments: - * 0, -1: download initialized - * 0, 0: non-download event - * x {x>0}, x: download complete - * x {x>0, x<y}, y {y > 0}: download progress, expected total is known */ - if(!payload->cb_initialized) { - cb_data.downloaded = 0; - cb_data.total = -1; - payload->cb_initialized = 1; - } - if(payload->prevprogress == current_size) { - cb_data.downloaded = 0; - cb_data.total = 0; - } else { /* do NOT include initial_size since it wasn't part of the package's * download_size (nor included in the total download size callback) */ - cb_data.downloaded = dlnow; - cb_data.total = dltotal; - } + cb_data.total = dltotal; + cb_data.downloaded = dlnow; payload->handle->dlcb(payload->remote_name, ALPM_DOWNLOAD_PROGRESS, &cb_data); - payload->prevprogress = current_size; return 0; @@ -1160,5 +1146,4 @@ void _alpm_dload_payload_reset_for_retry(struct dload_payload *payload) payload->initial_size += payload->prevprogress; payload->prevprogress = 0; payload->unlink_on_fail = 0; - payload->cb_initialized = 0; } diff --git a/lib/libalpm/dload.h b/lib/libalpm/dload.h index a40b51b7..142568e8 100644 --- a/lib/libalpm/dload.h +++ b/lib/libalpm/dload.h @@ -42,7 +42,6 @@ struct dload_payload { int errors_ok; int unlink_on_fail; int trust_remote_name; - int cb_initialized; #ifdef HAVE_LIBCURL CURL *curl; char error_buffer[CURL_ERROR_SIZE]; diff --git a/src/pacman/callback.c b/src/pacman/callback.c index 387319b6..a4aafada 100644 --- a/src/pacman/callback.c +++ b/src/pacman/callback.c @@ -18,6 +18,8 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <assert.h> +#include <stdbool.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -45,6 +47,8 @@ static off_t list_total = 0.0; static int on_progress = 0; static alpm_list_t *output = NULL; +static void cursor_goto_end(void); + /* update speed for the fill_progress based functions */ #define UPDATE_SPEED_MS 200 @@ -152,11 +156,7 @@ static void fill_progress(const int bar_percent, const int disp_percent, printf(" %3d%%", disp_percent); } - if(bar_percent == 100) { - putchar('\n'); - } else { - putchar('\r'); - } + putchar('\r'); fflush(stdout); } @@ -346,6 +346,7 @@ void cb_event(alpm_event_t *event) case ALPM_EVENT_DB_RETRIEVE_FAILED: case ALPM_EVENT_RETRIEVE_DONE: case ALPM_EVENT_RETRIEVE_FAILED: + cursor_goto_end(); flush_output_list(); on_progress = 0; break; @@ -629,6 +630,7 @@ void cb_progress(alpm_progress_t event, const char *pkgname, int percent, fill_progress(percent, percent, cols - infolen); if(percent == 100) { + putchar('\n'); flush_output_list(); on_progress = 0; } else { @@ -647,124 +649,121 @@ void cb_dl_total(off_t total) } } -/* callback to handle display of download progress */ -static void dload_progress_event(const char *filename, off_t file_xfered, off_t file_total) +static int dload_progressbar_enabled(void) { - static double rate_last; - static off_t xfered_last; - static int64_t initial_time = 0; - int infolen; - int filenamelen; - char *fname, *p; - /* used for wide character width determination and printing */ - int len, wclen, wcwid, padwid; - wchar_t *wcfname; + return !config->noprogressbar && getcols() != 0; +} - int totaldownload = 0; - off_t xfered, total; - double rate = 0.0; - unsigned int eta_h = 0, eta_m = 0, eta_s = 0; - double rate_human, xfered_human; - const char *rate_label, *xfered_label; - int file_percent = 0, total_percent = 0; +struct dl_progress_bar { + const char *filename; + off_t xfered; + off_t total_size; + uint64_t init_time; /* Time when this download started doing any progress */ + uint64_t sync_time; /* Last time we updated the bar info */ + double rate; + unsigned int eta; /* ETA in seconds */ + bool completed; /* transfer is completed */ + bool visible; /* whether to show the bar, some downloads (e.g. db.sig files) are not shown */ +}; - const unsigned short cols = getcols(); +/* List of active downloads handled by multibar UI. + * Once the first download in the list is completed it is removed + * from this list and we never redraw it anymore. + * If the download in this list then UI can redraw the progress bar or change + * the order of the bars (e.g. completed bar is moved to the top of the list) + */ +alpm_list_t *active_dls = NULL; +/* Number of active progress bars that multibar UI handles. + * Each progress bar has a corresponding active download. + * Note that some downloads might not have a bar, e.g. optional signature files + * do not have a corresponding bar. + */ +size_t active_bars_no = 0; - /* Nothing has changed since last callback; stop here */ - if(file_xfered == 0 && file_total == 0) { - return; - } +int multibar_move_complete_top = 0; - if(config->noprogressbar || cols == 0) { - if(file_xfered == 0 && file_total == -1) { - printf(_("downloading %s...\n"), filename); - fflush(stdout); - } - return; - } +/* Cursor position relative to the first active progress bar, + * e.g. 0 means the first active progress bar, active_bars_no-1 means the last bar, + * active_bars_no - is the line below all progress bars. + */ +int cursor_lineno = 0; - infolen = cols * 6 / 10; - if(infolen < 50) { - infolen = 50; - } - /* only use TotalDownload if enabled and we have a callback value */ - if(config->totaldownload && list_total) { - /* sanity check */ - if(list_xfered + file_total <= list_total) { - totaldownload = 1; - } else { - /* bogus values : don't enable totaldownload and reset */ - list_xfered = 0; - list_total = 0; - } +/* Move console cursor `lines` up */ +static void cursor_line_up(unsigned int lines) +{ + assert(lines > 0); + printf("\x1B[%dF", lines); +} + +/* Move console cursor `lines` down */ +static void cursor_line_down(unsigned int lines) +{ + assert(lines > 0); + printf("\x1B[%dE", lines); +} + +/* Erase line from the cursor position till the end of the line */ +static void console_erase_line(void) +{ + printf("\x1B[K"); +} + +/* Goto the line that corresponds to num-th active download */ +static void cursor_goto_bar(int num) +{ + if(num > cursor_lineno) { + cursor_line_down(num - cursor_lineno); + } else if(num < cursor_lineno) { + cursor_line_up(cursor_lineno - num); } + cursor_lineno = num; +} - if(totaldownload) { - xfered = list_xfered + file_xfered; - total = list_total; - } else { - xfered = file_xfered; - total = file_total; - } - - /* this is basically a switch on xfered: 0, total, and - * anything else */ - if(file_xfered == 0 && file_total == -1) { - /* set default starting values, ensure we only call this once - * if TotalDownload is enabled */ - if(!totaldownload || (totaldownload && list_xfered == 0)) { - initial_time = get_time_ms(); - xfered_last = (off_t)0; - rate_last = 0.0; - get_update_timediff(1); - } - } else if(xfered > total || xfered < 0) { - /* bogus values : stop here */ - return; - } else if(file_xfered == file_total) { - /* compute final values */ - int64_t timediff = get_time_ms() - initial_time; - if(timediff > 0) { - rate = (double)xfered / (timediff / 1000.0); - /* round elapsed time (in ms) to the nearest second */ - eta_s = (unsigned int)(timediff + 500) / 1000; - } else { - eta_s = 0; - } - } else { - /* compute current average values */ - int64_t timediff = get_update_timediff(0); +/* Goto the line *after* the last active progress bar */ +static void cursor_goto_end(void) +{ + cursor_goto_bar(active_bars_no); +} - if(timediff < UPDATE_SPEED_MS) { - /* return if the calling interval was too short */ - return; +/* Returns true if element with the specified name is found, false otherwise */ +static bool find_bar_for_filename(const char *filename, int *index, struct dl_progress_bar **bar) +{ + int i = 0; + alpm_list_t *listitem = active_dls; + for(; listitem; listitem = listitem->next) { + struct dl_progress_bar *b = listitem->data; + if (b->filename == filename /* or strcmp() ? */) { + *index = i; + *bar = b; + return true; } - rate = (double)(xfered - xfered_last) / (timediff / 1000.0); - /* average rate to reduce jumpiness */ - rate = (rate + 2 * rate_last) / 3; - if(rate > 0.0) { - eta_s = (total - xfered) / rate; - } else { - eta_s = UINT_MAX; + if(b->visible) { + i++; } - rate_last = rate; - xfered_last = xfered; } - if(file_total) { - file_percent = (file_xfered * 100) / file_total; - } else { - file_percent = 100; - } + return false; +} + +static void draw_dl_progress_bar(struct dl_progress_bar *bar) +{ + int infolen; + int filenamelen; + char *fname, *p; + /* used for wide character width determination and printing */ + int len, wclen, wcwid, padwid; + wchar_t *wcfname; + unsigned int eta_h = 0, eta_m = 0, eta_s = bar->eta; + double rate_human, xfered_human; + const char *rate_label, *xfered_label; + int file_percent = 0; - if(totaldownload) { - total_percent = ((list_xfered + file_xfered) * 100) / - list_total; + const unsigned short cols = getcols(); - /* if we are at the end, add the completed file to list_xfered */ - if(file_xfered == file_total) { - list_xfered += file_total; - } + if(bar->total_size) { + file_percent = (bar->xfered * 100) / bar->total_size; + } else { + file_percent = 100; } /* fix up time for display */ @@ -773,13 +772,13 @@ static void dload_progress_event(const char *filename, off_t file_xfered, off_t eta_m = eta_s / 60; eta_s -= eta_m * 60; - len = strlen(filename); + len = strlen(bar->filename); fname = malloc(len + 1); - memcpy(fname, filename, len + 1); + memcpy(fname, bar->filename, len + 1); /* strip package or DB extension for cleaner look */ if((p = strstr(fname, ".pkg")) || (p = strstr(fname, ".db")) || (p = strstr(fname, ".files"))) { /* tack on a .sig suffix for signatures */ - if(memcmp(&filename[len - 4], ".sig", 4) == 0) { + if(memcmp(&bar->filename[len - 4], ".sig", 4) == 0) { memcpy(p, ".sig", 4); /* adjust length for later calculations */ @@ -790,6 +789,11 @@ static void dload_progress_event(const char *filename, off_t file_xfered, off_t fname[len] = '\0'; } + infolen = cols * 6 / 10; + if(infolen < 50) { + infolen = 50; + } + /* 1 space + filenamelen + 1 space + 6 for size + 1 space + 3 for label + * + 2 spaces + 4 for rate + 1 space + 3 for label + 2 for /s + 1 space + * 8 for eta, gives us the magic 33 */ @@ -824,8 +828,8 @@ static void dload_progress_event(const char *filename, off_t file_xfered, off_t } - rate_human = humanize_size((off_t)rate, '\0', -1, &rate_label); - xfered_human = humanize_size(xfered, '\0', -1, &xfered_label); + rate_human = humanize_size((off_t)bar->rate, '\0', -1, &rate_label); + xfered_human = humanize_size(bar->xfered, '\0', -1, &xfered_label); printf(" %ls%-*s ", wcfname, padwid, ""); /* We will show 1.62 MiB/s, 11.6 MiB/s, but 116 KiB/s and 1116 KiB/s */ @@ -850,19 +854,176 @@ static void dload_progress_event(const char *filename, off_t file_xfered, off_t free(fname); free(wcfname); - if(totaldownload) { - fill_progress(file_percent, total_percent, cols - infolen); + fill_progress(file_percent, file_percent, cols - infolen); + return; +} + +static void dload_init_event(const char *filename, alpm_download_event_init_t *data) +{ + (void)data; + + if(!dload_progressbar_enabled()) { + printf(_(" %s downloading...\n"), filename); + return; + } + + struct dl_progress_bar *bar = calloc(1, sizeof(struct dl_progress_bar)); + if(!bar) { + return; + } + bar->filename = filename; + bar->init_time = get_time_ms(); + bar->rate = 0.0; + bar->visible = !data->optional; + active_dls = alpm_list_add(active_dls, bar); + + if(bar->visible) { + cursor_goto_end(); + printf(_(" %s downloading...\n"), filename); + cursor_lineno++; + active_bars_no++; + } +} + +/* render download progress */ +static void dload_progress_event(const char *filename, alpm_download_event_progress_t *data) +{ + int index; + struct dl_progress_bar *bar; + int64_t curr_time = get_time_ms(); + double last_chunk_rate; + int64_t timediff; + + if(!dload_progressbar_enabled()) { + return; + } + + assert(find_bar_for_filename(filename, &index, &bar)); + + /* compute current average values */ + timediff = curr_time - bar->sync_time; + + if(timediff < UPDATE_SPEED_MS) { + /* return if the calling interval was too short */ + return; + } + bar->sync_time = curr_time; + + last_chunk_rate = (double)(data->downloaded - bar->xfered) / (timediff / 1000.0); + /* average rate to reduce jumpiness */ + bar->rate = (last_chunk_rate + 2 * bar->rate) / 3; + if(bar->rate > 0.0) { + bar->eta = (data->total - data->downloaded) / bar->rate; } else { - fill_progress(file_percent, file_percent, cols - infolen); + bar->eta = UINT_MAX; + } + + /* Total size is received after the download starts. */ + bar->total_size = data->total; + bar->xfered = data->downloaded; + + if(bar->visible) { + cursor_goto_bar(index); + draw_dl_progress_bar(bar); + fflush(stdout); } - return; } +/* download completed */ +static void dload_complete_event(const char *filename, alpm_download_event_completed_t *data) +{ + int index; + struct dl_progress_bar *bar; + int64_t timediff; + + if(!dload_progressbar_enabled()) { + return; + } + + assert(find_bar_for_filename(filename, &index, &bar)); + bar->completed = true; + + /* This may not have been initialized if the download finished before + * an alpm_download_event_progress_t event happened */ + bar->total_size = data->total; + + if(bar->visible) { + if(data->result == 1) { + cursor_goto_bar(index); + printf(_(" %s is up to date"), bar->filename); + /* The line contains text from previous status. Erase these leftovers. */ + console_erase_line(); + } else if(data->result == 0) { + /* compute final values */ + bar->xfered = bar->total_size; + timediff = get_time_ms() - bar->init_time; + + /* if transfer was too fast, treat it as a 1ms transfer, for the sake + * of the rate calculation */ + if(timediff < 1) + timediff = 1; + + bar->rate = (double)bar->xfered / (timediff / 1000.0); + /* round elapsed time (in ms) to the nearest second */ + bar->eta = (unsigned int)(timediff + 500) / 1000; + + if(multibar_move_complete_top && index != 0) { + /* If this item completed then move it to the top. + * Swap 0-th bar data with `index`-th one + */ + struct dl_progress_bar *former_topbar = active_dls->data; + alpm_list_t *baritem = alpm_list_nth(active_dls, index); + active_dls->data = bar; + baritem->data = former_topbar; + + cursor_goto_bar(index); + draw_dl_progress_bar(former_topbar); + + index = 0; + } + + cursor_goto_bar(index); + draw_dl_progress_bar(bar); + } else { + cursor_goto_bar(index); + printf(_(" %s failed to download"), bar->filename); + console_erase_line(); + } + fflush(stdout); + } + + /* If the first bar is completed then there is no reason to keep it + * in the list as we are not going to redraw it anymore. + */ + while(active_dls) { + alpm_list_t *head = active_dls; + struct dl_progress_bar *j = head->data; + if(j->completed) { + if(j->visible) { + cursor_lineno--; + active_bars_no--; + } + active_dls = alpm_list_remove_item(active_dls, head); + free(head); + free(j); + } else { + break; + } + } +} + +/* callback to handle display of download progress */ void cb_download(const char *filename, alpm_download_event_type_t event, void *data) { - if(event == ALPM_DOWNLOAD_PROGRESS) { - alpm_download_event_progress_t *progress = data; - dload_progress_event(filename, progress->downloaded, progress->total); + if(event == ALPM_DOWNLOAD_INIT) { + dload_init_event(filename, data); + } else if(event == ALPM_DOWNLOAD_PROGRESS) { + dload_progress_event(filename, data); + } else if(event == ALPM_DOWNLOAD_COMPLETED) { + dload_complete_event(filename, data); + } else { + pm_printf(ALPM_LOG_ERROR, _("unknown callback event type %d for %s\n"), + event, filename); } } diff --git a/src/pacman/callback.h b/src/pacman/callback.h index 6d92e86b..916b9dad 100644 --- a/src/pacman/callback.h +++ b/src/pacman/callback.h @@ -44,4 +44,7 @@ void cb_download(const char *filename, alpm_download_event_type_t event, __attribute__((format(printf, 2, 0))) void cb_log(alpm_loglevel_t level, const char *fmt, va_list args); +/* specify if multibar should move complete bars to the top of the screen */ +int multibar_move_complete_top; + #endif /* PM_CALLBACK_H */ diff --git a/src/pacman/sync.c b/src/pacman/sync.c index f7dcb958..d5886f13 100644 --- a/src/pacman/sync.c +++ b/src/pacman/sync.c @@ -35,6 +35,7 @@ #include "pacman.h" #include "util.h" #include "package.h" +#include "callback.h" #include "conf.h" static int unlink_verbose(const char *pathname, int ignore_missing) @@ -824,6 +825,7 @@ int sync_prepare_execute(void) goto cleanup; } + multibar_move_complete_top = 1; if(alpm_trans_commit(config->handle, &data) == -1) { alpm_errno_t err = alpm_errno(config->handle); pm_printf(ALPM_LOG_ERROR, _("failed to commit transaction (%s)\n"), -- 2.26.1

1 1

[pacman-dev] [PATCH] Extend download callback interface with start/complete events
by Anatol Pomozov 16 Apr '20

16 Apr '20

With the previous download interface the callback uses the first progress event as 'download has started' signal. Unfortunately it does not work with up-to-date files that never receive 'download progress' events. Up-to-date database messages are currently handled in sync_syncdbs() after the sequential download is completed and a result from ALPM is received. But this is not going to work with multiplexed download interface that returns the result only after all files are completed. Another problem with 'first progress event is the beginning of the download' is that such events time are unpredictable. Thus the UI progress bar order might differ from what has been passed by client to alpm_dbs_update() function. We actually want to keep the dbs progress bars in a strict order. To help to solve the given problems extend the download callback to allow 2 more events - download started and completed. 'Download started' events appear in the same order as in the list given by a client. Signed-off-by: Anatol Pomozov <anatol.pomozov(a)gmail.com> --- lib/libalpm/alpm.h | 35 ++++++++++++++++++++++++++++++++--- lib/libalpm/dload.c | 19 ++++++++++++++----- src/pacman/callback.c | 10 +++++++++- src/pacman/callback.h | 3 ++- src/pacman/conf.c | 2 +- 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index aee5aa0d..8fe11a9b 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -715,13 +715,42 @@ typedef void (*alpm_cb_progress)(alpm_progress_t, const char *, int, size_t, siz * Downloading */ +/* File download events. + * These events are reported by ALPM via download callback. + */ +typedef enum { + ALPM_DOWNLOAD_INIT, /* alpm initializes file download logic */ + ALPM_DOWNLOAD_PROGRESS, /* download progress reported */ + ALPM_DOWNLOAD_COMPLETED /* alpm is about to release data related to the file */ +} alpm_download_event_type_t; + +typedef struct { + int optional; /* whether this file is optional and thus the errors could be ignored */ +} alpm_download_event_init_t; + +typedef struct { + off_t downloaded; /* amount of data downloaded */ + off_t total; /* total amount need to be downloaded */ +} alpm_download_event_progress_t; + +typedef struct { + /* total bytes in file */ + off_t total; + /* download result code: + * 0 - download completed successfully + * 1 - the file is up-to-date + * negative - error code + */ + int result; +} alpm_download_event_completed_t; + /** Type of download progress callbacks. * @param filename the name of the file being downloaded - * @param xfered the number of transferred bytes - * @param total the total number of bytes to transfer + * @param event the event type + * @param data the event data of type alpm_download_event_*_t */ typedef void (*alpm_cb_download)(const char *filename, - off_t xfered, off_t total); + alpm_download_event_type_t event, void *data); typedef void (*alpm_cb_totaldl)(off_t total); diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 786c11a1..c09abde4 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -88,6 +88,7 @@ static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow, { struct dload_payload *payload = (struct dload_payload *)file; off_t current_size, total_size; + alpm_download_event_progress_t cb_data = {0}; /* avoid displaying progress bar for redirects with a body */ if(payload->respcode >= 300) { @@ -127,16 +128,20 @@ static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow, * x {x>0}, x: download complete * x {x>0, x<y}, y {y > 0}: download progress, expected total is known */ if(!payload->cb_initialized) { - payload->handle->dlcb(payload->remote_name, 0, -1); + cb_data.downloaded = 0; + cb_data.total = -1; payload->cb_initialized = 1; } if(payload->prevprogress == current_size) { - payload->handle->dlcb(payload->remote_name, 0, 0); + cb_data.downloaded = 0; + cb_data.total = 0; } else { /* do NOT include initial_size since it wasn't part of the package's * download_size (nor included in the total download size callback) */ - payload->handle->dlcb(payload->remote_name, dlnow, dltotal); + cb_data.downloaded = dlnow; + cb_data.total = dltotal; } + payload->handle->dlcb(payload->remote_name, ALPM_DOWNLOAD_PROGRESS, &cb_data); payload->prevprogress = current_size; @@ -655,6 +660,7 @@ static int curl_multi_check_finished_download(CURLM *curlm, CURLMsg *msg, long remote_time = -1; struct stat st; char hostname[HOSTNAME_SIZE]; + alpm_download_event_completed_t cb_data = {0}; int ret = -1; curlerr = curl_easy_getinfo(curl, CURLINFO_PRIVATE, &payload); @@ -812,7 +818,9 @@ cleanup: unlink(payload->tempfile_name); } - // TODO: report that the download has been completed + cb_data.total = bytes_dl; + cb_data.result = ret; + handle->dlcb(payload->remote_name, ALPM_DOWNLOAD_COMPLETED, &cb_data); curl_multi_remove_handle(curlm, curl); curl_easy_cleanup(curl); @@ -920,9 +928,10 @@ static int curl_multi_download_internal(alpm_handle_t *handle, for(; still_running < parallel_downloads && payloads; still_running++) { struct dload_payload *payload = payloads->data; + alpm_download_event_init_t cb_data = {.optional = payload->errors_ok}; if(curl_multi_add_payload(handle, curlm, payloads->data, localpath) == 0) { - // TODO: report that download has started + handle->dlcb(payload->remote_name, ALPM_DOWNLOAD_INIT, &cb_data); payloads = payloads->next; } else { // the payload failed to start, do not start any new downloads just wait until diff --git a/src/pacman/callback.c b/src/pacman/callback.c index 585eae09..387319b6 100644 --- a/src/pacman/callback.c +++ b/src/pacman/callback.c @@ -648,7 +648,7 @@ void cb_dl_total(off_t total) } /* callback to handle display of download progress */ -void cb_dl_progress(const char *filename, off_t file_xfered, off_t file_total) +static void dload_progress_event(const char *filename, off_t file_xfered, off_t file_total) { static double rate_last; static off_t xfered_last; @@ -858,6 +858,14 @@ void cb_dl_progress(const char *filename, off_t file_xfered, off_t file_total) return; } +void cb_download(const char *filename, alpm_download_event_type_t event, void *data) +{ + if(event == ALPM_DOWNLOAD_PROGRESS) { + alpm_download_event_progress_t *progress = data; + dload_progress_event(filename, progress->downloaded, progress->total); + } +} + /* Callback to handle notifications from the library */ void cb_log(alpm_loglevel_t level, const char *fmt, va_list args) { diff --git a/src/pacman/callback.h b/src/pacman/callback.h index b4bde5f2..6d92e86b 100644 --- a/src/pacman/callback.h +++ b/src/pacman/callback.h @@ -37,7 +37,8 @@ void cb_progress(alpm_progress_t event, const char *pkgname, int percent, /* callback to handle receipt of total download value */ void cb_dl_total(off_t total); /* callback to handle display of download progress */ -void cb_dl_progress(const char *filename, off_t file_xfered, off_t file_total); +void cb_download(const char *filename, alpm_download_event_type_t event, + void *data); /* callback to handle messages/notifications from pacman library */ __attribute__((format(printf, 2, 0))) diff --git a/src/pacman/conf.c b/src/pacman/conf.c index 2dcf7a5b..5a450426 100644 --- a/src/pacman/conf.c +++ b/src/pacman/conf.c @@ -835,7 +835,7 @@ static int setup_libalpm(void) config->handle = handle; alpm_option_set_logcb(handle, cb_log); - alpm_option_set_dlcb(handle, cb_dl_progress); + alpm_option_set_dlcb(handle, cb_download); alpm_option_set_eventcb(handle, cb_event); alpm_option_set_questioncb(handle, cb_question); alpm_option_set_progresscb(handle, cb_progress); -- 2.26.1

1 0