[pacman-dev] [PATCH] Support parallel download with xfercommand
lesto fante
lestofante88 at gmail.com
Tue Oct 20 14:21:46 UTC 2020
The idea here is that if XferLockCommand is set, XferCommand can be
non-blocking; all invocations of XferCommand are executed and only
then XferLockCommand is executed to wait for all the downloads to
complete.
I am also thinking of changing XferLockCommand to XferWaitCommand, if
you have better names please let me know.
I will fix and resubmit.
BTW i noticed that master fails some check, is it safe to be used for
testing (in a virtual machine) or should i use a specific commit?
Il giorno mar 20 ott 2020 alle ore 04:01 Eli Schwartz
<eschwartz at archlinux.org> ha scritto:
>
> On 10/19/20 7:19 PM, lesto wrote:
> > Signed-off-by: lesto <lestofante88 at gmail.com>
> > ---
> > lib/libalpm/alpm.h | 10 ++++++++++
> > lib/libalpm/dload.c | 9 +++++++++
> > lib/libalpm/handle.c | 13 +++++++++++++
> > lib/libalpm/handle.h | 1 +
> > src/pacman/conf.c | 5 ++++-
> > src/pacman/conf.h | 1 +
> > src/pacman/pacman-conf.c | 3 +++
> > 7 files changed, 41 insertions(+), 1 deletion(-)
>
>
> Your patch never even reads the XferLockCommand key to store its value
> in "config". It's not clear what this is intended to do or how it
> functions. Trivial check: try adding "XferLockCommand" to /etc/pacman.conf
>
> warning: config file /etc/pacman.conf, line 22: directive
> 'XferLockCommand' in section 'options' not recognized.
>
> Please fix this, and include an update to doc/pacman.conf.5.asciidoc
> describing the new key and how to use it.
>
>
> >
> > diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h
> > index 614a530c..c2af60e8 100644
> > --- a/lib/libalpm/alpm.h
> > +++ b/lib/libalpm/alpm.h
> > @@ -755,6 +755,11 @@ typedef void (*alpm_cb_totaldl)(off_t total);
> > typedef int (*alpm_cb_fetch)(const char *url, const char *localpath,
> > int force);
> >
> > +/** A callback for waiting for download of files
> > + * @return 0 on success, -1 on error.
> > + */
> > +typedef int (*alpm_cb_fetch_lock)(void);
> > +
> > /** Fetch a list of remote packages.
> > * @param handle the context handle
> > * @param urls list of package URLs to download
> > @@ -787,6 +792,11 @@ alpm_cb_fetch alpm_option_get_fetchcb(alpm_handle_t *handle);
> > /** Sets the downloading callback. */
> > int alpm_option_set_fetchcb(alpm_handle_t *handle, alpm_cb_fetch cb);
> >
> > +/** Returns the downloading lock callback. */
> > +alpm_cb_fetch_lock alpm_option_get_fetch_lockcb(alpm_handle_t *handle);
> > +/** Sets the downloading lock callback. */
> > +int alpm_option_set_fetch_lockcb(alpm_handle_t *handle, alpm_cb_fetch_lock cb);
> > +
> > /** Returns the callback used to report total download size. */
> > alpm_cb_totaldl alpm_option_get_totaldlcb(alpm_handle_t *handle);
> > /** Sets the callback used to report total download size. */
> > diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c
> > index 673e769f..174d559d 100644
> > --- a/lib/libalpm/dload.c
> > +++ b/lib/libalpm/dload.c
> > @@ -824,6 +824,15 @@ int _alpm_download(alpm_handle_t *handle,
> > RET_ERR(handle, ALPM_ERR_EXTERNAL_DOWNLOAD, -1);
> > }
> > }
> > +
> > + if (handle->fetch_lockcb != NULL) {
> > + // if fetch_lockcb is set, fetchcb is non-blocking; here we wait for all download to complete
> > + int ret = handle->fetch_lockcb();
> > + if (ret == -1) {
> > + RET_ERR(handle, ALPM_ERR_EXTERNAL_DOWNLOAD, -1);
> > + }
> > + }
> > +
> > return 0;
> > }
> > }
> > diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c
> > index 1310601a..683e678d 100644
> > --- a/lib/libalpm/handle.c
> > +++ b/lib/libalpm/handle.c
> > @@ -174,6 +174,12 @@ alpm_cb_fetch SYMEXPORT alpm_option_get_fetchcb(alpm_handle_t *handle)
> > return handle->fetchcb;
> > }
> >
> > +alpm_cb_fetch_lock SYMEXPORT alpm_option_get_fetch_lockcb(alpm_handle_t *handle)
> > +{
> > + CHECK_HANDLE(handle, return NULL);
> > + return handle->fetch_lockcb;
> > +}
> > +
> > alpm_cb_totaldl SYMEXPORT alpm_option_get_totaldlcb(alpm_handle_t *handle)
> > {
> > CHECK_HANDLE(handle, return NULL);
> > @@ -321,6 +327,13 @@ int SYMEXPORT alpm_option_set_fetchcb(alpm_handle_t *handle, alpm_cb_fetch cb)
> > return 0;
> > }
> >
> > +int SYMEXPORT alpm_option_set_fetch_lockcb(alpm_handle_t *handle, alpm_cb_fetch_lock cb)
> > +{
> > + CHECK_HANDLE(handle, return -1);
> > + handle->fetch_lockcb = cb;
> > + return 0;
> > +}
> > +
> > int SYMEXPORT alpm_option_set_totaldlcb(alpm_handle_t *handle, alpm_cb_totaldl cb)
> > {
> > CHECK_HANDLE(handle, return -1);
> > diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h
> > index 9fef0fbf..dc00751b 100644
> > --- a/lib/libalpm/handle.h
> > +++ b/lib/libalpm/handle.h
> > @@ -73,6 +73,7 @@ struct __alpm_handle_t {
> > alpm_cb_download dlcb; /* Download callback function */
> > alpm_cb_totaldl totaldlcb; /* Total download callback function */
> > alpm_cb_fetch fetchcb; /* Download file callback function */
> > + alpm_cb_fetch_lock fetch_lockcb; /* Download lock file callback function */
> > alpm_cb_event eventcb;
> > alpm_cb_question questioncb;
> > alpm_cb_progress progresscb;
> > diff --git a/src/pacman/conf.c b/src/pacman/conf.c
> > index 3a3ef605..53de73b8 100644
> > --- a/src/pacman/conf.c
> > +++ b/src/pacman/conf.c
> > @@ -157,6 +157,7 @@ int config_free(config_t *oldconfig)
> > FREELIST(oldconfig->hookdirs);
> > FREELIST(oldconfig->cachedirs);
> > free(oldconfig->xfercommand);
> > + free(oldconfig->xferlockcommand);
> > free(oldconfig->print_format);
> > free(oldconfig->arch);
> > wordsplit_free(oldconfig->xfercommand_argv);
> > @@ -319,7 +320,9 @@ static int download_with_xfercommand(const char *url, const char *localpath,
> > for(i = 0; i <= config->xfercommand_argc; i++) {
> > const char *val = config->xfercommand_argv[i];
> > if(val && strcmp(val, "%o") == 0) {
> > - usepart = 1;
> > + if (config->xferlockcommand == NULL) {
> > + usepart = 1;
> > + }
> > val = tempfile;
> > } else if(val && strcmp(val, "%u") == 0) {
> > val = url;
> > diff --git a/src/pacman/conf.h b/src/pacman/conf.h
> > index b8a451ad..1a9d637d 100644
> > --- a/src/pacman/conf.h
> > +++ b/src/pacman/conf.h
> > @@ -130,6 +130,7 @@ typedef struct __config_t {
> > char *xfercommand;
> > char **xfercommand_argv;
> > size_t xfercommand_argc;
> > + char *xferlockcommand;
> >
> > /* our connection to libalpm */
> > alpm_handle_t *handle;
> > diff --git a/src/pacman/pacman-conf.c b/src/pacman/pacman-conf.c
> > index 463badf1..7c4f4cc9 100644
> > --- a/src/pacman/pacman-conf.c
> > +++ b/src/pacman/pacman-conf.c
> > @@ -259,6 +259,7 @@ static void dump_config(void)
> >
> > show_str("Architecture", config->arch);
> > show_str("XferCommand", config->xfercommand);
> > + show_str("XferLockCommand", config->xferlockcommand);
> >
> > show_bool("UseSyslog", config->usesyslog);
> > show_bool("Color", config->color);
> > @@ -366,6 +367,8 @@ static int list_directives(void)
> > show_str("Architecture", config->arch);
> > } else if(strcasecmp(i->data, "XferCommand") == 0) {
> > show_str("XferCommand", config->xfercommand);
> > + } else if(strcasecmp(i->data, "XferLockCommand") == 0) {
> > + show_str("XferLockCommand", config->xferlockcommand);
> >
> > } else if(strcasecmp(i->data, "UseSyslog") == 0) {
> > show_bool("UseSyslog", config->usesyslog);
> >
>
>
> --
> Eli Schwartz
> Bug Wrangler and Trusted User
>
More information about the pacman-dev
mailing list