[pacman-dev] [PATCH] libalpm: look for up to date file anywhere in cache
Currently when a package is in a secondary cachedir and there is no sig file, the downloader redownloads the whole file into the primary cachedir. So lets check all cachedirs for an up to date file. Fixes FS#71109 --- lib/libalpm/be_sync.c | 2 +- lib/libalpm/dload.c | 61 +++++++++++++++++++++++++++++-------------- lib/libalpm/dload.h | 3 ++- lib/libalpm/sync.c | 2 +- 4 files changed, 45 insertions(+), 23 deletions(-) diff --git a/lib/libalpm/be_sync.c b/lib/libalpm/be_sync.c index d85f36ee..f5f70a10 100644 --- a/lib/libalpm/be_sync.c +++ b/lib/libalpm/be_sync.c @@ -205,7 +205,7 @@ int SYMEXPORT alpm_db_update(alpm_handle_t *handle, alpm_list_t *dbs, int force) event.type = ALPM_EVENT_DB_RETRIEVE_START; EVENT(handle, &event); - ret = _alpm_download(handle, payloads, syncpath); + ret = _alpm_download(handle, payloads, syncpath, NULL); if(ret < 0) { event.type = ALPM_EVENT_DB_RETRIEVE_FAILED; EVENT(handle, &event); diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index 2c14841f..d5b52cd2 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -55,7 +55,7 @@ #define HOSTNAME_SIZE 256 static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, - struct dload_payload *payload, const char *localpath); + struct dload_payload *payload, const char *localpath, alpm_list_t *cachedirs); static int curl_gethost(const char *url, char *buffer, size_t buf_len); /* number of "soft" errors required to blacklist a server, set to 0 to disable @@ -308,7 +308,7 @@ static size_t dload_parseheader_cb(void *ptr, size_t size, size_t nmemb, void *u return realsize; } -static void curl_set_handle_opts(CURL *curl, struct dload_payload *payload) +static void curl_set_handle_opts(CURL *curl, struct dload_payload *payload, alpm_list_t *cachedirs) { alpm_handle_t *handle = payload->handle; const char *useragent = getenv("HTTP_USER_AGENT"); @@ -353,14 +353,33 @@ static void curl_set_handle_opts(CURL *curl, struct dload_payload *payload) curl_easy_setopt(curl, CURLOPT_USERAGENT, useragent); } - if(!payload->force && payload->destfile_name && - stat(payload->destfile_name, &st) == 0) { - /* start from scratch, but only download if our local is out of date. */ - curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); - curl_easy_setopt(curl, CURLOPT_TIMEVALUE, (long)st.st_mtime); - _alpm_log(handle, ALPM_LOG_DEBUG, - "%s: using time condition %ld\n", - payload->remote_name, (long)st.st_mtime); + if(!payload->force && payload->destfile_name) { + if(stat(payload->destfile_name, &st) == 0) { + /* start from scratch, but only download if our local is out of date. */ + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, (long)st.st_mtime); + _alpm_log(handle, ALPM_LOG_DEBUG, + "%s: using time condition %ld\n", + payload->remote_name, (long)st.st_mtime); + } else { + /* try and find an up to date file anywhere in cache */ + const char *name = get_filename(payload->destfile_name); + for(alpm_list_t *i = cachedirs; i; i = i->next) { + char *cachedir = i->data; + char *filepath = get_fullpath(cachedir, name, ""); + if(stat(filepath, &st) == 0) { + /* start from scratch, but only download if our local is out of date. */ + curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); + curl_easy_setopt(curl, CURLOPT_TIMEVALUE, (long)st.st_mtime); + _alpm_log(handle, ALPM_LOG_DEBUG, + "%s: using time condition %ld\n", + payload->remote_name, (long)st.st_mtime); + free(filepath); + break; + } + free(filepath); + } + } } else if(stat(payload->tempfile_name, &st) == 0 && payload->allow_resume) { /* a previous partial download exists, resume from end of file. */ payload->tempfile_openmode = "ab"; @@ -469,7 +488,7 @@ static int curl_retry_next_server(CURLM *curlm, CURL *curl, struct dload_payload * Returns -2 if an error happened for an optional file */ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, - const char *localpath, int *active_downloads_num) + const char *localpath, int *active_downloads_num, alpm_list_t *cachedirs) { alpm_handle_t *handle = NULL; struct dload_payload *payload = NULL; @@ -638,7 +657,7 @@ static int curl_check_finished_download(CURLM *curlm, CURLMsg *msg, /* set hard upper limit of 16KiB */ sig->max_size = 16 * 1024; - curl_add_payload(handle, curlm, sig, localpath); + curl_add_payload(handle, curlm, sig, localpath, cachedirs); (*active_downloads_num)++; } @@ -721,7 +740,7 @@ cleanup: * Returns -1 if am error happened while starting a new download */ static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, - struct dload_payload *payload, const char *localpath) + struct dload_payload *payload, const char *localpath, alpm_list_t *cachedirs) { size_t len; CURL *curl = NULL; @@ -778,7 +797,7 @@ static int curl_add_payload(alpm_handle_t *handle, CURLM *curlm, } } - curl_set_handle_opts(curl, payload); + curl_set_handle_opts(curl, payload, cachedirs); if(payload->max_size == payload->initial_size && payload->max_size != 0) { /* .part file is complete */ @@ -823,7 +842,8 @@ cleanup: */ static int curl_download_internal(alpm_handle_t *handle, alpm_list_t *payloads /* struct dload_payload */, - const char *localpath) + const char *localpath, + alpm_list_t *cachedirs) { int active_downloads_num = 0; int err = 0; @@ -837,7 +857,7 @@ static int curl_download_internal(alpm_handle_t *handle, for(; active_downloads_num < max_streams && payloads; active_downloads_num++) { struct dload_payload *payload = payloads->data; - if(curl_add_payload(handle, curlm, payload, localpath) == 0) { + if(curl_add_payload(handle, curlm, payload, localpath, cachedirs) == 0) { payloads = payloads->next; } else { /* The payload failed to start. Do not start any new downloads. @@ -868,7 +888,7 @@ static int curl_download_internal(alpm_handle_t *handle, } if(msg->msg == CURLMSG_DONE) { int ret = curl_check_finished_download(curlm, msg, - localpath, &active_downloads_num); + localpath, &active_downloads_num, cachedirs); if(ret == -1) { /* if current payload failed to download then stop adding new payloads but wait for the * current ones @@ -896,11 +916,12 @@ static int curl_download_internal(alpm_handle_t *handle, */ int _alpm_download(alpm_handle_t *handle, alpm_list_t *payloads /* struct dload_payload */, - const char *localpath) + const char *localpath, + alpm_list_t *cachedirs) { if(handle->fetchcb == NULL) { #ifdef HAVE_LIBCURL - return curl_download_internal(handle, payloads, localpath); + return curl_download_internal(handle, payloads, localpath, cachedirs); #else RET_ERR(handle, ALPM_ERR_EXTERNAL_DOWNLOAD, -1); #endif @@ -995,7 +1016,7 @@ int SYMEXPORT alpm_fetch_pkgurl(alpm_handle_t *handle, const alpm_list_t *urls, event.type = ALPM_EVENT_PKG_RETRIEVE_START; event.pkg_retrieve.num = alpm_list_count(payloads); EVENT(handle, &event); - if(_alpm_download(handle, payloads, cachedir) == -1) { + if(_alpm_download(handle, payloads, cachedir, handle->cachedirs) == -1) { _alpm_log(handle, ALPM_LOG_WARNING, _("failed to retrieve some files\n")); event.type = ALPM_EVENT_PKG_RETRIEVE_FAILED; EVENT(handle, &event); diff --git a/lib/libalpm/dload.h b/lib/libalpm/dload.h index 8f3d17b4..f5dabae1 100644 --- a/lib/libalpm/dload.h +++ b/lib/libalpm/dload.h @@ -61,6 +61,7 @@ void _alpm_dload_payload_reset(struct dload_payload *payload); int _alpm_download(alpm_handle_t *handle, alpm_list_t *payloads /* struct dload_payload */, - const char *localpath); + const char *localpath, + alpm_list_t *cachedirs); #endif /* ALPM_DLOAD_H */ diff --git a/lib/libalpm/sync.c b/lib/libalpm/sync.c index 36ad6242..a1226a4c 100644 --- a/lib/libalpm/sync.c +++ b/lib/libalpm/sync.c @@ -833,7 +833,7 @@ static int download_files(alpm_handle_t *handle) payloads = alpm_list_add(payloads, payload); } - ret = _alpm_download(handle, payloads, cachedir); + ret = _alpm_download(handle, payloads, cachedir, handle->cachedirs); if(ret == -1) { event.type = ALPM_EVENT_PKG_RETRIEVE_FAILED; EVENT(handle, &event); -- 2.32.0
On 15/6/21 1:42 am, morganamilo wrote:
Currently when a package is in a secondary cachedir and there is no sig file, the downloader redownloads the whole file into the primary cachedir.
So lets check all cachedirs for an up to date file.
Fixes FS#71109
So... this is a feature and not a bug. Sort of. After this patch, the .sig file ends up in the first cache and the package file in the second. If the signature file is not in the package database, pacman looks alongside the package to find the signature file and fails. Without this patch, databases without signatures work. Either, the .sig file needs downloaded into the secondary cache directory alongside the package, or the package directly copied into first cache directory. Allan
On 24/06/2021 14:42, Allan McRae wrote:
On 15/6/21 1:42 am, morganamilo wrote:
Currently when a package is in a secondary cachedir and there is no sig file, the downloader redownloads the whole file into the primary cachedir.
So lets check all cachedirs for an up to date file.
Fixes FS#71109
So... this is a feature and not a bug. Sort of.
After this patch, the .sig file ends up in the first cache and the package file in the second. If the signature file is not in the package database, pacman looks alongside the package to find the signature file and fails.
Without this patch, databases without signatures work.
And they don't with this patch? This patch shouldn't touch the database download code.
Either, the .sig file needs downloaded into the secondary cache directory alongside the package, or the package directly copied into first cache directory.
What if the cachedir is read only?
Allan
On 06/24/21 at 08:45pm, Morgan Adamiec wrote:
On 24/06/2021 14:42, Allan McRae wrote:
On 15/6/21 1:42 am, morganamilo wrote:
Currently when a package is in a secondary cachedir and there is no sig file, the downloader redownloads the whole file into the primary cachedir.
So lets check all cachedirs for an up to date file.
Fixes FS#71109
So... this is a feature and not a bug. Sort of.
After this patch, the .sig file ends up in the first cache and the package file in the second. If the signature file is not in the package database, pacman looks alongside the package to find the signature file and fails.
Without this patch, databases without signatures work.
And they don't with this patch? This patch shouldn't touch the database download code.
Not unsigned databases, databases that don't include the package signatures.
Either, the .sig file needs downloaded into the secondary cache directory alongside the package, or the package directly copied into first cache directory.
What if the cachedir is read only?
Then both files need to be put into the cache we're writing to.
On 6/24/21 9:42 AM, Allan McRae wrote:
Either, the .sig file needs downloaded into the secondary cache directory alongside the package, or the package directly copied into first cache directory.
Hmm, is there any reason we need them to be in the same directory, rather than reading the package from one cachedir and the sig data from another? -- Eli Schwartz Bug Wrangler and Trusted User
On 25/6/21 6:04 am, Eli Schwartz wrote:
On 6/24/21 9:42 AM, Allan McRae wrote:
Either, the .sig file needs downloaded into the secondary cache directory alongside the package, or the package directly copied into first cache directory.
Hmm, is there any reason we need them to be in the same directory, rather than reading the package from one cachedir and the sig data from another?
Technically, there is nothing stopping that. You would just need to remove the assumption that the signature file is stored alongside the package file throughout the database. And that does seem a reasonable assumption to keep... Allan
participants (5)
-
Allan McRae
-
Andrew Gregory
-
Eli Schwartz
-
Morgan Adamiec
-
morganamilo