[pacman-dev] [PATCH 00/13] Files database operations
This patchset adds support for reading the .files database and performing the following: - downloading/updating the .files database - checking which package contains a file - searching for a file (including regex support) - listing all files from a package Allan McRae (13): Simplify sync_cleandb Fix formatting in parsearg_upgrade Add files option to handle libalpm: handle both .db and .files extensions Read file lists from files database Add pacman support for .files databases pacman: move database syncing to util.c Prototype pacman files database operations Implement locating file owner in sync files database Implement listing files from sync packages Implement searching for a file in the sync databases Add regex search option to sync database file searching Handle repo/pkg style arguments to sync repo file listing lib/libalpm/alpm.h | 3 + lib/libalpm/be_sync.c | 61 +++++++++-- lib/libalpm/db.c | 10 +- lib/libalpm/handle.c | 13 +++ lib/libalpm/handle.h | 1 + src/pacman/Makefile.am | 1 + src/pacman/callback.c | 2 +- src/pacman/conf.c | 4 + src/pacman/conf.h | 6 +- src/pacman/files.c | 271 +++++++++++++++++++++++++++++++++++++++++++++++++ src/pacman/pacman.c | 88 +++++++++++++++- src/pacman/pacman.h | 2 + src/pacman/sync.c | 84 ++++----------- src/pacman/util.c | 33 ++++++ src/pacman/util.h | 1 + 15 files changed, 498 insertions(+), 82 deletions(-) create mode 100644 src/pacman/files.c -- 2.4.4
1) Remove checks for removing pre-tardb files 2) Remove the long redundant keep_used parameter 3) Fix pacman error due to removing .sig file along with database Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/sync.c | 43 ++++++++++++------------------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/src/pacman/sync.c b/src/pacman/sync.c index 0c3151e..c42c38d 100644 --- a/src/pacman/sync.c +++ b/src/pacman/sync.c @@ -53,7 +53,7 @@ static int unlink_verbose(const char *pathname, int ignore_missing) /* if keep_used != 0, then the db files which match an used syncdb * will be kept */ -static int sync_cleandb(const char *dbpath, int keep_used) +static int sync_cleandb(const char *dbpath) { DIR *dir; struct dirent *ent; @@ -77,18 +77,11 @@ static int sync_cleandb(const char *dbpath, int keep_used) const char *dname = ent->d_name; char *dbname; size_t len; + alpm_list_t *i; if(strcmp(dname, ".") == 0 || strcmp(dname, "..") == 0) { continue; } - /* skip the local and sync directories */ - if(strcmp(dname, "sync") == 0 || strcmp(dname, "local") == 0) { - continue; - } - /* skip the db.lck file */ - if(strcmp(dname, "db.lck") == 0) { - continue; - } /* build the full path */ snprintf(path, PATH_MAX, "%s%s", dbpath, dname); @@ -116,24 +109,16 @@ static int sync_cleandb(const char *dbpath, int keep_used) continue; } - if(keep_used) { - alpm_list_t *i; - for(i = syncdbs; i && !found; i = alpm_list_next(i)) { - alpm_db_t *db = i->data; - found = !strcmp(dbname, alpm_db_get_name(db)); - } + for(i = syncdbs; i && !found; i = alpm_list_next(i)) { + alpm_db_t *db = i->data; + found = !strcmp(dbname, alpm_db_get_name(db)); } - /* We have a database that doesn't match any syncdb. */ + /* We have a file that doesn't match any syncdb. */ if(!found) { - /* ENOENT check is because the signature and database could come in any - * order in our readdir() call, so either file may already be gone. */ - snprintf(path, PATH_MAX, "%s%s.db", dbpath, dbname); - ret += unlink_verbose(path, 1); - /* unlink a signature file if present too */ - snprintf(path, PATH_MAX, "%s%s.db.sig", dbpath, dbname); - ret += unlink_verbose(path, 1); + ret += unlink_verbose(path, 0); } + free(dbname); } closedir(dir); @@ -143,7 +128,7 @@ static int sync_cleandb(const char *dbpath, int keep_used) static int sync_cleandb_all(void) { const char *dbpath; - char *newdbpath; + char *syncdbpath; int ret = 0; dbpath = alpm_option_get_dbpath(config->handle); @@ -152,17 +137,13 @@ static int sync_cleandb_all(void) return 0; } printf(_("removing unused sync repositories...\n")); - /* The sync dbs were previously put in dbpath/ but are now in dbpath/sync/. - * We will clean everything in dbpath/ except local/, sync/ and db.lck, and - * only the unused sync dbs in dbpath/sync/ */ - ret += sync_cleandb(dbpath, 0); - if(asprintf(&newdbpath, "%s%s", dbpath, "sync/") < 0) { + if(asprintf(&syncdbpath, "%s%s", dbpath, "sync/") < 0) { ret += 1; return ret; } - ret += sync_cleandb(newdbpath, 1); - free(newdbpath); + ret += sync_cleandb(syncdbpath); + free(syncdbpath); return ret; } -- 2.4.4
Hi, On 20.06.2015 09:42, Allan McRae wrote:
1) Remove checks for removing pre-tardb files 2) Remove the long redundant keep_used parameter 3) Fix pacman error due to removing .sig file along with database
Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/sync.c | 43 ++++++++++++------------------------------- 1 file changed, 12 insertions(+), 31 deletions(-)
diff --git a/src/pacman/sync.c b/src/pacman/sync.c index 0c3151e..c42c38d 100644 --- a/src/pacman/sync.c +++ b/src/pacman/sync.c @@ -53,7 +53,7 @@ static int unlink_verbose(const char *pathname, int ignore_missing)
/* if keep_used != 0, then the db files which match an used syncdb * will be kept */ -static int sync_cleandb(const char *dbpath, int keep_used) +static int sync_cleandb(const char *dbpath)
You probably want to remove the comment above this, too.
{ DIR *dir; struct dirent *ent; @@ -77,18 +77,11 @@ static int sync_cleandb(const char *dbpath, int keep_used) const char *dname = ent->d_name; char *dbname; size_t len; + alpm_list_t *i;
if(strcmp(dname, ".") == 0 || strcmp(dname, "..") == 0) { continue; } - /* skip the local and sync directories */ - if(strcmp(dname, "sync") == 0 || strcmp(dname, "local") == 0) { - continue; - } - /* skip the db.lck file */ - if(strcmp(dname, "db.lck") == 0) { - continue; - }
/* build the full path */ snprintf(path, PATH_MAX, "%s%s", dbpath, dname); @@ -116,24 +109,16 @@ static int sync_cleandb(const char *dbpath, int keep_used) continue; }
- if(keep_used) { - alpm_list_t *i; - for(i = syncdbs; i && !found; i = alpm_list_next(i)) { - alpm_db_t *db = i->data; - found = !strcmp(dbname, alpm_db_get_name(db)); - } + for(i = syncdbs; i && !found; i = alpm_list_next(i)) { + alpm_db_t *db = i->data; + found = !strcmp(dbname, alpm_db_get_name(db)); }
- /* We have a database that doesn't match any syncdb. */ + /* We have a file that doesn't match any syncdb. */ if(!found) { - /* ENOENT check is because the signature and database could come in any - * order in our readdir() call, so either file may already be gone. */ - snprintf(path, PATH_MAX, "%s%s.db", dbpath, dbname); - ret += unlink_verbose(path, 1); - /* unlink a signature file if present too */ - snprintf(path, PATH_MAX, "%s%s.db.sig", dbpath, dbname); - ret += unlink_verbose(path, 1); + ret += unlink_verbose(path, 0); } + free(dbname); } closedir(dir); @@ -143,7 +128,7 @@ static int sync_cleandb(const char *dbpath, int keep_used) static int sync_cleandb_all(void) { const char *dbpath; - char *newdbpath; + char *syncdbpath; int ret = 0;
dbpath = alpm_option_get_dbpath(config->handle); @@ -152,17 +137,13 @@ static int sync_cleandb_all(void) return 0; } printf(_("removing unused sync repositories...\n")); - /* The sync dbs were previously put in dbpath/ but are now in dbpath/sync/. - * We will clean everything in dbpath/ except local/, sync/ and db.lck, and - * only the unused sync dbs in dbpath/sync/ */ - ret += sync_cleandb(dbpath, 0);
- if(asprintf(&newdbpath, "%s%s", dbpath, "sync/") < 0) { + if(asprintf(&syncdbpath, "%s%s", dbpath, "sync/") < 0) { ret += 1; return ret; } - ret += sync_cleandb(newdbpath, 1); - free(newdbpath); + ret += sync_cleandb(syncdbpath); + free(syncdbpath);
return ret; }
-- regards, brainpower
On 21/06/15 02:27, brainpower wrote:
/* if keep_used != 0, then the db files which match an used syncdb
* will be kept */ -static int sync_cleandb(const char *dbpath, int keep_used) +static int sync_cleandb(const char *dbpath) You probably want to remove the comment above this, too.
Good catch! Fixed on my branch. A
Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/pacman.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/pacman/pacman.c b/src/pacman/pacman.c index fce0131..764193e 100644 --- a/src/pacman/pacman.c +++ b/src/pacman/pacman.c @@ -724,10 +724,18 @@ static int parsearg_upgrade(int opt) return 0; } switch(opt) { - case OP_FORCE: config->flags |= ALPM_TRANS_FLAG_FORCE; break; - case OP_ASDEPS: config->flags |= ALPM_TRANS_FLAG_ALLDEPS; break; - case OP_ASEXPLICIT: config->flags |= ALPM_TRANS_FLAG_ALLEXPLICIT; break; - case OP_NEEDED: config->flags |= ALPM_TRANS_FLAG_NEEDED; break; + case OP_FORCE: + config->flags |= ALPM_TRANS_FLAG_FORCE; + break; + case OP_ASDEPS: + config->flags |= ALPM_TRANS_FLAG_ALLDEPS; + break; + case OP_ASEXPLICIT: + config->flags |= ALPM_TRANS_FLAG_ALLEXPLICIT; + break; + case OP_NEEDED: + config->flags |= ALPM_TRANS_FLAG_NEEDED; + break; case OP_IGNORE: parsearg_util_addlist(&(config->ignorepkg)); break; -- 2.4.4
This option will be used to determine if the .files database should be used for reading sync information. Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/alpm.h | 3 +++ lib/libalpm/handle.c | 13 +++++++++++++ lib/libalpm/handle.h | 1 + 3 files changed, 17 insertions(+) diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 06e080b..b6b9175 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -859,6 +859,9 @@ int alpm_option_set_deltaratio(alpm_handle_t *handle, double ratio); int alpm_option_get_checkspace(alpm_handle_t *handle); int alpm_option_set_checkspace(alpm_handle_t *handle, int checkspace); +int alpm_option_get_files(alpm_handle_t *handle); +int alpm_option_set_files(alpm_handle_t *handle, int files); + alpm_siglevel_t alpm_option_get_default_siglevel(alpm_handle_t *handle); int alpm_option_set_default_siglevel(alpm_handle_t *handle, alpm_siglevel_t level); diff --git a/lib/libalpm/handle.c b/lib/libalpm/handle.c index 4915d0b..ead1e98 100644 --- a/lib/libalpm/handle.c +++ b/lib/libalpm/handle.c @@ -284,6 +284,12 @@ int SYMEXPORT alpm_option_get_checkspace(alpm_handle_t *handle) return handle->checkspace; } +int SYMEXPORT alpm_option_get_files(alpm_handle_t *handle) +{ + CHECK_HANDLE(handle, return -1); + return handle->files; +} + int SYMEXPORT alpm_option_set_logcb(alpm_handle_t *handle, alpm_cb_log cb) { CHECK_HANDLE(handle, return -1); @@ -664,6 +670,13 @@ int SYMEXPORT alpm_option_set_checkspace(alpm_handle_t *handle, int checkspace) return 0; } +int SYMEXPORT alpm_option_set_files(alpm_handle_t *handle, int files) +{ + CHECK_HANDLE(handle, return -1); + handle->files = files; + return 0; +} + int SYMEXPORT alpm_option_set_default_siglevel(alpm_handle_t *handle, alpm_siglevel_t level) { diff --git a/lib/libalpm/handle.h b/lib/libalpm/handle.h index 5893139..3914822 100644 --- a/lib/libalpm/handle.h +++ b/lib/libalpm/handle.h @@ -95,6 +95,7 @@ struct __alpm_handle_t { double deltaratio; /* Download deltas if possible; a ratio value */ int usesyslog; /* Use syslog instead of logfile? */ /* TODO move to frontend */ int checkspace; /* Check disk space before installing */ + int files; /* Load package information from .files database */ alpm_siglevel_t siglevel; /* Default signature verification level */ alpm_siglevel_t localfilesiglevel; /* Signature verification level for local file upgrade operations */ -- 2.4.4
On 06/20/15 at 05:42pm, Allan McRae wrote:
This option will be used to determine if the .files database should be used for reading sync information.
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/alpm.h | 3 +++ lib/libalpm/handle.c | 13 +++++++++++++ lib/libalpm/handle.h | 1 + 3 files changed, 17 insertions(+)
diff --git a/lib/libalpm/alpm.h b/lib/libalpm/alpm.h index 06e080b..b6b9175 100644 --- a/lib/libalpm/alpm.h +++ b/lib/libalpm/alpm.h @@ -859,6 +859,9 @@ int alpm_option_set_deltaratio(alpm_handle_t *handle, double ratio); int alpm_option_get_checkspace(alpm_handle_t *handle); int alpm_option_set_checkspace(alpm_handle_t *handle, int checkspace);
+int alpm_option_get_files(alpm_handle_t *handle); +int alpm_option_set_files(alpm_handle_t *handle, int files);
Can we use a more descriptive option name? If I didn't already know what it does, the name "files" wouldn't give me much of a clue. apg
Reads from the .db or .files database depending on the flags in the handle. Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 23 +++++++++++++++++------ lib/libalpm/db.c | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/libalpm/be_sync.c b/lib/libalpm/be_sync.c index ea979e6..2214edf 100644 --- a/lib/libalpm/be_sync.c +++ b/lib/libalpm/be_sync.c @@ -174,6 +174,7 @@ valid: int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) { char *syncpath; + const char *db_suffix = NULL; alpm_list_t *i; int ret = -1; mode_t oldmask; @@ -208,6 +209,13 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) RET_ERR(handle, ALPM_ERR_HANDLE_LOCK, -1); } + /* determine db suffix */ + if(handle->files == 0) { + db_suffix = ".db"; + } else { + db_suffix = ".files"; + } + for(i = db->servers; i; i = i->next) { const char *server = i->data, *final_db_url = NULL; struct dload_payload payload; @@ -220,10 +228,11 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) payload.max_size = 25 * 1024 * 1024; /* print server + filename into a buffer */ - len = strlen(server) + strlen(db->treename) + 5; + len = strlen(server) + strlen(db->treename) + strlen(db_suffix) + 2; /* TODO fix leak syncpath and umask unset */ MALLOC(payload.fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); - snprintf(payload.fileurl, len, "%s/%s.db", server, db->treename); + snprintf(payload.fileurl, len, "%s/%s%s", server, db->treename, db_suffix); + payload.handle = handle; payload.force = force; payload.unlink_on_fail = 1; @@ -244,7 +253,9 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) /* check if the final URL from internal downloader looks reasonable */ if(final_db_url != NULL) { - if(strlen(final_db_url) < 3 || strcmp(final_db_url + strlen(final_db_url) - 3, ".db") != 0) { + if(strlen(final_db_url) < 3 || + strcmp(final_db_url + strlen(final_db_url) - strlen(db_suffix), + db_suffix) != 0) { final_db_url = NULL; } } @@ -254,8 +265,8 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) /* print final_db_url into a buffer (leave space for .sig) */ len = strlen(final_db_url) + 5; } else { - /* print server + filename into a buffer (leave space for separator and .db.sig) */ - len = strlen(server) + strlen(db->treename) + 9; + /* print server + filename into a buffer (leave space for separator and .sig) */ + len = strlen(server) + 1 + strlen(db->treename) + strlen(db_suffix) + 5; } /* TODO fix leak syncpath and umask unset */ @@ -264,7 +275,7 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) if(final_db_url != NULL) { snprintf(payload.fileurl, len, "%s.sig", final_db_url); } else { - snprintf(payload.fileurl, len, "%s/%s.db.sig", server, db->treename); + snprintf(payload.fileurl, len, "%s/%s%s.sig", server, db->treename, db_suffix); } payload.handle = handle; diff --git a/lib/libalpm/db.c b/lib/libalpm/db.c index fe208be..7c79e1c 100644 --- a/lib/libalpm/db.c +++ b/lib/libalpm/db.c @@ -375,10 +375,16 @@ const char *_alpm_db_path(alpm_db_t *db) CALLOC(db->_path, 1, pathsize, RET_ERR(db->handle, ALPM_ERR_MEMORY, NULL)); sprintf(db->_path, "%s%s/", dbpath, db->treename); } else { - pathsize = strlen(dbpath) + 5 + strlen(db->treename) + 4; + const char *suffix; + if(db->handle->files == 0) { + suffix = ".db"; + } else { + suffix = ".files"; + } + pathsize = strlen(dbpath) + 5 + strlen(db->treename) + strlen(suffix) + 1; CALLOC(db->_path, 1, pathsize, RET_ERR(db->handle, ALPM_ERR_MEMORY, NULL)); /* all sync DBs now reside in the sync/ subdir of the dbpath */ - sprintf(db->_path, "%ssync/%s.db", dbpath, db->treename); + sprintf(db->_path, "%ssync/%s%s", dbpath, db->treename, suffix); } _alpm_log(db->handle, ALPM_LOG_DEBUG, "database path for tree %s set to %s\n", db->treename, db->_path); -- 2.4.4
On 06/20/15 at 05:42pm, Allan McRae wrote:
Reads from the .db or .files database depending on the flags in the handle.
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 23 +++++++++++++++++------ lib/libalpm/db.c | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-)
I don't really like hard-coding a distinction between normal db's and file db's in alpm. We already have to teach alpm to handle multiple db extensions, so we could just make the extension configurable and allow the front-end to do the selection. That way alpm would simply use whatever it's given and treat all sync db's the same. With proper front-end support, which could easily be added later, this would allow a distribution to choose to provide only a files db in order to prevent the primary and files db's from getting out of sync. In order to reduce unnecessary overhead, the files option could be retained solely to indicate whether or not to actually load the files from syncdb's or we could lazy load them as we do for local packages, although we would probably want to load all file lists from a given db at once. apg
diff --git a/lib/libalpm/be_sync.c b/lib/libalpm/be_sync.c index ea979e6..2214edf 100644 --- a/lib/libalpm/be_sync.c +++ b/lib/libalpm/be_sync.c @@ -174,6 +174,7 @@ valid: int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) { char *syncpath; + const char *db_suffix = NULL; alpm_list_t *i; int ret = -1; mode_t oldmask; @@ -208,6 +209,13 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) RET_ERR(handle, ALPM_ERR_HANDLE_LOCK, -1); }
+ /* determine db suffix */ + if(handle->files == 0) { + db_suffix = ".db"; + } else { + db_suffix = ".files"; + } + for(i = db->servers; i; i = i->next) { const char *server = i->data, *final_db_url = NULL; struct dload_payload payload; @@ -220,10 +228,11 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) payload.max_size = 25 * 1024 * 1024;
/* print server + filename into a buffer */ - len = strlen(server) + strlen(db->treename) + 5; + len = strlen(server) + strlen(db->treename) + strlen(db_suffix) + 2; /* TODO fix leak syncpath and umask unset */ MALLOC(payload.fileurl, len, RET_ERR(handle, ALPM_ERR_MEMORY, -1)); - snprintf(payload.fileurl, len, "%s/%s.db", server, db->treename); + snprintf(payload.fileurl, len, "%s/%s%s", server, db->treename, db_suffix); + payload.handle = handle; payload.force = force; payload.unlink_on_fail = 1; @@ -244,7 +253,9 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db)
/* check if the final URL from internal downloader looks reasonable */ if(final_db_url != NULL) { - if(strlen(final_db_url) < 3 || strcmp(final_db_url + strlen(final_db_url) - 3, ".db") != 0) { + if(strlen(final_db_url) < 3 || + strcmp(final_db_url + strlen(final_db_url) - strlen(db_suffix), + db_suffix) != 0) { final_db_url = NULL; } } @@ -254,8 +265,8 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) /* print final_db_url into a buffer (leave space for .sig) */ len = strlen(final_db_url) + 5; } else { - /* print server + filename into a buffer (leave space for separator and .db.sig) */ - len = strlen(server) + strlen(db->treename) + 9; + /* print server + filename into a buffer (leave space for separator and .sig) */ + len = strlen(server) + 1 + strlen(db->treename) + strlen(db_suffix) + 5; }
/* TODO fix leak syncpath and umask unset */ @@ -264,7 +275,7 @@ int SYMEXPORT alpm_db_update(int force, alpm_db_t *db) if(final_db_url != NULL) { snprintf(payload.fileurl, len, "%s.sig", final_db_url); } else { - snprintf(payload.fileurl, len, "%s/%s.db.sig", server, db->treename); + snprintf(payload.fileurl, len, "%s/%s%s.sig", server, db->treename, db_suffix); }
payload.handle = handle; diff --git a/lib/libalpm/db.c b/lib/libalpm/db.c index fe208be..7c79e1c 100644 --- a/lib/libalpm/db.c +++ b/lib/libalpm/db.c @@ -375,10 +375,16 @@ const char *_alpm_db_path(alpm_db_t *db) CALLOC(db->_path, 1, pathsize, RET_ERR(db->handle, ALPM_ERR_MEMORY, NULL)); sprintf(db->_path, "%s%s/", dbpath, db->treename); } else { - pathsize = strlen(dbpath) + 5 + strlen(db->treename) + 4; + const char *suffix; + if(db->handle->files == 0) { + suffix = ".db"; + } else { + suffix = ".files"; + } + pathsize = strlen(dbpath) + 5 + strlen(db->treename) + strlen(suffix) + 1; CALLOC(db->_path, 1, pathsize, RET_ERR(db->handle, ALPM_ERR_MEMORY, NULL)); /* all sync DBs now reside in the sync/ subdir of the dbpath */ - sprintf(db->_path, "%ssync/%s.db", dbpath, db->treename); + sprintf(db->_path, "%ssync/%s%s", dbpath, db->treename, suffix); } _alpm_log(db->handle, ALPM_LOG_DEBUG, "database path for tree %s set to %s\n", db->treename, db->_path); -- 2.4.4
On 20/06/15 23:29, Andrew Gregory wrote:
On 06/20/15 at 05:42pm, Allan McRae wrote:
Reads from the .db or .files database depending on the flags in the handle.
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 23 +++++++++++++++++------ lib/libalpm/db.c | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-)
I don't really like hard-coding a distinction between normal db's and file db's in alpm. We already have to teach alpm to handle multiple db extensions, so we could just make the extension configurable and allow the front-end to do the selection. That way alpm would simply use whatever it's given and treat all sync db's the same.
How is that different from what is implemented here? handle->files (rename pending...) is set by the front-end and that determines what database is used.
With proper front-end support, which could easily be added later, this would allow a distribution to choose to provide only a files db in order to prevent the primary and files db's from getting out of sync.
This could be added in about three lines now... Add a config option and have it set handle->files as appropriate. I don't understand what you are suggesting here. A
On 20/06/15 23:29, Andrew Gregory wrote:
On 06/20/15 at 05:42pm, Allan McRae wrote:
Reads from the .db or .files database depending on the flags in the handle.
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 23 +++++++++++++++++------ lib/libalpm/db.c | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-)
I don't really like hard-coding a distinction between normal db's and file db's in alpm. We already have to teach alpm to handle multiple db extensions, so we could just make the extension configurable and allow the front-end to do the selection. That way alpm would simply use whatever it's given and treat all sync db's the same. With proper front-end support, which could easily be added later, this would allow a distribution to choose to provide only a files db in order to prevent the primary and files db's from getting out of sync.
In order to reduce unnecessary overhead, the files option could be retained solely to indicate whether or not to actually load the files from syncdb's or we could lazy load them as we do for local packages, although we would probably want to load all file lists from a given db at once.
While not being entirely sure what you envision here, I have just sent through some changes that hopefully addresses some of this... I now have added an enum with the different database types, this is what is stored in the handle, and a function to convert these to a string. Adding a new database extension (e.g. the .source databases I am slowly working on), will be a few lines (excluding code to parse them). I have not added support for configuring database extensions, or for lazy loading files. These can be readily added if a frontend ever wants them, and I think development time is best spent elsewhere without direct interest in that feature. Allan
On 06/23/15 at 11:03pm, Allan McRae wrote:
On 20/06/15 23:29, Andrew Gregory wrote:
On 06/20/15 at 05:42pm, Allan McRae wrote:
Reads from the .db or .files database depending on the flags in the handle.
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 23 +++++++++++++++++------ lib/libalpm/db.c | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-)
I don't really like hard-coding a distinction between normal db's and file db's in alpm. We already have to teach alpm to handle multiple db extensions, so we could just make the extension configurable and allow the front-end to do the selection. That way alpm would simply use whatever it's given and treat all sync db's the same. With proper front-end support, which could easily be added later, this would allow a distribution to choose to provide only a files db in order to prevent the primary and files db's from getting out of sync.
In order to reduce unnecessary overhead, the files option could be retained solely to indicate whether or not to actually load the files from syncdb's or we could lazy load them as we do for local packages, although we would probably want to load all file lists from a given db at once.
While not being entirely sure what you envision here, I have just sent through some changes that hopefully addresses some of this...
I now have added an enum with the different database types, this is what is stored in the handle, and a function to convert these to a string. Adding a new database extension (e.g. the .source databases I am slowly working on), will be a few lines (excluding code to parse them).
I have not added support for configuring database extensions, or for lazy loading files. These can be readily added if a frontend ever wants them, and I think development time is best spent elsewhere without direct interest in that feature.
Allan
The point I was trying to make, and clearly did a poor job of, is just that I generally prefer to keep logic in the front-end wherever that can be easily done. The less we hard-code in the back-end, the more flexibility front-ends have. What I envision in this case is to allow front-ends to specify the db extension. So front-ends would simply call something like `alpm_option_set_syncdb_extension(".files")` instead of `alpm_option_set_dbtype`. libalpm would then simply load the sync databases normally using whatever extension was specified and load the files data if the db happens to include it. I'm not sure how you intend to implement the source db's, so having front-ends set the extension may not make sense in light of that. apg
On 24/06/15 02:04, Andrew Gregory wrote:
On 06/23/15 at 11:03pm, Allan McRae wrote:
On 20/06/15 23:29, Andrew Gregory wrote:
On 06/20/15 at 05:42pm, Allan McRae wrote:
Reads from the .db or .files database depending on the flags in the handle.
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 23 +++++++++++++++++------ lib/libalpm/db.c | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-)
I don't really like hard-coding a distinction between normal db's and file db's in alpm. We already have to teach alpm to handle multiple db extensions, so we could just make the extension configurable and allow the front-end to do the selection. That way alpm would simply use whatever it's given and treat all sync db's the same. With proper front-end support, which could easily be added later, this would allow a distribution to choose to provide only a files db in order to prevent the primary and files db's from getting out of sync.
In order to reduce unnecessary overhead, the files option could be retained solely to indicate whether or not to actually load the files from syncdb's or we could lazy load them as we do for local packages, although we would probably want to load all file lists from a given db at once.
While not being entirely sure what you envision here, I have just sent through some changes that hopefully addresses some of this...
I now have added an enum with the different database types, this is what is stored in the handle, and a function to convert these to a string. Adding a new database extension (e.g. the .source databases I am slowly working on), will be a few lines (excluding code to parse them).
I have not added support for configuring database extensions, or for lazy loading files. These can be readily added if a frontend ever wants them, and I think development time is best spent elsewhere without direct interest in that feature.
Allan
The point I was trying to make, and clearly did a poor job of, is just that I generally prefer to keep logic in the front-end wherever that can be easily done. The less we hard-code in the back-end, the more flexibility front-ends have. What I envision in this case is to allow front-ends to specify the db extension. So front-ends would simply call something like `alpm_option_set_syncdb_extension(".files")` instead of `alpm_option_set_dbtype`. libalpm would then simply load the sync databases normally using whatever extension was specified and load the files data if the db happens to include it.
I'm not sure how you intend to implement the source db's, so having front-ends set the extension may not make sense in light of that.
It will be a db ending in .source, and probably require a be_source to parse it given the architecture specific depends etc. When that happens, the extensions will have meaning. We will not be able to just throw a db at makepkg and parse it as if it is a sync db. What I am not understanding is why there is a need for the frontend to be able to set the extension rather than the type? I am all for flexibility when there is a defined need. In this case, repo-add only creates db with the ".db" extension - with the equivilant .files db made at the same time. Allan
On 06/24/15 at 09:08am, Allan McRae wrote:
On 24/06/15 02:04, Andrew Gregory wrote:
On 06/23/15 at 11:03pm, Allan McRae wrote:
On 20/06/15 23:29, Andrew Gregory wrote:
On 06/20/15 at 05:42pm, Allan McRae wrote:
Reads from the .db or .files database depending on the flags in the handle.
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 23 +++++++++++++++++------ lib/libalpm/db.c | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-)
<snip>
The point I was trying to make, and clearly did a poor job of, is just that I generally prefer to keep logic in the front-end wherever that can be easily done. The less we hard-code in the back-end, the more flexibility front-ends have. What I envision in this case is to allow front-ends to specify the db extension. So front-ends would simply call something like `alpm_option_set_syncdb_extension(".files")` instead of `alpm_option_set_dbtype`. libalpm would then simply load the sync databases normally using whatever extension was specified and load the files data if the db happens to include it.
I'm not sure how you intend to implement the source db's, so having front-ends set the extension may not make sense in light of that.
It will be a db ending in .source, and probably require a be_source to parse it given the architecture specific depends etc.
Will the source db's just be normal sync db's with additional information for building from source or do you intend them to be something different altogether? If they're just sync db's with extra source information it could still be conditionally loaded when present the same way file information is even if it does require more complex parsing. If they aren't going to have the standard sync db information, I'm curious what alpm will do when syncing a package from a source db. Once a front-end selects source db's will it be an all-or-nothing situation where only source packages can be used, including any dependencies that get brought in?
When that happens, the extensions will have meaning. We will not be able to just throw a db at makepkg and parse it as if it is a sync db.
If we are going to give the db extension real meaning, then I agree that having front-ends just select the type and letting libalpm figure out the extension is the way to go.
What I am not understanding is why there is a need for the frontend to be able to set the extension rather than the type? I am all for flexibility when there is a defined need. In this case, repo-add only creates db with the ".db" extension - with the equivilant .files db made at the same time.
Part of the point of providing extra flexibility where we can is so that front-ends can handle situations that we don't think of. I'm only suggesting this alternative because, as it stands now, this seems to me like a case where we can provide that extra flexibility at *zero* cost to either the back-end or front-end implementations. If it's going to complicate future development though, it's not worth it. Also keep in mind that just because repo-add is the officially provided tool for managing repositories doesn't mean it's the only one. There is another problem that stems from the selection being global, but would be somewhat alleviated by allowing file information to be parsed from normal .db files. As it stands now front-ends cannot load file information for repos that have it while falling back to a standard db otherwise. This would be particularly hard on GUI front-ends which would have to switch back and forth between normal and file db's in order to provide file information to users while still allowing the use of the repos lacking a files db. apg
On 24/06/15 11:24, Andrew Gregory wrote:
On 06/24/15 at 09:08am, Allan McRae wrote:
On 24/06/15 02:04, Andrew Gregory wrote:
On 06/23/15 at 11:03pm, Allan McRae wrote:
On 20/06/15 23:29, Andrew Gregory wrote:
On 06/20/15 at 05:42pm, Allan McRae wrote:
Reads from the .db or .files database depending on the flags in the handle.
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 23 +++++++++++++++++------ lib/libalpm/db.c | 10 ++++++++-- 2 files changed, 25 insertions(+), 8 deletions(-)
<snip>
The point I was trying to make, and clearly did a poor job of, is just that I generally prefer to keep logic in the front-end wherever that can be easily done. The less we hard-code in the back-end, the more flexibility front-ends have. What I envision in this case is to allow front-ends to specify the db extension. So front-ends would simply call something like `alpm_option_set_syncdb_extension(".files")` instead of `alpm_option_set_dbtype`. libalpm would then simply load the sync databases normally using whatever extension was specified and load the files data if the db happens to include it.
I'm not sure how you intend to implement the source db's, so having front-ends set the extension may not make sense in light of that.
It will be a db ending in .source, and probably require a be_source to parse it given the architecture specific depends etc.
Will the source db's just be normal sync db's with additional information for building from source or do you intend them to be something different altogether? If they're just sync db's with extra source information it could still be conditionally loaded when present the same way file information is even if it does require more complex parsing.
For source repos, I just want to get it to the ABS replacement stage. pacman -B glibc would download an extract glibc-x.xx.src.tar.gz and dump it in /var/build (configurable). Then the user would run makepkg.
If they aren't going to have the standard sync db information, I'm curious what alpm will do when syncing a package from a source db.
I suppose having separate source repos for every architecture would mean no architecture specific fields... and be_sync could be used. Hrm... But still handling a source package will be very different than a
Once a front-end selects source db's will it be an all-or-nothing situation where only source packages can be used, including any dependencies that get brought in?
This is currently the case. You register the sync db with alpm_register_syncdb. To change between dbs we need to unregister the db and register a new one.
When that happens, the extensions will have meaning. We will not be able to just throw a db at makepkg and parse it as if it is a sync db.
If we are going to give the db extension real meaning, then I agree that having front-ends just select the type and letting libalpm figure out the extension is the way to go.
What I am not understanding is why there is a need for the frontend to be able to set the extension rather than the type? I am all for flexibility when there is a defined need. In this case, repo-add only creates db with the ".db" extension - with the equivilant .files db made at the same time.
Part of the point of providing extra flexibility where we can is so that front-ends can handle situations that we don't think of. I'm only suggesting this alternative because, as it stands now, this seems to me like a case where we can provide that extra flexibility at *zero* cost to either the back-end or front-end implementations. If it's going to complicate future development though, it's not worth it.
There is cost due to alpm_register_syncdb(). I suppose that changing the extension could release the syncdbs and laod the new one.
Also keep in mind that just because repo-add is the officially provided tool for managing repositories doesn't mean it's the only one.
The filenaming is documented in the repo-add man page. If other tools are not following the example of repo-add, they are doing it wrong...
There is another problem that stems from the selection being global, but would be somewhat alleviated by allowing file information to be parsed from normal .db files. As it stands now front-ends cannot load file information for repos that have it while falling back to a standard db otherwise. This would be particularly hard on GUI front-ends which would have to switch back and forth between normal and file db's in order to provide file information to users while still allowing the use of the repos lacking a files db.
repo-add now unconditionally creates the .files db. So all repos should provide it (assumption I know...). I should check what happens if a repo does not provide the .files db... I am more than happy to make adjustments to the alpm backend to help GUI developers, but I would require actual GUI developers to post here with what they need. I am not programming for a potential need, when there are actual needs going unaddressed. If you happen to be writing a GUI, then there is a need! Allan
Signed-off-by: Allan McRae <allan@archlinux.org> --- lib/libalpm/be_sync.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/libalpm/be_sync.c b/lib/libalpm/be_sync.c index 2214edf..00c31a3 100644 --- a/lib/libalpm/be_sync.c +++ b/lib/libalpm/be_sync.c @@ -40,6 +40,7 @@ #include "delta.h" #include "deps.h" #include "dload.h" +#include "filelist.h" static char *get_sync_dir(alpm_handle_t *handle) { @@ -444,6 +445,7 @@ static size_t estimate_package_count(struct stat *st, struct archive *archive) /* assume it is at least somewhat compressed */ per_package = 500; } + return (size_t)((st->st_size / per_package) + 1); } @@ -476,6 +478,11 @@ static int sync_db_populate(alpm_db_t *db) } est_count = estimate_package_count(&buf, archive); + if(db->handle->files) { + /* files databases are about four times larger on average */ + est_count /= 4; + } + db->pkgcache = _alpm_pkghash_create(est_count); if(db->pkgcache == NULL) { db->handle->pm_errno = ALPM_ERR_MEMORY; @@ -606,7 +613,8 @@ static int sync_db_read(alpm_db_t *db, struct archive *archive, } if(strcmp(filename, "desc") == 0 || strcmp(filename, "depends") == 0 - || (strcmp(filename, "deltas") == 0 && db->handle->deltaratio > 0.0) ) { + || (strcmp(filename, "deltas") == 0 && db->handle->deltaratio > 0.0) + || (strcmp(filename, "files") == 0 && db->handle->files == 1)) { int ret; while((ret = _alpm_archive_fgets(archive, &buf)) == ARCHIVE_OK) { char *line = buf.line; @@ -691,6 +699,32 @@ static int sync_db_read(alpm_db_t *db, struct archive *archive, pkg->deltas = alpm_list_add(pkg->deltas, _alpm_delta_parse(db->handle, line)); } + } else if(strcmp(line, "%FILES%") == 0) { + size_t files_count = 0, files_size = 0; + alpm_file_t *files = NULL; + + while(1) { + if(_alpm_archive_fgets(archive, &buf) != ARCHIVE_OK) { + goto error; + } + line = buf.line; + if(_alpm_strip_newline(line, buf.real_line_size) == 0) { + break; + } + + if(!_alpm_greedy_grow((void **)&files, &files_size, + (files_count ? (files_count + 1) * sizeof(alpm_file_t) : 8 * sizeof(alpm_file_t)))) { + goto error; + } + STRDUP(files[files_count].name, line, goto error); + files_count++; + } + /* attempt to hand back any memory we don't need */ + files = realloc(files, sizeof(alpm_file_t) * files_count); + /* make sure the list is sorted */ + qsort(files, files_count, sizeof(alpm_file_t), _alpm_files_cmp); + pkg->files.count = files_count; + pkg->files.files = files; } } if(ret != ARCHIVE_EOF) { @@ -700,7 +734,7 @@ static int sync_db_read(alpm_db_t *db, struct archive *archive, } else if(strcmp(filename, "deltas") == 0) { /* skip reading delta files if UseDelta is unset */ } else if(strcmp(filename, "files") == 0) { - /* currently do nothing with this file */ + /* skip reading files file if handle->files is 0 */ } else { /* unknown database file */ _alpm_log(db->handle, ALPM_LOG_DEBUG, "unknown database file: %s\n", filename); -- 2.4.4
Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/callback.c | 2 +- src/pacman/sync.c | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pacman/callback.c b/src/pacman/callback.c index 1f08675..9260ece 100644 --- a/src/pacman/callback.c +++ b/src/pacman/callback.c @@ -750,7 +750,7 @@ void cb_dl_progress(const char *filename, off_t file_xfered, off_t file_total) fname = malloc(len + 1); memcpy(fname, filename, len); /* strip package or DB extension for cleaner look */ - if((p = strstr(fname, ".pkg")) || (p = strstr(fname, ".db"))) { + if((p = strstr(fname, ".pkg")) || (p = strstr(fname, ".db")) || (p = strstr(fname, ".files"))) { /* tack on a .sig suffix for signatures */ if(memcmp(&filename[len - 4], ".sig", 4) == 0) { memcpy(p, ".sig", 4); diff --git a/src/pacman/sync.c b/src/pacman/sync.c index c42c38d..4ed36ca 100644 --- a/src/pacman/sync.c +++ b/src/pacman/sync.c @@ -104,6 +104,10 @@ static int sync_cleandb(const char *dbpath) dbname = strndup(dname, len - 3); } else if(len > 7 && strcmp(dname + len - 7, ".db.sig") == 0) { dbname = strndup(dname, len - 7); + } else if(len > 6 && strcmp(dname + len - 6, ".files") == 0) { + dbname = strndup(dname, len - 6); + } else if(len > 6 && strcmp(dname + len - 6, ".files.sig") == 0) { + dbname = strndup(dname, len - 10); } else { ret += unlink_verbose(path, 0); continue; @@ -217,8 +221,8 @@ static int sync_cleancache(int level) static const char *const glob_skips[] = { /* skip signature files - they are removed with their package file */ "*.sig", - /* skip package database within the cache directory */ - "*.db*", + /* skip package databases within the cache directory */ + "*.db*", "*.files*", /* skip source packages within the cache directory */ "*.src.tar.*", /* skip package deltas, we aren't smart enough to clean these yet */ -- 2.4.4
Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/sync.c | 33 +-------------------------------- src/pacman/util.c | 31 +++++++++++++++++++++++++++++++ src/pacman/util.h | 1 + 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/pacman/sync.c b/src/pacman/sync.c index 4ed36ca..d990f09 100644 --- a/src/pacman/sync.c +++ b/src/pacman/sync.c @@ -307,37 +307,6 @@ static int sync_cleancache(int level) return ret; } -static int sync_synctree(int level, alpm_list_t *syncs) -{ - alpm_list_t *i; - unsigned int success = 0; - - for(i = syncs; i; i = alpm_list_next(i)) { - alpm_db_t *db = i->data; - - int ret = alpm_db_update((level < 2 ? 0 : 1), db); - if(ret < 0) { - pm_printf(ALPM_LOG_ERROR, _("failed to update %s (%s)\n"), - alpm_db_get_name(db), alpm_strerror(alpm_errno(config->handle))); - } else if(ret == 1) { - printf(_(" %s is up to date\n"), alpm_db_get_name(db)); - success++; - } else { - success++; - } - } - - /* We should always succeed if at least one DB was upgraded - we may possibly - * fail later with unresolved deps, but that should be rare, and would be - * expected - */ - if(!success) { - pm_printf(ALPM_LOG_ERROR, _("failed to synchronize any databases\n")); - trans_init_error(); - } - return (success > 0); -} - /* search the sync dbs for a matching package */ static int sync_search(alpm_list_t *syncs, alpm_list_t *targets) { @@ -905,7 +874,7 @@ int pacman_sync(alpm_list_t *targets) colon_printf(_("Synchronizing package databases...\n")); alpm_logaction(config->handle, PACMAN_CALLER_PREFIX, "synchronizing package lists\n"); - if(!sync_synctree(config->op_s_sync, sync_dbs)) { + if(!sync_syncdbs(config->op_s_sync, sync_dbs)) { return 1; } } diff --git a/src/pacman/util.c b/src/pacman/util.c index ea7faf8..4662024 100644 --- a/src/pacman/util.c +++ b/src/pacman/util.c @@ -143,6 +143,37 @@ int check_syncdbs(size_t need_repos, int check_valid) return ret; } +int sync_syncdbs(int level, alpm_list_t *syncs) +{ + alpm_list_t *i; + unsigned int success = 0; + + for(i = syncs; i; i = alpm_list_next(i)) { + alpm_db_t *db = i->data; + + int ret = alpm_db_update((level < 2 ? 0 : 1), db); + if(ret < 0) { + pm_printf(ALPM_LOG_ERROR, _("failed to update %s (%s)\n"), + alpm_db_get_name(db), alpm_strerror(alpm_errno(config->handle))); + } else if(ret == 1) { + printf(_(" %s is up to date\n"), alpm_db_get_name(db)); + success++; + } else { + success++; + } + } + + /* We should always succeed if at least one DB was upgraded - we may possibly + * fail later with unresolved deps, but that should be rare, and would be + * expected + */ + if(!success) { + pm_printf(ALPM_LOG_ERROR, _("failed to synchronize any databases\n")); + trans_init_error(); + } + return (success > 0); +} + /* discard unhandled input on the terminal's input buffer */ static int flush_term_input(int fd) { diff --git a/src/pacman/util.h b/src/pacman/util.h index fda9b51..c82d816 100644 --- a/src/pacman/util.h +++ b/src/pacman/util.h @@ -49,6 +49,7 @@ int trans_init(alpm_transflag_t flags, int check_valid); int trans_release(void); int needs_root(void); int check_syncdbs(size_t need_repos, int check_valid); +int sync_syncdbs(int level, alpm_list_t *syncs); unsigned short getcols(void); void columns_cache_reset(void); int rmrf(const char *path); -- 2.4.4
Add the -F/--files operations, -s/--sync support and nd provide dummy functions for -s/--search, -l/-list and -o/--owns. Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/Makefile.am | 1 + src/pacman/conf.c | 4 +++ src/pacman/conf.h | 3 +- src/pacman/files.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/pacman/pacman.c | 64 +++++++++++++++++++++++++++++++++++++- src/pacman/pacman.h | 2 ++ src/pacman/util.c | 2 ++ 7 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 src/pacman/files.c diff --git a/src/pacman/Makefile.am b/src/pacman/Makefile.am index dc127a2..d3ae071 100644 --- a/src/pacman/Makefile.am +++ b/src/pacman/Makefile.am @@ -32,6 +32,7 @@ pacman_SOURCES = \ conf.h conf.c \ database.c \ deptest.c \ + files.c \ ini.h ini.c \ package.h package.c \ pacman.h pacman.c \ diff --git a/src/pacman/conf.c b/src/pacman/conf.c index ccf8183..83ec950 100644 --- a/src/pacman/conf.c +++ b/src/pacman/conf.c @@ -729,6 +729,10 @@ static int setup_libalpm(void) alpm_option_set_questioncb(handle, cb_question); alpm_option_set_progresscb(handle, cb_progress); + if(config->op == PM_OP_FILES) { + alpm_option_set_files(handle, 1); + } + config->logfile = config->logfile ? config->logfile : strdup(LOGFILE); ret = alpm_option_set_logfile(handle, config->logfile); if(ret != 0) { diff --git a/src/pacman/conf.h b/src/pacman/conf.h index d6feb7a..84b5a25 100644 --- a/src/pacman/conf.h +++ b/src/pacman/conf.h @@ -138,7 +138,8 @@ enum { PM_OP_QUERY, PM_OP_SYNC, PM_OP_DEPTEST, - PM_OP_DATABASE + PM_OP_DATABASE, + PM_OP_FILES }; /* Long Operations */ diff --git a/src/pacman/files.c b/src/pacman/files.c new file mode 100644 index 0000000..18d4c91 --- /dev/null +++ b/src/pacman/files.c @@ -0,0 +1,84 @@ +/* + * files.c + * + * Copyright (c) 2015 Pacman Development Team <pacman-dev@archlinux.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <alpm.h> +#include <alpm_list.h> + +/* pacman */ +#include "pacman.h" +#include "util.h" +#include "conf.h" + + +static int files_fileowner(alpm_list_t __attribute__((unused)) *syncs, alpm_list_t __attribute__((unused)) *targets) { + return 0; +} + +static int files_search(alpm_list_t __attribute__((unused)) *syncs, alpm_list_t __attribute__((unused)) *targets) { + return 0; +} + +static int files_list(alpm_list_t __attribute__((unused)) *syncs, alpm_list_t __attribute__((unused)) *targets) { + return 0; +} + + +int pacman_files(alpm_list_t *targets) +{ + alpm_list_t *files_dbs = NULL; + + if(check_syncdbs(1, 0)) { + return 1; + } + + files_dbs = alpm_get_syncdbs(config->handle); + + if(config->op_s_sync) { + /* grab a fresh package list */ + colon_printf(_("Synchronizing package databases...\n")); + alpm_logaction(config->handle, PACMAN_CALLER_PREFIX, + "synchronizing package lists\n"); + if(!sync_syncdbs(config->op_s_sync, files_dbs)) { + return 1; + } + } + + if(targets == NULL && (config->op_s_search || config->op_q_owns)) { + pm_printf(ALPM_LOG_ERROR, _("no targets specified (use -h for help)\n")); + return 1; + } + + /* determine the owner of a file */ + if(config->op_q_owns) { + return files_fileowner(files_dbs, targets); + } + + /* search for a file */ + if(config->op_s_search) { + return files_search(files_dbs, targets); + } + + /* get a listing of files in sync DBs */ + if(config->op_q_list) { + return files_list(files_dbs, targets); + } + + + return 0; +} diff --git a/src/pacman/pacman.c b/src/pacman/pacman.c index 764193e..951d628 100644 --- a/src/pacman/pacman.c +++ b/src/pacman/pacman.c @@ -110,6 +110,7 @@ static void usage(int op, const char * const myname) printf(" %s {-h --help}\n", myname); printf(" %s {-V --version}\n", myname); printf(" %s {-D --database} <%s> <%s>\n", myname, str_opt, str_pkg); + printf(" %s {-F --files} [%s] [%s]\n", myname, str_opt, str_pkg); printf(" %s {-Q --query} [%s] [%s]\n", myname, str_opt, str_pkg); printf(" %s {-R --remove} [%s] <%s>\n", myname, str_opt, str_pkg); printf(" %s {-S --sync} [%s] [%s]\n", myname, str_opt, str_pkg); @@ -173,6 +174,12 @@ static void usage(int op, const char * const myname) } else if(op == PM_OP_DEPTEST) { printf("%s: %s {-T --deptest} [%s] [%s]\n", str_usg, myname, str_opt, str_pkg); printf("%s:\n", str_opt); + } else if(op == PM_OP_FILES) { + addlist(_(" -l, --list list the files owned by the queried package\n")); + addlist(_(" -o, --owns <file> query the package that owns <file>\n")); + addlist(_(" -s, --search <regex> search package file names for matching strings\n")); + addlist(_(" -y, --refresh download fresh package databases from the server\n" + " (-yy to force a refresh even if up to date)\n")); } switch(op) { case PM_OP_SYNC: @@ -363,6 +370,9 @@ static int parsearg_op(int opt, int dryrun) case 'D': if(dryrun) break; config->op = (config->op != PM_OP_MAIN ? 0 : PM_OP_DATABASE); break; + case 'F': + if(dryrun) break; + config->op = (config->op != PM_OP_MAIN ? 0 : PM_OP_FILES); break; case 'Q': if(dryrun) break; config->op = (config->op != PM_OP_MAIN ? 0 : PM_OP_QUERY); break; @@ -755,6 +765,48 @@ static void checkargs_upgrade(void) "--asdeps", "--asexplicit"); } +static int parsearg_files(int opt) +{ + if(parsearg_trans(opt) == 0) { + return 0; + } + switch(opt) { + case OP_OWNS: + case 'o': + config->op_q_owns = 1; + break; + case OP_LIST: + case 'l': + config->op_q_list = 1; + break; + case OP_SEARCH: + case 's': + config->op_s_search = 1; + break; + case OP_REFRESH: + case 'y': + (config->op_s_sync)++; + break; + case OP_QUIET: + case 'q': + config->quiet = 1; + break; + default: + return 1; + } + return 0; +} + +static void checkargs_files(void) +{ + if(config->op_q_owns) { + invalid_opt(config->op_q_list, "--owns", "--list"); + invalid_opt(config->op_q_search, "--owns", "--search"); + } else if(config->op_q_list) { + invalid_opt(config->op_q_search, "--list", "--search"); + } +} + static int parsearg_sync(int opt) { if(parsearg_upgrade(opt) == 0) { @@ -847,10 +899,11 @@ static int parseargs(int argc, char *argv[]) int opt; int option_index = 0; int result; - const char *optstring = "DQRSTUVb:cdefghiklmnopqr:stuvwy"; + const char *optstring = "DFQRSTUVb:cdefghiklmnopqr:stuvwy"; static const struct option opts[] = { {"database", no_argument, 0, 'D'}, + {"files", no_argument, 0, 'F'}, {"query", no_argument, 0, 'Q'}, {"remove", no_argument, 0, 'R'}, {"sync", no_argument, 0, 'S'}, @@ -964,6 +1017,9 @@ static int parseargs(int argc, char *argv[]) case PM_OP_UPGRADE: result = parsearg_upgrade(opt); break; + case PM_OP_FILES: + result = parsearg_files(opt); + break; case PM_OP_DEPTEST: default: result = 1; @@ -1012,6 +1068,9 @@ static int parseargs(int argc, char *argv[]) case PM_OP_UPGRADE: checkargs_upgrade(); break; + case PM_OP_FILES: + checkargs_files(); + break; default: break; } @@ -1238,6 +1297,9 @@ int main(int argc, char *argv[]) case PM_OP_DEPTEST: ret = pacman_deptest(pm_targets); break; + case PM_OP_FILES: + ret = pacman_files(pm_targets); + break; default: pm_printf(ALPM_LOG_ERROR, _("no operation specified (use -h for help)\n")); ret = EXIT_FAILURE; diff --git a/src/pacman/pacman.h b/src/pacman/pacman.h index ddf6e62..e7d3aa6 100644 --- a/src/pacman/pacman.h +++ b/src/pacman/pacman.h @@ -28,6 +28,8 @@ int pacman_database(alpm_list_t *targets); /* deptest.c */ int pacman_deptest(alpm_list_t *targets); +/* files.c */ +int pacman_files(alpm_list_t *files); /* query.c */ int pacman_query(alpm_list_t *targets); /* remove.c */ diff --git a/src/pacman/util.c b/src/pacman/util.c index 4662024..dd6e218 100644 --- a/src/pacman/util.c +++ b/src/pacman/util.c @@ -113,6 +113,8 @@ int needs_root(void) return (config->op_s_clean || config->op_s_sync || (!config->group && !config->op_s_info && !config->op_q_list && !config->op_s_search && !config->print)); + case PM_OP_FILES: + return config->op_s_sync; default: return 0; } -- 2.4.4
Equivalent to -Qo but for packages in the sync database e.g. pacman -Fo /usr/bin/pacman Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/files.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/src/pacman/files.c b/src/pacman/files.c index 18d4c91..667e8cc 100644 --- a/src/pacman/files.c +++ b/src/pacman/files.c @@ -26,7 +26,59 @@ #include "conf.h" -static int files_fileowner(alpm_list_t __attribute__((unused)) *syncs, alpm_list_t __attribute__((unused)) *targets) { +static int files_fileowner(alpm_list_t *syncs, alpm_list_t *targets) { + int ret = 0; + alpm_list_t *t; + + for(t = targets; t; t = alpm_list_next(t)) { + char *filename = NULL, *f; + int found = 0; + alpm_list_t *s; + size_t len; + + if((filename = strdup(t->data)) == NULL) { + goto notfound; + } + + len = strlen(filename); + f = filename; + while(len > 1 && f[0] == '/') { + f = f + 1; + len--; + } + + for(s = syncs; s; s = alpm_list_next(s)) { + alpm_list_t *p; + alpm_db_t *repo = s->data; + alpm_list_t *packages = alpm_db_get_pkgcache(repo); + + for(p = packages; p; p = alpm_list_next(p)) { + alpm_pkg_t *pkg = p->data; + alpm_filelist_t *files = alpm_pkg_get_files(pkg); + + if(alpm_filelist_contains(files, f)) { + + if(!config->quiet) { + printf(_("%s is owned by %s/%s %s\n"), filename, + alpm_db_get_name(repo), alpm_pkg_get_name(pkg), + alpm_pkg_get_version(pkg)); + } else { + printf("%s/%s\n", alpm_db_get_name(repo), alpm_pkg_get_name(pkg)); + } + + found = 1; + } + } + } + + free(filename); + +notfound: + if(!found) { + ret++; + } + } + return 0; } -- 2.4.4
Does the equivalent of the -Ql option for local packages e.g. pacman -Fl glibc Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/files.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/pacman/files.c b/src/pacman/files.c index 667e8cc..695f13d 100644 --- a/src/pacman/files.c +++ b/src/pacman/files.c @@ -24,6 +24,7 @@ #include "pacman.h" #include "util.h" #include "conf.h" +#include "package.h" static int files_fileowner(alpm_list_t *syncs, alpm_list_t *targets) { @@ -86,8 +87,41 @@ static int files_search(alpm_list_t __attribute__((unused)) *syncs, alpm_list_t return 0; } -static int files_list(alpm_list_t __attribute__((unused)) *syncs, alpm_list_t __attribute__((unused)) *targets) { - return 0; +static int files_list(alpm_list_t *syncs, alpm_list_t *targets) { + alpm_list_t *i, *j; + int ret = 0, found = 0; + + if(targets != NULL) { + for(i = targets; i; i = alpm_list_next(i)) { + /* TODO: handle repo/pkg stype arguements */ + char *targ = i->data; + + for(j = syncs; j; j = alpm_list_next(j)) { + alpm_pkg_t *pkg; + alpm_db_t *db = j->data; + if((pkg = alpm_db_get_pkg(db, targ)) != NULL) { + found = 1; + dump_pkg_files(pkg, config->quiet); + } + } + if(!found) { + pm_printf(ALPM_LOG_ERROR, + _("package '%s' was not found\n"), targ); + ret += 1; + } + } + } else { + for(i = syncs; i; i = alpm_list_next(i)) { + alpm_db_t *db = i->data; + + for(j = alpm_db_get_pkgcache(db); j; j = alpm_list_next(j)) { + alpm_pkg_t *pkg = j->data; + dump_pkg_files(pkg, config->quiet); + } + } + } + + return ret; } -- 2.4.4
Locates all packages that contain the listed file e.g. pacman -Fs libpng.so Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/files.c | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/src/pacman/files.c b/src/pacman/files.c index 695f13d..fc06ae7 100644 --- a/src/pacman/files.c +++ b/src/pacman/files.c @@ -83,7 +83,69 @@ notfound: return 0; } -static int files_search(alpm_list_t __attribute__((unused)) *syncs, alpm_list_t __attribute__((unused)) *targets) { +static int files_search(alpm_list_t *syncs, alpm_list_t *targets) { + int ret = 0; + alpm_list_t *t; + const colstr_t *colstr = &config->colstr; + + for(t = targets; t; t = alpm_list_next(t)) { + char *filename = NULL; + alpm_list_t *s; + int found = 0; + + if((filename = strdup(t->data)) == NULL) { + goto notfound; + } + + for(s = syncs; s; s = alpm_list_next(s)) { + alpm_list_t *p; + alpm_db_t *repo = s->data; + alpm_list_t *packages = alpm_db_get_pkgcache(repo); + + for(p = packages; p; p = alpm_list_next(p)) { + size_t f = 0; + char* c; + alpm_pkg_t *pkg = p->data; + alpm_filelist_t *files = alpm_pkg_get_files(pkg); + alpm_list_t *match = NULL; + + while(f < files->count) { + c = strrchr(files->files[f].name, '/'); + if(c && *(c + 1)) { + if(strcmp(c + 1, filename) == 0) { + match = alpm_list_add(match, strdup(files->files[f].name)); + found = 1; + } + } + f++; + } + + if(match != NULL) { + if(config->quiet) { + printf("%s/%s\n", alpm_db_get_name(repo), alpm_pkg_get_name(pkg)); + } else { + alpm_list_t *ml; + printf("%s%s/%s%s %s%s%s\n", colstr->repo, alpm_db_get_name(repo), + colstr->title, alpm_pkg_get_name(pkg), + colstr->version, alpm_pkg_get_version(pkg), colstr->nocolor); + + for(ml = match; ml; ml = alpm_list_next(ml)) { + c = ml->data; + printf(" %s\n", c); + } + FREELIST(match); + } + } + } + } + free(filename); + +notfound: + if(!found) { + ret++; + } + } + return 0; } -- 2.4.4
e.g. pacman -Fsx kcm.*print.*\.so Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/conf.h | 3 +++ src/pacman/files.c | 28 ++++++++++++++++++++++------ src/pacman/pacman.c | 12 ++++++++++-- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/pacman/conf.h b/src/pacman/conf.h index 84b5a25..3fff900 100644 --- a/src/pacman/conf.h +++ b/src/pacman/conf.h @@ -88,6 +88,8 @@ typedef struct __config_t { unsigned short op_s_search; unsigned short op_s_upgrade; + unsigned short op_f_regex; + unsigned short group; unsigned short noask; unsigned int ask; @@ -187,6 +189,7 @@ enum { OP_ROOT, OP_RECURSIVE, OP_SEARCH, + OP_REGEX, OP_UNREQUIRED, OP_UPGRADES, OP_SYSUPGRADE, diff --git a/src/pacman/files.c b/src/pacman/files.c index fc06ae7..5156e46 100644 --- a/src/pacman/files.c +++ b/src/pacman/files.c @@ -19,6 +19,7 @@ #include <alpm.h> #include <alpm_list.h> +#include <regex.h> /* pacman */ #include "pacman.h" @@ -83,24 +84,33 @@ notfound: return 0; } -static int files_search(alpm_list_t *syncs, alpm_list_t *targets) { +static int files_search(alpm_list_t *syncs, alpm_list_t *targets, int regex) { int ret = 0; alpm_list_t *t; const colstr_t *colstr = &config->colstr; for(t = targets; t; t = alpm_list_next(t)) { - char *filename = NULL; + char *targ = NULL; alpm_list_t *s; int found = 0; + regex_t reg; - if((filename = strdup(t->data)) == NULL) { + if((targ = strdup(t->data)) == NULL) { goto notfound; } + if(regex) { + if(regcomp(®, targ, REG_EXTENDED | REG_NOSUB | REG_ICASE | REG_NEWLINE) != 0) { + /* TODO: error message */ + goto notfound; + } + } + for(s = syncs; s; s = alpm_list_next(s)) { alpm_list_t *p; alpm_db_t *repo = s->data; alpm_list_t *packages = alpm_db_get_pkgcache(repo); + int m; for(p = packages; p; p = alpm_list_next(p)) { size_t f = 0; @@ -112,7 +122,12 @@ static int files_search(alpm_list_t *syncs, alpm_list_t *targets) { while(f < files->count) { c = strrchr(files->files[f].name, '/'); if(c && *(c + 1)) { - if(strcmp(c + 1, filename) == 0) { + if(regex) { + m = regexec(®, (c + 1), 0, 0, 0); + } else { + m = strcmp(c + 1, targ); + } + if(m == 0) { match = alpm_list_add(match, strdup(files->files[f].name)); found = 1; } @@ -138,7 +153,8 @@ static int files_search(alpm_list_t *syncs, alpm_list_t *targets) { } } } - free(filename); + + free(targ); notfound: if(!found) { @@ -219,7 +235,7 @@ int pacman_files(alpm_list_t *targets) /* search for a file */ if(config->op_s_search) { - return files_search(files_dbs, targets); + return files_search(files_dbs, targets, config->op_f_regex); } /* get a listing of files in sync DBs */ diff --git a/src/pacman/pacman.c b/src/pacman/pacman.c index 951d628..c680067 100644 --- a/src/pacman/pacman.c +++ b/src/pacman/pacman.c @@ -177,7 +177,8 @@ static void usage(int op, const char * const myname) } else if(op == PM_OP_FILES) { addlist(_(" -l, --list list the files owned by the queried package\n")); addlist(_(" -o, --owns <file> query the package that owns <file>\n")); - addlist(_(" -s, --search <regex> search package file names for matching strings\n")); + addlist(_(" -s, --search <file> search package file names for matching strings\n")); + addlist(_(" -x, --regex enable searching using regular expressions\n")); addlist(_(" -y, --refresh download fresh package databases from the server\n" " (-yy to force a refresh even if up to date)\n")); } @@ -787,6 +788,10 @@ static int parsearg_files(int opt) case 'y': (config->op_s_sync)++; break; + case OP_REGEX: + case 'x': + config->op_f_regex = 1; + break; case OP_QUIET: case 'q': config->quiet = 1; @@ -802,8 +807,10 @@ static void checkargs_files(void) if(config->op_q_owns) { invalid_opt(config->op_q_list, "--owns", "--list"); invalid_opt(config->op_q_search, "--owns", "--search"); + invalid_opt(config->op_f_regex, "--owns", "--regex"); } else if(config->op_q_list) { invalid_opt(config->op_q_search, "--list", "--search"); + invalid_opt(config->op_f_regex, "--list", "--regex"); } } @@ -899,7 +906,7 @@ static int parseargs(int argc, char *argv[]) int opt; int option_index = 0; int result; - const char *optstring = "DFQRSTUVb:cdefghiklmnopqr:stuvwy"; + const char *optstring = "DFQRSTUVb:cdefghiklmnopqr:stuvwxy"; static const struct option opts[] = { {"database", no_argument, 0, 'D'}, @@ -933,6 +940,7 @@ static int parseargs(int argc, char *argv[]) {"root", required_argument, 0, OP_ROOT}, {"recursive", no_argument, 0, OP_RECURSIVE}, {"search", no_argument, 0, OP_SEARCH}, + {"regex", no_argument, 0, OP_REGEX}, {"unrequired", no_argument, 0, OP_UNREQUIRED}, {"upgrades", no_argument, 0, OP_UPGRADES}, {"sysupgrade", no_argument, 0, OP_SYSUPGRADE}, -- 2.4.4
Passing "-Fl pkg" will print the filelist for the first occurance of "pkg" in the sync repos. Other version of the package can be printed using "-Fl repo/pkg". Signed-off-by: Allan McRae <allan@archlinux.org> --- src/pacman/files.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/pacman/files.c b/src/pacman/files.c index 5156e46..2250994 100644 --- a/src/pacman/files.c +++ b/src/pacman/files.c @@ -171,22 +171,45 @@ static int files_list(alpm_list_t *syncs, alpm_list_t *targets) { if(targets != NULL) { for(i = targets; i; i = alpm_list_next(i)) { - /* TODO: handle repo/pkg stype arguements */ char *targ = i->data; - + char *repo = NULL; + char *c = strchr(targ, '/'); + + if(c) { + if(! *(c + 1)) { + pm_printf(ALPM_LOG_ERROR, + _("invalid package: '%s'\n"), targ); + ret += 1; + continue; + } + + repo = strndup(targ, c - targ); + targ = c + 1; + } + for(j = syncs; j; j = alpm_list_next(j)) { alpm_pkg_t *pkg; alpm_db_t *db = j->data; + + if(repo) { + if(strcmp(alpm_db_get_name(db), repo) != 0) { + continue; + } + } + if((pkg = alpm_db_get_pkg(db, targ)) != NULL) { found = 1; dump_pkg_files(pkg, config->quiet); + break; } } if(!found) { + targ = i->data; pm_printf(ALPM_LOG_ERROR, _("package '%s' was not found\n"), targ); ret += 1; } + free(repo); } } else { for(i = syncs; i; i = alpm_list_next(i)) { -- 2.4.4
On 20/06/15 17:42, Allan McRae wrote:
This patchset adds support for reading the .files database and performing the following: - downloading/updating the .files database - checking which package contains a file - searching for a file (including regex support) - listing all files from a package
I will do documentation at some stage...
participants (3)
-
Allan McRae
-
Andrew Gregory
-
brainpower