- Pacman-dev - lists.archlinux.org

[pacman-dev] [PATCH] bash-completion: fix leaking "files" array into shell environment
by Eli Schwartz 02 Sep '16

02 Sep '16

--- contrib/bash_completion.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/bash_completion.in b/contrib/bash_completion.in index 6f99681..06963c4 100644 --- a/contrib/bash_completion.in +++ b/contrib/bash_completion.in @@ -97,7 +97,7 @@ _pacman_pkg() { } _pacman() { - local common core cur database prev query remove sync upgrade o + local common core cur database files prev query remove sync upgrade o COMPREPLY=() _get_comp_words_by_ref cur prev database=('asdeps asexplicit') -- 2.9.3

1 0

[pacman-dev] [PATCH] [RFC] Provide source files for useful debug packages
by Allan McRae 01 Sep '16

01 Sep '16

Debug packages are fairly useless currently because the soucre files needed for stepping through code etc are not packaged with them. This patch adds the needed source files to the debug package and adjusts the debug info to look at the /usr/src/debug/ directory for them rather than the build location. This requires using the "debugedit" program which is provided as part of the RPM sources. Signed-off-by: Allan McRae <allan(a)archlinux.org> --- This is a first pass at making debug packages useful... Some things I want comments on: 1) It feels dirty depending on an RPM tool for adjusting the paths inside ELF binaries, but I don't see any other choice. Suggestions welcome 2) Getting the list of src files from the binary involves an awk that works... but I don't know how! Can someone better at awk than me take a look and confirm I have not done something wrong 3) split packages... I need to figure out how to support these as source files will overlap. My current idea is to order the packaging by dependency chain so that e.g gcc-libs comes before gcc which comes before gcc-fortran. Then gcc-debug will not include files from gcc-libs-debug, and gcc-fortran-debug will not include files in either of those. I hope that would remove most file conflicts across packages. scripts/libmakepkg/tidy/strip.sh.in | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts/libmakepkg/tidy/strip.sh.in b/scripts/libmakepkg/tidy/strip.sh.in index 1c7aacf..bd5c79a 100644 --- a/scripts/libmakepkg/tidy/strip.sh.in +++ b/scripts/libmakepkg/tidy/strip.sh.in @@ -35,6 +35,11 @@ build_id() { LANG=C readelf -n $1 | sed -n '/Build ID/ { s/.*: //p; q; }' } +source_files() { + LANG=C readelf $1 --debug-dump | \ + awk '/DW_AT_name +:/{name=$8}/DW_AT_comp_dir +:/{print $8 "/" name}' +} + strip_file() { local binary=$1; shift @@ -50,6 +55,18 @@ strip_file() { return fi + # copy source files to debug directory + local f t + for f in $(source_files "$binary"); do + t=${f/$srcdir/$dbgsrc} + mkdir -p "${t%/*}" + cp "$f" "$t" + done + + # adjust debug symbols to point at sources + debugedit -b "${srcdir}" -d /usr/src/debug/ -i "$binary" &> /dev/null + + # copy debug symbols to debug directory mkdir -p "$dbgdir/${binary%/*}" objcopy --only-keep-debug "$binary" "$dbgdir/$binary.debug" objcopy --add-gnu-debuglink="$dbgdir/${binary#/}.debug" "$binary" @@ -89,7 +106,8 @@ tidy_strip() { if check_option "debug" "y"; then dbgdir="$pkgdir-@DEBUGSUFFIX@/usr/lib/debug" - mkdir -p "$dbgdir" + dbgsrc="$pkgdir-@DEBUGSUFFIX@/usr/src/debug" + mkdir -p "$dbgdir" "$dbgsrc" fi local binary strip_flags -- 2.9.3

1 0

[pacman-dev] [PATCH] Fix typo in pacman-db-upgrade usage message
by Allan McRae 01 Sep '16

01 Sep '16

Signed-off-by: Allan McRae <allan(a)archlinux.org> --- scripts/pacman-db-upgrade.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pacman-db-upgrade.sh.in b/scripts/pacman-db-upgrade.sh.in index 94085ba..79e1c59 100644 --- a/scripts/pacman-db-upgrade.sh.in +++ b/scripts/pacman-db-upgrade.sh.in @@ -40,7 +40,7 @@ usage() { printf -- "$(gettext "Usage: %s [options]")\n" "$0" echo printf -- "$(gettext "options:")\n" - printf -- "$(gettext " -b, --dbpath <path> set an alternate database location")\n" + printf -- "$(gettext " -d, --dbpath <path> set an alternate database location")\n" printf -- "$(gettext " -h, --help show this help message and exit")\n" printf -- "$(gettext " -r, --root <path> set an alternate installation root")\n" printf -- "$(gettext " -V, --version show version information and exit")\n" -- 2.9.3

1 0

[pacman-dev] [PATCH] Replace CURLOPT_PROGRESSFUNCTION with CURLOPT_XFERINFOFUNCTION
by ivy.foster＠gmail.com 31 Aug '16

31 Aug '16

From: Ivy Foster <ivy.foster(a)gmail.com> Curl 7.32.0 added CURLOPT_XFERINFOFUNCTION, which deprecates CURLOPT_PROGRESSFUNCTION and means less casting doubles to size_ts for alpm. This change has no user-facing nor frontend-facing effects. Signed-off-by: Ivy Foster <ivy.foster(a)gmail.com> --- lib/libalpm/dload.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/libalpm/dload.c b/lib/libalpm/dload.c index f4e6a27..dc57c92 100644 --- a/lib/libalpm/dload.c +++ b/lib/libalpm/dload.c @@ -90,8 +90,8 @@ static void inthandler(int UNUSED signum) dload_interrupted = ABORT_SIGINT; } -static int dload_progress_cb(void *file, double dltotal, double dlnow, - double UNUSED ultotal, double UNUSED ulnow) +static int dload_progress_cb(void *file, curl_off_t dltotal, curl_off_t dlnow, + curl_off_t UNUSED ultotal, curl_off_t UNUSED ulnow) { struct dload_payload *payload = (struct dload_payload *)file; off_t current_size, total_size; @@ -106,7 +106,7 @@ static int dload_progress_cb(void *file, double dltotal, double dlnow, return 1; } - current_size = payload->initial_size + (off_t)dlnow; + current_size = payload->initial_size + dlnow; /* is our filesize still under any set limit? */ if(payload->max_size && current_size > payload->max_size) { @@ -119,9 +119,9 @@ static int dload_progress_cb(void *file, double dltotal, double dlnow, return 0; } - total_size = payload->initial_size + (off_t)dltotal; + total_size = payload->initial_size + dltotal; - if(DOUBLE_EQ(dltotal, 0.0) || payload->prevprogress == total_size) { + if(dltotal == 0 || payload->prevprogress == total_size) { return 0; } @@ -134,7 +134,7 @@ static int dload_progress_cb(void *file, double dltotal, double dlnow, * x {x>0}, x: download complete * x {x>0, x<y}, y {y > 0}: download progress, expected total is known */ if(current_size == total_size) { - payload->handle->dlcb(payload->remote_name, (off_t)dlnow, (off_t)dltotal); + payload->handle->dlcb(payload->remote_name, dlnow, dltotal); } else if(!payload->prevprogress) { payload->handle->dlcb(payload->remote_name, 0, -1); } else if(payload->prevprogress == current_size) { @@ -142,7 +142,7 @@ static int dload_progress_cb(void *file, double dltotal, double dlnow, } else { /* do NOT include initial_size since it wasn't part of the package's * download_size (nor included in the total download size callback) */ - payload->handle->dlcb(payload->remote_name, (off_t)dlnow, (off_t)dltotal); + payload->handle->dlcb(payload->remote_name, dlnow, dltotal); } payload->prevprogress = current_size; @@ -303,8 +303,8 @@ static void curl_set_handle_opts(struct dload_payload *payload, curl_easy_setopt(curl, CURLOPT_FILETIME, 1L); curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, dload_progress_cb); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, (void *)payload); + curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, dload_progress_cb); + curl_easy_setopt(curl, CURLOPT_XFERINFODATA, (void *)payload); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1L); curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 10L); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, dload_parseheader_cb); -- 2.9.3

4 5

[pacman-dev] [PATCH] Remove AC_FUNC_MALLOC check.
by Alastair Hughes 31 Aug '16

31 Aug '16

We weren't supplying the rpl_malloc function needed if this failed, and didn't check for realloc, so just remove. Signed-off-by: Alastair Hughes <hobbitalastair(a)gmail.com> --- The build will currently fail if cross compiling or if ac_cv_func_malloc_0_nonnull=no is set, because we don't supply a rpl_malloc function anywhere. As we don't use AC_FUNC_REALLOC either, just assume that we will never encounter a buggy system and remove the check. The other option appears to be to provide a malloc definition somewhere, but in that case it appears that we should also check for realloc. I'm unsure which one should be used - opinions? configure.ac | 1 - 1 file changed, 1 deletion(-) diff --git a/configure.ac b/configure.ac index dd4ac04..f2d3b75 100644 --- a/configure.ac +++ b/configure.ac @@ -308,7 +308,6 @@ PATH_MAX_DEFINED AC_FUNC_FORK AC_FUNC_GETMNTENT AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK -AC_FUNC_MALLOC AC_FUNC_MKTIME AC_FUNC_STRCOLL AC_CHECK_FUNCS([dup2 getcwd getmntinfo gettimeofday memmove memset \ -- 2.9.3

2 1

[pacman-dev] I'm back (mostly)
by Allan McRae 30 Aug '16

30 Aug '16

Hi all, I had a low presence over the last couple of months due to purchasing a house, moving and the subsequent delay of internet connection. But I am back! Ping any patches that I have missed in my flurry tonight and I will review them. Cheers, Allan

1 0

[pacman-dev] Typo in pacman manpage
by Luca Weiss 30 Aug '16

30 Aug '16

Hey, just a quick "typo" report in the pacman manpage: In the category DATABASE OPTIONS there is no space between --asexplicit and <package>. Thanks!

3 2

[pacman-dev] [GIT] The official pacman repository branch, master, updated. v5.0.1-54-g406c9b6
by allan＠archlinux.org 30 Aug '16

30 Aug '16

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "The official pacman repository". The branch, master has been updated via 406c9b66b4f2cc54ea59e57de92db6566e59c4e7 (commit) via 6c15cc4d22e049486a5f773f5d81777b6087510b (commit) via 6d8e3d2a9171df8cbe479d0a582c468970802aed (commit) via 1ec7fa89ad6ad650cedeafd5c7cbc36dcf29239f (commit) via 69aee3e391d85a1c44de1c2e78f62a1e27453bca (commit) via 58140dba7440997e9d318fb56ed939a9c81fddf8 (commit) via 56de155296a57fb3fcd8ae64aed00fd18fe2f22e (commit) via 5b839c58ee1e78312edf69cd2cb5f96d8b649d01 (commit) via 56ae9603768bdc850f32bc789a474ae836305033 (commit) via be1ffedaf6fc44aeb9da235d64889dac71e9bf24 (commit) via c981f5ad76cb77363dcb4ebcc199670c3378995f (commit) via af83a585745d6108d7af2d4a2d1eae9b28a8c4b6 (commit) via 5fcd60e2641c9293c2783aad509baf217e77aa6f (commit) via 681509fd445ed6012e6ecf89b49e9c00d83b70cd (commit) via 1291c04961e6c27e93ef376583d46ec2aa3036a1 (commit) via 80d97fcf7526f16d9eb097b8061956662207ed78 (commit) via 0f0b192d8a59134e9a58781c58aa3202df2eda2c (commit) via 6ac2ee21b30f3c5f331d19349f96bb8e5b020b47 (commit) via 7a9d8b7001f3f90471dc94ab31ec017f32ef8760 (commit) from 839417e8c65e73e632c77d3ecd68539494850de1 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 406c9b66b4f2cc54ea59e57de92db6566e59c4e7 Author: Florian Pritz <bluewind(a)xinu.at> Date: Sun Aug 7 15:37:23 2016 +0200 Document contrib/verify-pacman-repo-db.pl Signed-off-by: Florian Pritz <bluewind(a)xinu.at> commit 6c15cc4d22e049486a5f773f5d81777b6087510b Author: Florian Pritz <bluewind(a)xinu.at> Date: Mon Jul 18 15:14:54 2016 +0200 contrib: Add documentation Makefile Makefile.am is mostly copied from ./doc/Makefile.am Signed-off-by: Florian Pritz <bluewind(a)xinu.at> commit 6d8e3d2a9171df8cbe479d0a582c468970802aed Author: Florian Pritz <bluewind(a)xinu.at> Date: Mon Jul 18 15:12:55 2016 +0200 contrib: Add verify-pacman-repo-db.pl From the documentation: verify-pacman-repo-db looks at a pacman repo database and verifies its content with the actual package files. The database is expected to be in the same directory as the packages (or symlinks to the packages). The following properties are verified for each package in the database: - existence of the package file - file size - MD5 and SHA256 checksum (--checksum) Signed-off-by: Florian Pritz <bluewind(a)xinu.at> commit 1ec7fa89ad6ad650cedeafd5c7cbc36dcf29239f Author: Allan McRae <allan(a)archlinux.org> Date: Tue Aug 30 19:06:45 2016 +1000 Fix file name in scripts potfile commit 69aee3e391d85a1c44de1c2e78f62a1e27453bca Author: Giulio Fidente <gfidente(a)gmail.com> Date: Sun Aug 7 15:55:59 2016 +1000 Merge Giolio Fidente into "Pacman Development Team" contribution As discussed on mailing list: https://lists.archlinux.org/pipermail/pacman-dev/2016-July/021239.html Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 58140dba7440997e9d318fb56ed939a9c81fddf8 Author: Ivy Foster <ivy.foster(a)gmail.com> Date: Fri Jul 8 22:11:25 2016 -0500 Normalize alpm download callback's frontend cb arguments When curl calls alpm's dlcb, alpm calls the frontend's cb with the following (dlsize, totalsize) arguments: 0, -1: initialize 0, 0: no change since last call x {x>0, x<y}, y {y>0}: data downloaded, total size known x {x>0}, x: download finished If total size is not known, do not call frontend cb (no change to original behavior); alpm's callback shouldn't be called if there is a download error. See agregory's original spec here: https://wiki.archlinux.org/index.php/User:Apg#download_callback Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 56de155296a57fb3fcd8ae64aed00fd18fe2f22e Author: Jack O'Connor <oconnor663(a)gmail.com> Date: Mon Aug 1 13:06:00 2016 +1000 libmakepkg: look for architecture-specific hashes in get_integlist `makepkg -g` looks for existing checksums in the PKGBUILD file, so that it can generate new sums of the same type. Previously it only checked variables of the form "sha256sums", and not "sha256sums_x86_64". That meant it would always fall back to MD5 for packages with only architecture-specific sources. This change makes it look at architecture-specific checksums too to determine the type. Signed-off-by: Jack O'Connor <oconnor663(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 5b839c58ee1e78312edf69cd2cb5f96d8b649d01 Author: Allan McRae <allan(a)archlinux.org> Date: Sun Jul 3 23:05:41 2016 +1000 Add newline to the end of error messages for signature format issues Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 56ae9603768bdc850f32bc789a474ae836305033 Author: Olivier Brunel <jjk(a)jjacky.com> Date: Sun Jun 26 21:30:49 2016 +0200 configure.ac: Fix handling --with-libcurl It was reported being --with-libcurl in the help (and the check for libcurl being installed used $with_libcurl accordingly), but the option handling was set using $with_curl and, therefore, expected option --with-curl. In the end, --with-libcurl wasn't recognized, and --with-curl had no effect. Signed-off-by: Olivier Brunel <jjk(a)jjacky.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit be1ffedaf6fc44aeb9da235d64889dac71e9bf24 Author: Andrew Gregory <andrew.gregory.8(a)gmail.com> Date: Mon Jun 27 22:23:43 2016 -0400 hook.c: replace fstatat with stat macOS < 10.10 do not provide fstatat. We were constructing the full path to the hook file for all other operations anyway, so there was no real benefit to using fstatat. Fixes FS#49771 Signed-off-by: Andrew Gregory <andrew.gregory.8(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit c981f5ad76cb77363dcb4ebcc199670c3378995f Author: Andrew Gregory <andrew.gregory.8(a)gmail.com> Date: Mon Jun 27 22:23:42 2016 -0400 hook.c: replace readdir_r with readdir glibc 2.24 deprecates readdir_r. Signed-off-by: Andrew Gregory <andrew.gregory.8(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit af83a585745d6108d7af2d4a2d1eae9b28a8c4b6 Author: Tobias Stoeckmann <tobias(a)stoeckmann.org> Date: Sat Jun 18 18:58:07 2016 +0200 Handle all POSIX compliant systems in mbscasecmp. The width of wchar_t is allowed to be of the same width as long, according to standards. The return type of mbscasecmp is int though. On amd64 with a 32 bit int, this means that mbscasecmp can return zero (indicating that strings are equal) even though the input strings differ. Signed-off-by: Tobias Stoeckmann <tobias(a)stoeckmann.org> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 5fcd60e2641c9293c2783aad509baf217e77aa6f Author: Tobias Stoeckmann <tobias(a)stoeckmann.org> Date: Sat Jun 18 18:41:07 2016 +0200 Reject files larger than 16384 bytes in read_sigfile. If signature files are larger than SIZE_MAX, not enough memory could be allocated for this file. The script repo-add rejects files which are larger than 16384 bytes, therefore handle these as errors here, too. While at it, I also rearranged the code to avoid a quite harmless TOCTOU race condition between stat() and fopen(). Signed-off-by: Tobias Stoeckmann <tobias(a)stoeckmann.org> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 681509fd445ed6012e6ecf89b49e9c00d83b70cd Author: Tobias Stoeckmann <tobias(a)stoeckmann.org> Date: Mon Jun 6 20:12:30 2016 +0200 Release resources on error paths. Some resources (memory or file descriptors) are not released on all error paths. Signed-off-by: Tobias Stoeckmann <tobias(a)stoeckmann.org> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 1291c04961e6c27e93ef376583d46ec2aa3036a1 Author: Allan McRae <allan(a)archlinux.org> Date: Sun Jun 12 14:18:24 2016 +1000 pacman-key: reject armored signatures pacman expects an unarmored signature. makepkg forces the generation of unarmored signatures, and repo-add will reject any armored signature. For consistency pacman-key should also reject armored signatures. Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 80d97fcf7526f16d9eb097b8061956662207ed78 Author: Tobias Stoeckmann <tobias(a)stoeckmann.org> Date: Sun Jun 5 19:23:31 2016 +0200 Always use proper error code in alpm_initialize. In out of memory conditions, an undefined error value is written into *err, because myerr is never explicitly set in these cases. I have also converted a calloc into a MALLOC call, because the memory will be properly filled by the snprintf call right after it. Signed-off-by: Tobias Stoeckmann <tobias(a)stoeckmann.org> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 0f0b192d8a59134e9a58781c58aa3202df2eda2c Author: Allan McRae <allan(a)archlinux.org> Date: Sat Jun 4 17:28:29 2016 +1000 Remove obsolete TODO Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 6ac2ee21b30f3c5f331d19349f96bb8e5b020b47 Author: Andrew Gregory <andrew.gregory.8(a)gmail.com> Date: Mon May 23 08:27:29 2016 -0400 recursedeps: include cyclic dependencies Cyclic dependencies (A depends on B, B depends on A) were not selected because neither package could be removed individually, so can_remove_package would always return false for both. By preselecting all dependencies then filtering back out any dependencies still required by any packages that will not be uninstalled, groups of unneeded cyclic dependencies can be found. Fixes FS#41031 Signed-off-by: Andrew Gregory <andrew.gregory.8(a)gmail.com> Signed-off-by: Allan McRae <allan(a)archlinux.org> commit 7a9d8b7001f3f90471dc94ab31ec017f32ef8760 Author: Allan McRae <allan(a)archlinux.org> Date: Sat Jun 4 17:00:48 2016 +1000 Turn off colours once finished using them The colour of the package version could leak onto the next line in both -Qo and -Fo. Signed-off-by: Allan McRae <allan(a)archlinux.org> ----------------------------------------------------------------------- Summary of changes: configure.ac | 5 +- contrib/Makefile.am | 6 + contrib/doc/.gitignore | 1 + contrib/doc/Makefile.am | 57 ++++++ contrib/doc/asciidoc.conf | 1 + contrib/doc/footer.txt | 1 + contrib/doc/verify-pacman-repo-db.1.txt | 60 +++++++ contrib/pacscripts.sh.in | 1 - contrib/verify-pacman-repo-db.pl | 259 ++++++++++++++++++++++++++++ lib/libalpm/add.c | 5 +- lib/libalpm/alpm.c | 13 +- lib/libalpm/backup.c | 5 +- lib/libalpm/be_local.c | 19 +- lib/libalpm/be_package.c | 15 +- lib/libalpm/deps.c | 104 +++++------ lib/libalpm/dload.c | 22 ++- lib/libalpm/hook.c | 26 ++- lib/libalpm/signing.c | 12 +- scripts/libmakepkg/util/pkgbuild.sh.in | 11 ++ scripts/pacman-key.sh.in | 4 + scripts/po/POTFILES.in | 10 +- src/pacman/callback.c | 19 +- src/pacman/check.c | 1 - src/pacman/files.c | 4 +- src/pacman/query.c | 5 +- src/pacman/util.c | 8 +- test/pacman/tests/TESTS | 1 + test/pacman/tests/remove-recursive-cycle.py | 41 +++++ 28 files changed, 597 insertions(+), 119 deletions(-) create mode 100644 contrib/doc/.gitignore create mode 100644 contrib/doc/Makefile.am create mode 120000 contrib/doc/asciidoc.conf create mode 120000 contrib/doc/footer.txt create mode 100644 contrib/doc/verify-pacman-repo-db.1.txt create mode 100755 contrib/verify-pacman-repo-db.pl create mode 100644 test/pacman/tests/remove-recursive-cycle.py hooks/post-receive -- The official pacman repository

1 0

[pacman-dev] Default XferCommand breaks when resuming incomplete downloads on some servers
by Quint Guvernator 22 Aug '16

22 Aug '16

Hello all, The default value of XferCommand (/usr/bin/curl -C - -f %u > %o) tells curl to try to resume downloads from a particular byte position if they are stopped prematurely. Not all servers support this, though, so curl may die with exit status 33 like this: :: Checking nwjs-bin integrity... ==> Making package: nwjs-bin 0.16.1-1 (Thu Aug 18 00:38:19 EDT 2016) ==> Retrieving sources... -> Downloading nwjs-v0.16.1-linux-x64.tar.gz... ** Resuming transfer from byte position 8736768 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 47.1M 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume. ==> ERROR: Failure while downloading http://dl.nwjs.io/v0.16.1/nwjs-v0.16.1-linux-x64.tar.gz Aborting... (To reproduce, grab the nwjs-bin PKGBUILD from the AUR. makepkg and kill the download, then makepkg again. There's gotta be an instance of this in community or extra, but I haven't run into any yet.) The same could theoretically happen with FTP; the corresponding error code there is 36. This raises the question: should `curl -C` be the default download command when we can't trust all servers to support resuming incomplete downloads? To me, it seems that adding `-C` should be documented on the wiki under pacman/Tips_and_Tricks rather than being enabled by default. This way, users (ostensibly) understand the gotcha involved before enabling it. There isn't much technical merit/demerit to discuss here, and there's a potential for bikeshedding, so maybe we could stick to "yes, I like it" or "no, I don't like it". It's barely a 1 LOC change. Cheers, Quint

3 2

[pacman-dev] [PATCH 1/2] contrib: Add verify-pacman-repo-db.pl
by Florian Pritz 21 Aug '16

21 Aug '16

Signed-off-by: Florian Pritz <bluewind(a)xinu.at> --- contrib/doc/verify-pacman-repo-db.1.txt | 50 ++++++ contrib/verify-pacman-repo-db.pl | 279 ++++++++++++++++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 contrib/doc/verify-pacman-repo-db.1.txt create mode 100755 contrib/verify-pacman-repo-db.pl diff --git a/contrib/doc/verify-pacman-repo-db.1.txt b/contrib/doc/verify-pacman-repo-db.1.txt new file mode 100644 index 0000000..01eafbe --- /dev/null +++ b/contrib/doc/verify-pacman-repo-db.1.txt @@ -0,0 +1,50 @@ +///// +vim:set ts=4 sw=4 syntax=asciidoc noet spell spelllang=en_us: +///// +verify-pacman-repo-db(1) +======================= + +Name +---- +verify-pacman-repo-db - package repository verification utility + + +Synopsis +-------- +'verify-pacman-repo-db' [options] + + +Description +----------- +'verify-pacman-repo-db' looks at a pacman repo database and verifies its +content with the actual package files. The database is expected to be in +the same directory as the packages (or symlinks to the packages). + +The following properties are verified for each package in the database: + + - existence of the package file + - file size + - MD5 and SHA256 checksum (--checksum) + +Options +------- +*-h, \--help*:: + Output a short help message. + +*\--debug*:: + Enable debug output. + +*-c, \--checksum*:: + Verify checksums of packages. Note that this means all packages files will + be read from disk. Otherwise only metadata is compared which does not + require to read package file contents. + +*-t, \--threads* <number>:: + Use 'number' threads to verify packages. Note that each thread may use up + to approximately 128MiB of memory. Default: 1 + +See Also +-------- +linkman:repo-add[8] + +include::footer.txt[] diff --git a/contrib/verify-pacman-repo-db.pl b/contrib/verify-pacman-repo-db.pl new file mode 100755 index 0000000..71cfddf --- /dev/null +++ b/contrib/verify-pacman-repo-db.pl @@ -0,0 +1,279 @@ +#!/usr/bin/perl -T +use warnings; +use strict; + +=pod + +=head1 SYNOPSIS + +verify-pacman-repo-db.pl [options] <database file> ... + + Options: + --help, -h Show short help message + --debug Enable debug output + --checksum, -c Verify checksums of packages + --thread, -t <num> Use num threads to verify packages. Default: 1 + NOTE: Each thread uses up to approx. 128MiB of memory + +=head1 DESCRIPTION + +verify-pacman-repo-db.pl looks at a pacman repo database and verifies its content +with the actual package files. The database is expected to be in the same +directory as the packages (or symlinks to the packages). + +The following properties are verified for each package in the database: + + - existence of the package file + - file size + - MD5 and SHA256 checksum (--checksum) + +=head1 NOTES + +This script does intentionally not use any ALPM libraries. The format is simple +enough to be parsed and this way we might just detect more problems because the +libalpm parsing code might also have bugs. We also stay much more portable +which might be good for people that want to check a db, but don't actually have +pacman installed. + +=cut + +package main; +use Getopt::Long; +use Pod::Usage; + +exit main(); + +sub main { + my %opts = ( + threads => 1, + ); + + Getopt::Long::Configure ("bundling"); + pod2usage(-verbose => 0) if (@ARGV== 0); + GetOptions(\%opts, "help|h", "debug", "threads|t=i", "checksum|c") or pod2usage(2); + pod2usage(0) if $opts{help}; + + my $verifier = Verifier->new(\%opts); + + for my $repodb (@ARGV) { + $verifier->check_repodb($repodb); + } + + $verifier->finalize(); + return $verifier->get_error_status(); +} + +package Verifier; +use Archive::Tar; +use Digest::MD5; +use Digest::SHA; +use File::Basename; +use threads; +use threads::shared; +use Thread::Queue; + +sub new { + my $class = shift; + my $opts = shift; + + my $self :shared = shared_clone({ + opts => \%{$opts}, + package_queue => Thread::Queue->new(), + output_queue => Thread::Queue->new(), + workers => [], + errors => 0, + }); + + bless $self, $class; + $self->start_workers(); + return $self; +} + +sub start_workers { + my $self = shift; + + threads->new(\&_worker_output_queue, $self); + + for (my $i = 0; $i < $self->{opts}->{threads}; $i++) { + my $thr :shared = shared_clone(threads->new(\&_worker_package_queue, $self)); + push @{$self->{workers}}, $thr; + } +} + +sub _worker_package_queue { + my $self = shift; + while (my $workpack = $self->{package_queue}->dequeue()) { + my $dbdata = $self->_parse_db_entry($workpack->{db_desc_content}); + $self->{errors} += $self->_verify_db_entry($workpack->{dirname}, $dbdata); + } +} + +sub _worker_output_queue { + my $self = shift; + while (my $output = $self->{output_queue}->dequeue()) { + print STDERR $output; + } +} + +sub finalize { + my $self = shift; + + $self->{package_queue}->end(); + $self->_join_threads($self->{workers}); + + $self->{output_queue}->end(); + $self->_join_threads([threads->list]); +} + +sub _join_threads { + my $self = shift; + my $threads = shift; + + for my $thr (@{$threads}) { + if ($thr->tid && !threads::equal($thr, threads->self)) { + print "waiting for thread ".$thr->tid()." to finish\n" if $self->{opts}->{debug}; + $thr->join; + } + } +} + +sub get_error_status { + my $self = shift; + + return $self->{errors} > 0; +} + +sub check_repodb { + my $self = shift; + my $repodb = shift; + + my $db = Archive::Tar->new(); + $db->read($repodb); + + my $dirname = dirname($repodb); + my $pkgcount = 0; + + my @files = $db->list_files(); + for my $file_object ($db->get_files()) { + if ($file_object->name =~ m/^([^\/]+)\/desc$/) { + my $package = $1; + $self->{package_queue}->enqueue({ + package => $package, + db_desc_content => $file_object->get_content(), + dirname => $dirname, + }); + $pkgcount++; + } + } + + $self->_debug(sprintf("Queued %d package from database '%s'\n", $pkgcount, $repodb)); +} + +sub _parse_db_entry { + my $self = shift; + my $content = shift; + my %db; + my $key; + + for my $line (split /\n/, $content) { + if ($line eq '') { + $key = undef; + next; + } + if ($line =~ m/^%(.+)%$/) { + $key = $1; + } else { + push @{$db{$key}}, $line; + die "\$key not set. Is the db formated incorrectly?" unless $key; + } + } + return \%db; +} + +sub _output { + my $self = shift; + my $output = shift; + + return if $output eq ""; + + $output = sprintf("Thread %s: %s", threads->self->tid(), $output); + $self->{output_queue}->enqueue($output); +} + +sub _debug { + my $self = shift; + my $output = shift; + $self->_output($output) if $self->{opts}->{debug}; +} + +sub _verify_db_entry { + my $self = shift; + my $basedir = shift; + my $dbdata = shift; + my $ret = 0; + my $output = ""; + + # verify package exists + my $pkgfile = $basedir.'/'.$dbdata->{FILENAME}[0]; + $self->_debug(sprintf("Checking package %s\n", $dbdata->{FILENAME}[0])); + unless (-e $pkgfile) { + $self->_output(sprintf("Package file missing: %s\n", $pkgfile)); + return 1; + } + + $ret += $self->_verify_package_size($dbdata, $pkgfile); + $ret += $self->_verify_package_checksum($dbdata, $pkgfile) if $self->{opts}->{checksum}; + # TODO verify gpg sigs? + + return $ret; +} + +sub _verify_package_size { + my $self = shift; + my $dbdata = shift; + my $pkgfile = shift; + + my $csize = $dbdata->{CSIZE}[0]; + my $filesize = (stat($pkgfile))[7]; + unless ($csize == $filesize) { + $self->_output(sprintf("Package file has incorrect size: %d vs %d: %s\n", $csize, $filesize, $pkgfile)); + return 1; + } + return 0; +} + +sub _verify_package_checksum { + my $self = shift; + my $dbdata = shift; + my $pkgfile = shift; + + my $md5 = Digest::MD5->new; + my $sha = Digest::SHA->new(256); + + my $content; + # 128MiB to keep random IO low when using multiple threads (only works for large packages though) + my $chunksize = 1024*1024*128; + open my $fh, "<", $pkgfile; + while (read($fh, $content, $chunksize)) { + $md5->add($content); + $sha->add($content); + } + + my $expected_sha = $dbdata->{SHA256SUM}[0]; + my $expected_md5 = $dbdata->{MD5SUM}[0]; + my $got_md5 = $md5->hexdigest; + my $got_sha = $sha->hexdigest; + + unless ($expected_sha eq $got_sha and $expected_md5 eq $got_md5) { + my $output; + $output .= sprintf "Package file has incorrect checksum: %s\n", $pkgfile; + $output .= sprintf "expected: SHA %s\n", $expected_sha; + $output .= sprintf "got: SHA %s\n", $got_sha; + $output .= sprintf "expected: MD5 %s\n", $expected_md5; + $output .= sprintf "got: MD5 %s\n", $got_md5; + $self->_output($output); + return 1; + } + return 0; +} + -- 2.9.0

3 11