July 2016 - Pacman-dev - lists.archlinux.org

[pacman-dev] [PATCH 1/2] contrib: Add verify-pacman-repo-db.pl
by Florian Pritz 21 Aug '16

21 Aug '16

Signed-off-by: Florian Pritz <bluewind(a)xinu.at> --- contrib/doc/verify-pacman-repo-db.1.txt | 50 ++++++ contrib/verify-pacman-repo-db.pl | 279 ++++++++++++++++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 contrib/doc/verify-pacman-repo-db.1.txt create mode 100755 contrib/verify-pacman-repo-db.pl diff --git a/contrib/doc/verify-pacman-repo-db.1.txt b/contrib/doc/verify-pacman-repo-db.1.txt new file mode 100644 index 0000000..01eafbe --- /dev/null +++ b/contrib/doc/verify-pacman-repo-db.1.txt @@ -0,0 +1,50 @@ +///// +vim:set ts=4 sw=4 syntax=asciidoc noet spell spelllang=en_us: +///// +verify-pacman-repo-db(1) +======================= + +Name +---- +verify-pacman-repo-db - package repository verification utility + + +Synopsis +-------- +'verify-pacman-repo-db' [options] + + +Description +----------- +'verify-pacman-repo-db' looks at a pacman repo database and verifies its +content with the actual package files. The database is expected to be in +the same directory as the packages (or symlinks to the packages). + +The following properties are verified for each package in the database: + + - existence of the package file + - file size + - MD5 and SHA256 checksum (--checksum) + +Options +------- +*-h, \--help*:: + Output a short help message. + +*\--debug*:: + Enable debug output. + +*-c, \--checksum*:: + Verify checksums of packages. Note that this means all packages files will + be read from disk. Otherwise only metadata is compared which does not + require to read package file contents. + +*-t, \--threads* <number>:: + Use 'number' threads to verify packages. Note that each thread may use up + to approximately 128MiB of memory. Default: 1 + +See Also +-------- +linkman:repo-add[8] + +include::footer.txt[] diff --git a/contrib/verify-pacman-repo-db.pl b/contrib/verify-pacman-repo-db.pl new file mode 100755 index 0000000..71cfddf --- /dev/null +++ b/contrib/verify-pacman-repo-db.pl @@ -0,0 +1,279 @@ +#!/usr/bin/perl -T +use warnings; +use strict; + +=pod + +=head1 SYNOPSIS + +verify-pacman-repo-db.pl [options] <database file> ... + + Options: + --help, -h Show short help message + --debug Enable debug output + --checksum, -c Verify checksums of packages + --thread, -t <num> Use num threads to verify packages. Default: 1 + NOTE: Each thread uses up to approx. 128MiB of memory + +=head1 DESCRIPTION + +verify-pacman-repo-db.pl looks at a pacman repo database and verifies its content +with the actual package files. The database is expected to be in the same +directory as the packages (or symlinks to the packages). + +The following properties are verified for each package in the database: + + - existence of the package file + - file size + - MD5 and SHA256 checksum (--checksum) + +=head1 NOTES + +This script does intentionally not use any ALPM libraries. The format is simple +enough to be parsed and this way we might just detect more problems because the +libalpm parsing code might also have bugs. We also stay much more portable +which might be good for people that want to check a db, but don't actually have +pacman installed. + +=cut + +package main; +use Getopt::Long; +use Pod::Usage; + +exit main(); + +sub main { + my %opts = ( + threads => 1, + ); + + Getopt::Long::Configure ("bundling"); + pod2usage(-verbose => 0) if (@ARGV== 0); + GetOptions(\%opts, "help|h", "debug", "threads|t=i", "checksum|c") or pod2usage(2); + pod2usage(0) if $opts{help}; + + my $verifier = Verifier->new(\%opts); + + for my $repodb (@ARGV) { + $verifier->check_repodb($repodb); + } + + $verifier->finalize(); + return $verifier->get_error_status(); +} + +package Verifier; +use Archive::Tar; +use Digest::MD5; +use Digest::SHA; +use File::Basename; +use threads; +use threads::shared; +use Thread::Queue; + +sub new { + my $class = shift; + my $opts = shift; + + my $self :shared = shared_clone({ + opts => \%{$opts}, + package_queue => Thread::Queue->new(), + output_queue => Thread::Queue->new(), + workers => [], + errors => 0, + }); + + bless $self, $class; + $self->start_workers(); + return $self; +} + +sub start_workers { + my $self = shift; + + threads->new(\&_worker_output_queue, $self); + + for (my $i = 0; $i < $self->{opts}->{threads}; $i++) { + my $thr :shared = shared_clone(threads->new(\&_worker_package_queue, $self)); + push @{$self->{workers}}, $thr; + } +} + +sub _worker_package_queue { + my $self = shift; + while (my $workpack = $self->{package_queue}->dequeue()) { + my $dbdata = $self->_parse_db_entry($workpack->{db_desc_content}); + $self->{errors} += $self->_verify_db_entry($workpack->{dirname}, $dbdata); + } +} + +sub _worker_output_queue { + my $self = shift; + while (my $output = $self->{output_queue}->dequeue()) { + print STDERR $output; + } +} + +sub finalize { + my $self = shift; + + $self->{package_queue}->end(); + $self->_join_threads($self->{workers}); + + $self->{output_queue}->end(); + $self->_join_threads([threads->list]); +} + +sub _join_threads { + my $self = shift; + my $threads = shift; + + for my $thr (@{$threads}) { + if ($thr->tid && !threads::equal($thr, threads->self)) { + print "waiting for thread ".$thr->tid()." to finish\n" if $self->{opts}->{debug}; + $thr->join; + } + } +} + +sub get_error_status { + my $self = shift; + + return $self->{errors} > 0; +} + +sub check_repodb { + my $self = shift; + my $repodb = shift; + + my $db = Archive::Tar->new(); + $db->read($repodb); + + my $dirname = dirname($repodb); + my $pkgcount = 0; + + my @files = $db->list_files(); + for my $file_object ($db->get_files()) { + if ($file_object->name =~ m/^([^\/]+)\/desc$/) { + my $package = $1; + $self->{package_queue}->enqueue({ + package => $package, + db_desc_content => $file_object->get_content(), + dirname => $dirname, + }); + $pkgcount++; + } + } + + $self->_debug(sprintf("Queued %d package from database '%s'\n", $pkgcount, $repodb)); +} + +sub _parse_db_entry { + my $self = shift; + my $content = shift; + my %db; + my $key; + + for my $line (split /\n/, $content) { + if ($line eq '') { + $key = undef; + next; + } + if ($line =~ m/^%(.+)%$/) { + $key = $1; + } else { + push @{$db{$key}}, $line; + die "\$key not set. Is the db formated incorrectly?" unless $key; + } + } + return \%db; +} + +sub _output { + my $self = shift; + my $output = shift; + + return if $output eq ""; + + $output = sprintf("Thread %s: %s", threads->self->tid(), $output); + $self->{output_queue}->enqueue($output); +} + +sub _debug { + my $self = shift; + my $output = shift; + $self->_output($output) if $self->{opts}->{debug}; +} + +sub _verify_db_entry { + my $self = shift; + my $basedir = shift; + my $dbdata = shift; + my $ret = 0; + my $output = ""; + + # verify package exists + my $pkgfile = $basedir.'/'.$dbdata->{FILENAME}[0]; + $self->_debug(sprintf("Checking package %s\n", $dbdata->{FILENAME}[0])); + unless (-e $pkgfile) { + $self->_output(sprintf("Package file missing: %s\n", $pkgfile)); + return 1; + } + + $ret += $self->_verify_package_size($dbdata, $pkgfile); + $ret += $self->_verify_package_checksum($dbdata, $pkgfile) if $self->{opts}->{checksum}; + # TODO verify gpg sigs? + + return $ret; +} + +sub _verify_package_size { + my $self = shift; + my $dbdata = shift; + my $pkgfile = shift; + + my $csize = $dbdata->{CSIZE}[0]; + my $filesize = (stat($pkgfile))[7]; + unless ($csize == $filesize) { + $self->_output(sprintf("Package file has incorrect size: %d vs %d: %s\n", $csize, $filesize, $pkgfile)); + return 1; + } + return 0; +} + +sub _verify_package_checksum { + my $self = shift; + my $dbdata = shift; + my $pkgfile = shift; + + my $md5 = Digest::MD5->new; + my $sha = Digest::SHA->new(256); + + my $content; + # 128MiB to keep random IO low when using multiple threads (only works for large packages though) + my $chunksize = 1024*1024*128; + open my $fh, "<", $pkgfile; + while (read($fh, $content, $chunksize)) { + $md5->add($content); + $sha->add($content); + } + + my $expected_sha = $dbdata->{SHA256SUM}[0]; + my $expected_md5 = $dbdata->{MD5SUM}[0]; + my $got_md5 = $md5->hexdigest; + my $got_sha = $sha->hexdigest; + + unless ($expected_sha eq $got_sha and $expected_md5 eq $got_md5) { + my $output; + $output .= sprintf "Package file has incorrect checksum: %s\n", $pkgfile; + $output .= sprintf "expected: SHA %s\n", $expected_sha; + $output .= sprintf "got: SHA %s\n", $got_sha; + $output .= sprintf "expected: MD5 %s\n", $expected_md5; + $output .= sprintf "got: MD5 %s\n", $got_md5; + $self->_output($output); + return 1; + } + return 0; +} + -- 2.9.0

3 11

[pacman-dev] [PATCH] Clarify alpm-hooks(5)
by Přemysl Janouch 07 Aug '16

07 Aug '16

While studying gtk-update-icon-cache.hook, which had the following contents: Target = usr/share/icons/*/ Target = !usr/share/icons/*/?* it was not obvious from the documentation what and how this matches, which I'm trying to rectify here, after some consultation with Andrew. Signed-off-by: Přemysl Janouch <p.janouch(a)gmail.com> --- doc/alpm-hooks.5.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/alpm-hooks.5.txt b/doc/alpm-hooks.5.txt index 7c523e2..80f9d09 100644 --- a/doc/alpm-hooks.5.txt +++ b/doc/alpm-hooks.5.txt @@ -58,10 +58,11 @@ defined the hook will run if the transaction matches *any* of the triggers. *Target =* <path|package>:: The file path or package name to match against the active transaction. - File paths refer to the files in the package archive; the installation root - should *not* be included in the path. Shell-style glob patterns are - allowed. It is possible to invert matches by prepending a file with an - exclamation mark. May be specified multiple times. Required. + File paths refer to the files and directories in the package archive; + the installation root should *not* be included in the path. Shell-style + glob patterns are allowed. It is possible to exclude matches by prepending + the path with an exclamation mark. May be specified multiple times, in + which case they are checked in reverse order until one matches. Required. ACTIONS ------- -- 2.9.0

2 1

[pacman-dev] [PATCH] Normalize alpm download callback function's args to frontend
by ivy.foster＠gmail.com 07 Aug '16

07 Aug '16

In response to my recent patches attempting to fix pacman's repetitive output with --noprogressbar, agregory kindly pointed out an obvious error I missed and suggested that I tackle the problem by making alpm's download callback function return normalized output[1]. [1]: https://wiki.archlinux.org/index.php/User:Apg#download_callback I deviated only slightly from his suggested spec. - When the total download size is unknown, return before calling the frontend's callback function. This is libalpm's original behavior. - Do not attempt to report download errors. I don't see any indication in the curl docs that it's possible for the progress function to be called after a download error. I'm happy to add either of those conditions, of course. Oh, and agregory was quite right: a side effect of this change is that it fixes the repeating-output bug that my previous, more complicated patches attempted to. As always, I welcome critiques. iff

2 2

[pacman-dev] [PATCH] libmakepkg: look for architecture-specific hashes in get_integlist
by Jack O'Connor 01 Aug '16

01 Aug '16

`makepkg -g` looks for existing checksums in the PKGBUILD file, so that it can generate new sums of the same type. Previously it only checked variables of the form "sha256sums", and not "sha256sums_x86_64". That meant it would always fall back to MD5 for packages with only architecture-specific sources. This change makes it look at architecture-specific checksums too to determine the type. Signed-off-by: Jack O'Connor <oconnor663(a)gmail.com> --- scripts/libmakepkg/util/pkgbuild.sh.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/libmakepkg/util/pkgbuild.sh.in b/scripts/libmakepkg/util/pkgbuild.sh.in index 1a4567d..2157b56 100644 --- a/scripts/libmakepkg/util/pkgbuild.sh.in +++ b/scripts/libmakepkg/util/pkgbuild.sh.in @@ -195,10 +195,21 @@ get_integlist() { local integlist=() for integ in "${known_hash_algos[@]}"; do + # check for e.g. "sha256sums" local sumname="${integ}sums[@]" if [[ -n ${!sumname} ]]; then integlist+=("$integ") + continue fi + + # check for e.g. "sha256sums_x86_64" + for a in "${arch[@]}"; do + local sumname="${integ}sums_${a}[@]" + if [[ -n ${!sumname} ]]; then + integlist+=("$integ") + break + fi + done done if (( ${#integlist[@]} > 0 )); then -- 2.9.0

2 1

[pacman-dev] [PATCH] Update email address in copyright message
by Giulio Fidente 28 Jul '16

28 Jul '16

Signed-off-by: Giulio Fidente <gfidente(a)gmail.com> --- contrib/pacscripts.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pacscripts.sh.in b/contrib/pacscripts.sh.in index 0edd89d..4264777 100644 --- a/contrib/pacscripts.sh.in +++ b/contrib/pacscripts.sh.in @@ -3,7 +3,7 @@ # pacscripts : tries to print out the {pre,post}_{install,remove,upgrade} # scripts of a given package # -# Copyright (c) 2009 Giulio "giulivo" Fidente <giulivo.navigante(a)gmail.com> +# Copyright (c) 2009 Giulio Fidente <gfidente(a)gmail.com> # Copyright (c) 2009 Xavier Chantry <shiningxc(a)gmail.com> # Copyright (c) 2009-2016 Pacman Development Team <pacman-dev(a)archlinux.org> # -- 2.9.0

2 5

[pacman-dev] [translation]
by Viktor Alojzije Ćorić 11 Jul '16

11 Jul '16

I've translated what could be translated. I've used poedit, I did not follow the exact instructions from your translation help, so can you please let me know if I did something wrong so I can correct it. Also, this is my first contribution to the community :) Cheers!

2 1

[pacman-dev] Error in patchset
by Ivy Foster 04 Jul '16

04 Jul '16

In the IRC channel, agregory immediately pointed out a pretty obvious bug I missed: the third patch caues pacman to print "downloading <reponame>" even if the repository is up-to-date. My apologies; I don't know how I missed that, and I'll put it right.

1 0

[pacman-dev] Another attempt at fixing sync printing multiple times with --noprogressbar
by ivy.foster＠gmail.com 04 Jul '16

04 Jul '16

Last month, I sent in a patch[1] trying to fix the issue that when pacman is run with --noprogressbar, pacman sometimes prints the message "downloading foo" multiple times. Allan suggested that I fix the underlying issue; looking at a previous thread[2], I gather that the issue is that a callback function called multiple times isn't the correct place to print something just once. [1]: https://lists.archlinux.org/pipermail/pacman-dev/2016-June/021149.html [2]: https://lists.archlinux.org/pipermail/pacman-dev/2015-May/020094.html To try and fix it properly, I've sent the following patches: 1) Add new alpm event type for database refresh events 2) Raise database refresh events for refresh start, success and failure 3) Move printing the informational message from the progress callback function to the event handler The main caveat here is that originally, pacman would print "downloading core.db", but now prints "downloading core". There is no user-visible change in package downloading. I tested the changes in valgrind; there are no leaks. I look forward to your critiques. (-: Cheers, Ivy

1 3

[pacman-dev] [PATCH] Add newline to the end of error messages for signature format issues
by Allan McRae 03 Jul '16

03 Jul '16

Signed-off-by: Allan McRae <allan(a)archlinux.org> --- I'll add this to the list of things I should manually update in the translation files to avoid duplicate work occuring. lib/libalpm/signing.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/libalpm/signing.c b/lib/libalpm/signing.c index 0267158..6557c20 100644 --- a/lib/libalpm/signing.c +++ b/lib/libalpm/signing.c @@ -998,21 +998,21 @@ int SYMEXPORT alpm_extract_keyid(alpm_handle_t *handle, const char *identifier, while(pos < len) { if(!(sig[pos] & 0x80)) { _alpm_log(handle, ALPM_LOG_ERROR, - _("%s: signature format error"), identifier); + _("%s: signature format error\n"), identifier); return -1; } if(sig[pos] & 0x40) { /* "new" packet format is not supported */ _alpm_log(handle, ALPM_LOG_ERROR, - _("%s: unsupported signature format"), identifier); + _("%s: unsupported signature format\n"), identifier); return -1; } if(((sig[pos] & 0x3f) >> 2) != 2) { /* signature is not a "Signature Packet" */ _alpm_log(handle, ALPM_LOG_ERROR, - _("%s: signature format error"), identifier); + _("%s: signature format error\n"), identifier); return -1; } @@ -1035,21 +1035,21 @@ int SYMEXPORT alpm_extract_keyid(alpm_handle_t *handle, const char *identifier, case 3: /* partial body length not supported */ _alpm_log(handle, ALPM_LOG_ERROR, - _("%s: unsupported signature format"), identifier); + _("%s: unsupported signature format\n"), identifier); return -1; } if(sig[pos] != 4) { /* only support version 4 signature packet format */ _alpm_log(handle, ALPM_LOG_ERROR, - _("%s: unsupported signature format"), identifier); + _("%s: unsupported signature format\n"), identifier); return -1; } if(sig[pos + 1] != 0x00) { /* not a signature of a binary document */ _alpm_log(handle, ALPM_LOG_ERROR, - _("%s: signature format error"), identifier); + _("%s: signature format error\n"), identifier); return -1; } -- 2.9.0

1 0