On 19/07/16 00:28, Florian Pritz wrote:
Signed-off-by: Florian Pritz <bluewind@xinu.at>
A commit message would be nice... Is there any reason PGP checksums are not checked?
--- contrib/doc/verify-pacman-repo-db.1.txt | 50 ++++++ contrib/verify-pacman-repo-db.pl | 279 ++++++++++++++++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 contrib/doc/verify-pacman-repo-db.1.txt create mode 100755 contrib/verify-pacman-repo-db.pl
diff --git a/contrib/doc/verify-pacman-repo-db.1.txt b/contrib/doc/verify-pacman-repo-db.1.txt new file mode 100644 index 0000000..01eafbe --- /dev/null +++ b/contrib/doc/verify-pacman-repo-db.1.txt @@ -0,0 +1,50 @@ +///// +vim:set ts=4 sw=4 syntax=asciidoc noet spell spelllang=en_us: +///// +verify-pacman-repo-db(1) +======================= + +Name +---- +verify-pacman-repo-db - package repository verification utility + + +Synopsis +-------- +'verify-pacman-repo-db' [options] + + +Description +----------- +'verify-pacman-repo-db' looks at a pacman repo database and verifies its +content with the actual package files. The database is expected to be in +the same directory as the packages (or symlinks to the packages). + +The following properties are verified for each package in the database: + + - existence of the package file + - file size + - MD5 and SHA256 checksum (--checksum) + +Options +------- +*-h, \--help*:: + Output a short help message. + +*\--debug*:: + Enable debug output. + +*-c, \--checksum*:: + Verify checksums of packages. Note that this means all packages files will + be read from disk. Otherwise only metadata is compared which does not + require to read package file contents. + +*-t, \--threads* <number>:: + Use 'number' threads to verify packages. Note that each thread may use up + to approximately 128MiB of memory. Default: 1 + +See Also +-------- +linkman:repo-add[8] + +include::footer.txt[] diff --git a/contrib/verify-pacman-repo-db.pl b/contrib/verify-pacman-repo-db.pl new file mode 100755 index 0000000..71cfddf --- /dev/null +++ b/contrib/verify-pacman-repo-db.pl @@ -0,0 +1,279 @@ +#!/usr/bin/perl -T +use warnings; +use strict; + +=pod + +=head1 SYNOPSIS + +verify-pacman-repo-db.pl [options] <database file> ... + + Options: + --help, -h Show short help message + --debug Enable debug output + --checksum, -c Verify checksums of packages + --thread, -t <num> Use num threads to verify packages. Default: 1 + NOTE: Each thread uses up to approx. 128MiB of memory + +=head1 DESCRIPTION + +verify-pacman-repo-db.pl looks at a pacman repo database and verifies its content +with the actual package files. The database is expected to be in the same +directory as the packages (or symlinks to the packages). + +The following properties are verified for each package in the database: + + - existence of the package file + - file size + - MD5 and SHA256 checksum (--checksum) + +=head1 NOTES + +This script does intentionally not use any ALPM libraries. The format is simple +enough to be parsed and this way we might just detect more problems because the +libalpm parsing code might also have bugs. We also stay much more portable +which might be good for people that want to check a db, but don't actually have +pacman installed. + +=cut + +package main; +use Getopt::Long; +use Pod::Usage; + +exit main(); + +sub main { + my %opts = ( + threads => 1, + ); + + Getopt::Long::Configure ("bundling"); + pod2usage(-verbose => 0) if (@ARGV== 0); + GetOptions(\%opts, "help|h", "debug", "threads|t=i", "checksum|c") or pod2usage(2); + pod2usage(0) if $opts{help}; + + my $verifier = Verifier->new(\%opts); + + for my $repodb (@ARGV) { + $verifier->check_repodb($repodb); + } + + $verifier->finalize(); + return $verifier->get_error_status(); +} + +package Verifier; +use Archive::Tar; +use Digest::MD5; +use Digest::SHA; +use File::Basename; +use threads; +use threads::shared; +use Thread::Queue; + +sub new { + my $class = shift; + my $opts = shift; + + my $self :shared = shared_clone({ + opts => \%{$opts}, + package_queue => Thread::Queue->new(), + output_queue => Thread::Queue->new(), + workers => [], + errors => 0, + }); + + bless $self, $class; + $self->start_workers(); + return $self; +} + +sub start_workers { + my $self = shift; + + threads->new(\&_worker_output_queue, $self); + + for (my $i = 0; $i < $self->{opts}->{threads}; $i++) { + my $thr :shared = shared_clone(threads->new(\&_worker_package_queue, $self)); + push @{$self->{workers}}, $thr; + } +} + +sub _worker_package_queue { + my $self = shift; + while (my $workpack = $self->{package_queue}->dequeue()) { + my $dbdata = $self->_parse_db_entry($workpack->{db_desc_content}); + $self->{errors} += $self->_verify_db_entry($workpack->{dirname}, $dbdata); + } +} + +sub _worker_output_queue { + my $self = shift; + while (my $output = $self->{output_queue}->dequeue()) { + print STDERR $output; + } +} + +sub finalize { + my $self = shift; + + $self->{package_queue}->end(); + $self->_join_threads($self->{workers}); + + $self->{output_queue}->end(); + $self->_join_threads([threads->list]); +} + +sub _join_threads { + my $self = shift; + my $threads = shift; + + for my $thr (@{$threads}) { + if ($thr->tid && !threads::equal($thr, threads->self)) { + print "waiting for thread ".$thr->tid()." to finish\n" if $self->{opts}->{debug}; + $thr->join; + } + } +} + +sub get_error_status { + my $self = shift; + + return $self->{errors} > 0; +} + +sub check_repodb { + my $self = shift; + my $repodb = shift; + + my $db = Archive::Tar->new(); + $db->read($repodb); + + my $dirname = dirname($repodb); + my $pkgcount = 0; + + my @files = $db->list_files(); + for my $file_object ($db->get_files()) { + if ($file_object->name =~ m/^([^\/]+)\/desc$/) { + my $package = $1; + $self->{package_queue}->enqueue({ + package => $package, + db_desc_content => $file_object->get_content(), + dirname => $dirname, + }); + $pkgcount++; + } + } + + $self->_debug(sprintf("Queued %d package from database '%s'\n", $pkgcount, $repodb)); +} + +sub _parse_db_entry { + my $self = shift; + my $content = shift; + my %db; + my $key; + + for my $line (split /\n/, $content) { + if ($line eq '') { + $key = undef; + next; + } + if ($line =~ m/^%(.+)%$/) { + $key = $1; + } else { + push @{$db{$key}}, $line; + die "\$key not set. Is the db formated incorrectly?" unless $key; + } + } + return \%db; +} + +sub _output { + my $self = shift; + my $output = shift; + + return if $output eq ""; + + $output = sprintf("Thread %s: %s", threads->self->tid(), $output); + $self->{output_queue}->enqueue($output); +} + +sub _debug { + my $self = shift; + my $output = shift; + $self->_output($output) if $self->{opts}->{debug}; +} + +sub _verify_db_entry { + my $self = shift; + my $basedir = shift; + my $dbdata = shift; + my $ret = 0; + my $output = ""; + + # verify package exists + my $pkgfile = $basedir.'/'.$dbdata->{FILENAME}[0]; + $self->_debug(sprintf("Checking package %s\n", $dbdata->{FILENAME}[0])); + unless (-e $pkgfile) { + $self->_output(sprintf("Package file missing: %s\n", $pkgfile)); + return 1; + } + + $ret += $self->_verify_package_size($dbdata, $pkgfile); + $ret += $self->_verify_package_checksum($dbdata, $pkgfile) if $self->{opts}->{checksum}; + # TODO verify gpg sigs? + + return $ret; +} + +sub _verify_package_size { + my $self = shift; + my $dbdata = shift; + my $pkgfile = shift; + + my $csize = $dbdata->{CSIZE}[0]; + my $filesize = (stat($pkgfile))[7]; + unless ($csize == $filesize) { + $self->_output(sprintf("Package file has incorrect size: %d vs %d: %s\n", $csize, $filesize, $pkgfile)); + return 1; + } + return 0; +} + +sub _verify_package_checksum { + my $self = shift; + my $dbdata = shift; + my $pkgfile = shift; + + my $md5 = Digest::MD5->new; + my $sha = Digest::SHA->new(256); + + my $content; + # 128MiB to keep random IO low when using multiple threads (only works for large packages though) + my $chunksize = 1024*1024*128; + open my $fh, "<", $pkgfile; + while (read($fh, $content, $chunksize)) { + $md5->add($content); + $sha->add($content); + } + + my $expected_sha = $dbdata->{SHA256SUM}[0]; + my $expected_md5 = $dbdata->{MD5SUM}[0]; + my $got_md5 = $md5->hexdigest; + my $got_sha = $sha->hexdigest; + + unless ($expected_sha eq $got_sha and $expected_md5 eq $got_md5) { + my $output; + $output .= sprintf "Package file has incorrect checksum: %s\n", $pkgfile; + $output .= sprintf "expected: SHA %s\n", $expected_sha; + $output .= sprintf "got: SHA %s\n", $got_sha; + $output .= sprintf "expected: MD5 %s\n", $expected_md5; + $output .= sprintf "got: MD5 %s\n", $got_md5; + $self->_output($output); + return 1; + } + return 0; +} +