[pacman-dev] [PATCH 1/7] pacman: add -w to -U
Mostly for testing. Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com> --- doc/pacman.8.asciidoc | 6 +++--- src/pacman/pacman.c | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/pacman.8.asciidoc b/doc/pacman.8.asciidoc index 476c16f3..bb205627 100644 --- a/doc/pacman.8.asciidoc +++ b/doc/pacman.8.asciidoc @@ -241,6 +241,9 @@ Transaction Options (apply to '-S', '-R' and '-U') Upgrade Options (apply to '-S' and '-U')[[UO]] ---------------------------------------------- +*-w, \--downloadonly*:: + Retrieve all packages from the server, but do not install/upgrade anything. + *\--asdeps*:: Install packages non-explicitly; in other words, fake their install reason to be installed as a dependency. This is useful for makepkg and other @@ -445,9 +448,6 @@ can be useful when the user switches from a testing repository to a stable one. Additional targets can also be specified manually, so that '-Su foo' will do a system upgrade and install/upgrade the "foo" package in the same operation. -*-w, \--downloadonly*:: - Retrieve all packages from the server, but do not install/upgrade anything. - *-y, \--refresh*:: Download a fresh copy of the master package database from the server(s) defined in linkman:pacman.conf[5]. This should typically be used each time diff --git a/src/pacman/pacman.c b/src/pacman/pacman.c index 3045a58e..cc280d41 100644 --- a/src/pacman/pacman.c +++ b/src/pacman/pacman.c @@ -161,7 +161,6 @@ static void usage(int op, const char * const myname) addlist(_(" -q, --quiet show less information for query and search\n")); addlist(_(" -s, --search <regex> search remote repositories for matching strings\n")); addlist(_(" -u, --sysupgrade upgrade installed packages (-uu enables downgrades)\n")); - addlist(_(" -w, --downloadonly download packages but do not install/upgrade anything\n")); addlist(_(" -y, --refresh download fresh package databases from the server\n" " (-yy to force a refresh even if up to date)\n")); addlist(_(" --needed do not reinstall up to date packages\n")); @@ -189,6 +188,7 @@ static void usage(int op, const char * const myname) switch(op) { case PM_OP_SYNC: case PM_OP_UPGRADE: + addlist(_(" -w, --downloadonly download packages but do not install/upgrade anything\n")); addlist(_(" --overwrite <glob>\n" " overwrite conflicting files (can be used more than once)\n")); addlist(_(" --asdeps install packages as non-explicitly installed\n")); @@ -735,6 +735,12 @@ static int parsearg_upgrade(int opt) case OP_IGNOREGROUP: parsearg_util_addlist(&(config->ignoregrp)); break; + case OP_DOWNLOADONLY: + case 'w': + config->op_s_downloadonly = 1; + config->flags |= ALPM_TRANS_FLAG_DOWNLOADONLY; + config->flags |= ALPM_TRANS_FLAG_NOCONFLICTS; + break; default: return 1; } return 0; @@ -820,12 +826,6 @@ static int parsearg_sync(int opt) case 'u': (config->op_s_upgrade)++; break; - case OP_DOWNLOADONLY: - case 'w': - config->op_s_downloadonly = 1; - config->flags |= ALPM_TRANS_FLAG_DOWNLOADONLY; - config->flags |= ALPM_TRANS_FLAG_NOCONFLICTS; - break; case OP_REFRESH: case 'y': (config->op_s_sync)++; -- 2.30.0
Makes it easier to pass options when not running pactest directly. Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com> --- test/pacman/pactest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pacman/pactest.py b/test/pacman/pactest.py index 20af41dc..c295bfc0 100755 --- a/test/pacman/pactest.py +++ b/test/pacman/pactest.py @@ -125,7 +125,8 @@ def create_parser(): # parse options opt_parser = create_parser() - (opts, args) = opt_parser.parse_args() + (opts, args) = opt_parser.parse_args(args=os.getenv('PACTEST_OPTS', '').split()) + (opts, args) = opt_parser.parse_args(values=opts) if args is None or len(args) == 0: tap.bail("no tests defined, nothing to do") -- 2.30.0
--- .gitlab-ci.yml | 5 +++++ build-aux/cat-test-file | 11 +++++++++++ build-aux/print-failed-test-output | 17 +++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100755 build-aux/cat-test-file create mode 100755 build-aux/print-failed-test-output diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e889c498..6ca35941 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ variables: MAKEFLAGS: "-j10" VERBOSE: 1 PACMAN_OPTS: --needed --noconfirm --cachedir .pkg-cache + PACTEST_OPTS: --review --editor=../build-aux/cat-test-file cache: key: pkgs-v1 @@ -9,6 +10,10 @@ cache: # For some reason Gitlab CI only supports storing cache/artifacts in a path relative to the build directory - .pkg-cache +default: + after_script: + - build-aux/print-failed-test-output build/meson-logs/testlog.json + .arch-test: image: archlinux/base before_script: diff --git a/build-aux/cat-test-file b/build-aux/cat-test-file new file mode 100755 index 00000000..1bcc1113 --- /dev/null +++ b/build-aux/cat-test-file @@ -0,0 +1,11 @@ +#!/usr/bin/python3 + +import sys + +for path in sys.argv[1:]: + print('# -----------------------------------') + print('# ' + path + ':') + print('# -----------------------------------') + with open(path, 'r') as f: + for line in f: + print('# ' + line, end='') diff --git a/build-aux/print-failed-test-output b/build-aux/print-failed-test-output new file mode 100755 index 00000000..1f1ab76d --- /dev/null +++ b/build-aux/print-failed-test-output @@ -0,0 +1,17 @@ +#!/usr/bin/python + +import json +import sys + +def print_result(result): + print('==================================================================') + print(result['name']) + print(' '.join(result['command'])) + print('==================================================================') + print(result['stdout']) + +with open(sys.argv[1], 'r') as f: + for line in f: + result = json.loads(line) + if result['result'] == 'FAIL': + print_result(result) -- 2.30.0
The existing CACHE_EXISTS rule takes a package, which is not suitable for -U tests that need to be able to check for specific files. Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com> --- test/pacman/pmrule.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/pacman/pmrule.py b/test/pacman/pmrule.py index 59293266..aef73cdb 100644 --- a/test/pacman/pmrule.py +++ b/test/pacman/pmrule.py @@ -181,6 +181,16 @@ def check(self, test): if not pkg or not os.path.isfile( os.path.join(cachedir, pkg.filename())): success = 0 + elif case == "FEXISTS": + if not os.path.isfile(os.path.join(cachedir, key)): + success = 0 + elif case == "FCONTENTS": + filename = os.path.join(cachedir, key) + try: + with open(filename, 'r') as f: + success = f.read() == value + except: + success = 0 else: tap.diag("Rule kind '%s' not found" % kind) success = -1 -- 2.30.0
Populating a file:// Server prevents any manually registered HTTP servers from ever being used. Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com> --- test/pacman/pmdb.py | 1 + test/pacman/pmtest.py | 7 ++++--- test/pacman/util.py | 9 ++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test/pacman/pmdb.py b/test/pacman/pmdb.py index 971e28eb..cae51b4e 100644 --- a/test/pacman/pmdb.py +++ b/test/pacman/pmdb.py @@ -54,6 +54,7 @@ def __init__(self, treename, root): self.root = root self.pkgs = [] self.option = {} + self.syncdir = True if self.treename == "local": self.dbdir = os.path.join(root, util.PM_DBPATH, treename) self.dbfile = None diff --git a/test/pacman/pmtest.py b/test/pacman/pmtest.py index ed6eab6a..8669f31d 100644 --- a/test/pacman/pmtest.py +++ b/test/pacman/pmtest.py @@ -167,10 +167,11 @@ def generate(self, pacman): vprint("\t%s" % os.path.join(util.PM_CACHEDIR, pkg.filename())) if self.cachepkgs: pkg.makepkg(cachedir) - else: + elif value.syncdir: pkg.makepkg(os.path.join(syncdir, value.treename)) - pkg.md5sum = util.getmd5sum(pkg.path) - pkg.csize = os.stat(pkg.path)[stat.ST_SIZE] + if pkg.path: + pkg.md5sum = util.getmd5sum(pkg.path) + pkg.csize = os.stat(pkg.path)[stat.ST_SIZE] # Creating sync database archives vprint(" Creating databases") diff --git a/test/pacman/util.py b/test/pacman/util.py index 1e0eb385..2957a020 100644 --- a/test/pacman/util.py +++ b/test/pacman/util.py @@ -122,11 +122,10 @@ def mkcfgfile(filename, root, option, db): for key in sorted(db.keys()): if key != "local": value = db[key] - data.append("[%s]\n" \ - "SigLevel = %s\n" \ - "Server = file://%s" \ - % (value.treename, value.getverify(), \ - os.path.join(root, SYNCREPO, value.treename))) + data.append("[%s]\n" % (value.treename)) + data.append("SigLevel = %s\n" % (value.getverify())) + if value.syncdir: + data.append("Server = file://%s" % (os.path.join(root, SYNCREPO, value.treename))) for optkey, optval in value.option.items(): data.extend(["%s = %s" % (optkey, j) for j in optval]) -- 2.30.0
Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com> --- test/pacman/README | 23 ++++++++++++ test/pacman/pmserve.py | 84 ++++++++++++++++++++++++++++++++++++++++++ test/pacman/pmtest.py | 28 ++++++++++++++ 3 files changed, 135 insertions(+) create mode 100644 test/pacman/pmserve.py diff --git a/test/pacman/README b/test/pacman/README index 224acd84..61a67d05 100644 --- a/test/pacman/README +++ b/test/pacman/README @@ -317,3 +317,26 @@ Example: pactest will ensure the file /etc/test.conf exists in the filesystem. +Serving Files +============= + +Tests can run a simple http server using the `add_simple_http_server` method, +which takes a dict with request paths for keys and their responses as values +and returns a url for the server. Responses may either be a simple string or +a dict with the following keys: `code`, `headers`, and `body`. If a value is +provided for the empty path it will be used as a fallback response for any +requests that do not match a path. + + url = self.add_simple_http_server({ + "/": "simple response", + "/custom": { + "headers": { "Content-Disposition": "attachment; filename=foo" }, + "body": ("Custom response. Code and any necessary headers " + "will by automatically set if not provided"), + } + "": { + "code": 404, + "headers": { "Content-Length": "14" }, + "body": "Page Not Found", + } + }) diff --git a/test/pacman/pmserve.py b/test/pacman/pmserve.py new file mode 100644 index 00000000..0ca34093 --- /dev/null +++ b/test/pacman/pmserve.py @@ -0,0 +1,84 @@ +# Copyright (c) 2020 Pacman Development Team <pacman-dev@archlinux.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import http +import http.server +import sys +import re + +class pmHTTPServer(http.server.ThreadingHTTPServer): + pass + +class pmHTTPRequestHandler(http.server.BaseHTTPRequestHandler): + """BaseHTTPRequestHandler subclass with helper methods and common setup""" + + logfile = sys.stderr + + def respond(self, response, headers={}, code=200): + self.protocol_version = "HTTP/1.1" + self.send_response(code) + for header, value in headers.items(): + self.send_header(header, value) + self.end_headers() + self.wfile.write(response) + + def parse_range_bytes(self, text): + parser = re.compile(r'^bytes=(\d+)-(\d+)?$') + if m := parser.match(text): + return map(lambda d: None if d is None else int(d), m.groups()) + else: + raise ValueError("Unrecognized Range value") + + def respond_bytes(self, response, headers={}, code=200): + headers = headers.copy() + if code == 200 and self.headers['Range']: + (start, end) = self.parse_range_bytes(self.headers['Range']) + code = 206 + response = response[start:end] + headers.setdefault('Content-Range', 'bytes */%s' % (len(response))) + headers.setdefault('Content-Type', "application/octet-stream") + headers.setdefault('Content-Length', str(len(response))) + self.respond(response, headers, code) + + def respond_string(self, response, headers={}, code=200): + headers = headers.copy() + headers.setdefault('Content-Type', 'text/plain; charset=utf-8') + self.respond_bytes(response.encode('UTF-8'), headers, code) + + def log_message(self, format, *args): + if callable(self.logfile): + self.logfile = self.logfile() + self.logfile.write("%s - - [%s] %s\n" % + (self.address_string(), + self.log_date_time_string(), + format%args)) + +class pmStringHTTPRequestHandler(pmHTTPRequestHandler): + """pmHTTPRequestHandler subclass to respond with simple string messages""" + + responses = dict() + + def do_GET(self): + response = self.responses.get(self.path, self.responses.get('')) + if response is not None: + if isinstance(response, dict): + self.respond_string( + response.get('body', ''), + headers=response.get('headers', {}), + code=response.get('code', 200)) + else: + self.respond_string(response) + else: + self.send_error(http.HTTPStatus.NOT_FOUND); diff --git a/test/pacman/pmtest.py b/test/pacman/pmtest.py index 8669f31d..7e62fc92 100644 --- a/test/pacman/pmtest.py +++ b/test/pacman/pmtest.py @@ -20,9 +20,11 @@ import shutil import stat import subprocess +import threading import time import pmrule +import pmserve import pmdb import pmfile import tap @@ -47,6 +49,8 @@ def __init__(self, name, root, config): "--hookdir", self.hookdir(), "--cachedir", self.cachedir()] + self.http_servers = [] + def __str__(self): return "name = %s\n" \ "testname = %s\n" \ @@ -285,6 +289,8 @@ def run(self, pacman): output = None vprint("\trunning: %s" % " ".join(cmd)) + self.start_http_servers() + # Change to the tmp dir before running pacman, so that local package # archives are made available more easily. time_start = time.time() @@ -293,6 +299,8 @@ def run(self, pacman): time_end = time.time() vprint("\ttime elapsed: %.2fs" % (time_end - time_start)) + self.stop_http_servers() + if output: output.close() @@ -330,3 +338,23 @@ def cachedir(self): def hookdir(self): return os.path.join(self.root, util.PM_HOOKDIR) + + def add_simple_http_server(self, responses): + logfile = lambda h: open(os.path.join(self.root, 'var/log/httpd.log'), 'a') + handler = type(self.name + 'HTTPServer', + (pmserve.pmStringHTTPRequestHandler,), + {'responses': responses, 'logfile': logfile}) + server = pmserve.pmHTTPServer(('127.0.0.1', 0), handler) + self.http_servers.append(server) + host, port = server.server_address[:2] + return 'http://%s:%d' % (host, port) + + def start_http_servers(self): + for srv in self.http_servers: + thread = threading.Thread(target=srv.serve_forever) + thread.daemon = True + thread.start() + + def stop_http_servers(self): + for srv in self.http_servers: + srv.shutdown() -- 2.30.0
Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com> --- test/pacman/meson.build | 3 + test/pacman/tests/upgrade-download-404.py | 10 +++ ...rade-download-pkg-and-sig-with-filename.py | 61 ++++++++++++++++++ ...e-download-pkg-and-sig-without-filename.py | 63 +++++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 test/pacman/tests/upgrade-download-404.py create mode 100644 test/pacman/tests/upgrade-download-pkg-and-sig-with-filename.py create mode 100644 test/pacman/tests/upgrade-download-pkg-and-sig-without-filename.py diff --git a/test/pacman/meson.build b/test/pacman/meson.build index fd0dbf90..52ff9b9a 100644 --- a/test/pacman/meson.build +++ b/test/pacman/meson.build @@ -328,6 +328,9 @@ pacman_tests = [ 'tests/upgrade090.py', 'tests/upgrade100.py', 'tests/xfercommand001.py', + 'tests/upgrade-download-404.py', + 'tests/upgrade-download-pkg-and-sig-with-filename.py', + 'tests/upgrade-download-pkg-and-sig-without-filename.py', ] xfail_tests = { diff --git a/test/pacman/tests/upgrade-download-404.py b/test/pacman/tests/upgrade-download-404.py new file mode 100644 index 00000000..ed5ad2b4 --- /dev/null +++ b/test/pacman/tests/upgrade-download-404.py @@ -0,0 +1,10 @@ +self.description = 'download a remote package with -U' +self.require_capability("curl") + +url = self.add_simple_http_server({}) + +self.args = '-Uw {url}/foo.pkg'.format(url=url) + +self.addrule('!PACMAN_RETCODE=0') +self.addrule('!CACHE_FEXISTS=foo.pkg') +self.addrule('!CACHE_FEXISTS=foo.pkg.sig') diff --git a/test/pacman/tests/upgrade-download-pkg-and-sig-with-filename.py b/test/pacman/tests/upgrade-download-pkg-and-sig-with-filename.py new file mode 100644 index 00000000..ac5072c9 --- /dev/null +++ b/test/pacman/tests/upgrade-download-pkg-and-sig-with-filename.py @@ -0,0 +1,61 @@ +self.description = 'download remote packages with -U with a URL filename' +self.require_capability("gpg") +self.require_capability("curl") + +url = self.add_simple_http_server({ + # simple + '/simple.pkg': 'simple', + '/simple.pkg.sig': { + 'headers': { 'Content-Disposition': 'attachment; filename="simple.sig-alt' }, + 'body': 'simple.sig', + }, + + # content-disposition filename + '/cd.pkg': { + 'headers': { 'Content-Disposition': 'attachment; filename="cd-alt.pkg"' }, + 'body': 'cd' + }, + '/cd.pkg.sig': 'cd.sig', + + # redirect + '/redir.pkg': { 'code': 303, 'headers': { 'Location': '/redir-dest.pkg' } }, + '/redir-dest.pkg': 'redir-dest', + '/redir-dest.pkg.sig': 'redir-dest.sig', + + # content-disposition and redirect + '/cd-redir.pkg': { 'code': 303, 'headers': { 'Location': '/cd-redir-dest.pkg' } }, + '/cd-redir-dest.pkg': { + 'headers': { 'Content-Disposition': 'attachment; filename="cd-redir-dest-alt.pkg"' }, + 'body': 'cd-redir-dest' + }, + '/cd-redir-dest.pkg.sig': 'cd-redir-dest.sig', + + # TODO: absolutely terrible hack to prevent pacman from attempting to + # validate packages, which causes failure under --valgrind thanks to + # a memory leak in gpgme that is too general for inclusion in valgrind.supp + '/404': { 'code': 404 }, + + '': 'fallback', +}) + +self.args = '-Uw {url}/simple.pkg {url}/cd.pkg {url}/redir.pkg {url}/cd-redir.pkg {url}/404'.format(url=url) + +# packages/sigs are not valid, error is expected +self.addrule('!PACMAN_RETCODE=0') + +self.addrule('CACHE_FCONTENTS=simple.pkg|simple') +self.addrule('CACHE_FCONTENTS=simple.pkg.sig|simple.sig') + +self.addrule('!CACHE_FEXISTS=cd.pkg') +self.addrule('!CACHE_FEXISTS=cd.pkg.sig') +self.addrule('CACHE_FCONTENTS=cd-alt.pkg|cd') +self.addrule('CACHE_FCONTENTS=cd-alt.pkg.sig|cd.sig') + +self.addrule('!CACHE_FEXISTS=redir.pkg') +self.addrule('CACHE_FCONTENTS=redir-dest.pkg|redir-dest') +self.addrule('CACHE_FCONTENTS=redir-dest.pkg.sig|redir-dest.sig') + +self.addrule('!CACHE_FEXISTS=cd-redir.pkg') +self.addrule('!CACHE_FEXISTS=cd-redir-dest.pkg') +self.addrule('CACHE_FCONTENTS=cd-redir-dest-alt.pkg|cd-redir-dest') +self.addrule('CACHE_FCONTENTS=cd-redir-dest-alt.pkg.sig|cd-redir-dest.sig') diff --git a/test/pacman/tests/upgrade-download-pkg-and-sig-without-filename.py b/test/pacman/tests/upgrade-download-pkg-and-sig-without-filename.py new file mode 100644 index 00000000..651fb3f5 --- /dev/null +++ b/test/pacman/tests/upgrade-download-pkg-and-sig-without-filename.py @@ -0,0 +1,63 @@ +self.description = 'download remote packages with -U without a URL filename' +self.require_capability("gpg") +self.require_capability("curl") + +url = self.add_simple_http_server({ + # simple + '/simple.pkg/': 'simple', + '/simple.pkg/.sig': 'simple.sig', + + # content-disposition filename + '/cd.pkg/': { + 'headers': { 'Content-Disposition': 'attachment; filename="cd-alt.pkg"' }, + 'body': 'cd' + }, + '/cd.pkg/.sig': { + 'headers': { 'Content-Disposition': 'attachment; filename="cd-alt-bad.pkg.sig"' }, + 'body': 'cd.sig' + }, + + # redirect + '/redir.pkg/': { 'code': 303, 'headers': { 'Location': '/redir-dest.pkg' } }, + '/redir-dest.pkg': 'redir-dest', + '/redir-dest.pkg.sig': 'redir-dest.sig', + + # content-disposition and redirect + '/cd-redir.pkg/': { 'code': 303, 'headers': { 'Location': '/cd-redir-dest.pkg' } }, + '/cd-redir-dest.pkg': { + 'headers': { 'Content-Disposition': 'attachment; filename="cd-redir-dest-alt.pkg"' }, + 'body': 'cd-redir-dest' + }, + '/cd-redir-dest.pkg.sig': 'cd-redir-dest.sig', + + # TODO: absolutely terrible hack to prevent pacman from attempting to + # validate packages, which causes failure under --valgrind thanks to + # a memory leak in gpgme that is too general for inclusion in valgrind.supp + '/404': { 'code': 404 }, + + '': 'fallback', +}) + +self.args = '-Uw {url}/simple.pkg/ {url}/cd.pkg/ {url}/redir.pkg/ {url}/cd-redir.pkg/ {url}/404'.format(url=url) + +# packages/sigs are not valid, error is expected +self.addrule('!PACMAN_RETCODE=0') + +# TODO: use a predictable file name +#self.addrule('CACHE_FCONTENTS=simple.pkg|simple') +#self.addrule('CACHE_FCONTENTS=simple.pkg.sig|simple.sig') + +self.addrule('!CACHE_FEXISTS=cd.pkg') +self.addrule('CACHE_FCONTENTS=cd-alt.pkg|cd') +self.addrule('CACHE_FCONTENTS=cd-alt.pkg.sig|cd.sig') + +self.addrule('!CACHE_FEXISTS=redir.pkg') +self.addrule('CACHE_FCONTENTS=redir-dest.pkg|redir-dest') +self.addrule('CACHE_FCONTENTS=redir-dest.pkg.sig|redir-dest.sig') + +self.addrule('!CACHE_FEXISTS=cd-redir.pkg') +self.addrule('!CACHE_FEXISTS=cd-redir-dest.pkg') +self.addrule('CACHE_FCONTENTS=cd-redir-dest-alt.pkg|cd-redir-dest') +self.addrule('CACHE_FCONTENTS=cd-redir-dest-alt.pkg.sig|cd-redir-dest.sig') + +self.addrule('!CACHE_FEXISTS=.sig') -- 2.30.0
participants (1)
-
Andrew Gregory