Each test produces a single TAP result with the rules run in a sub-test. This reduces output when run under automake and makes it possible to continue setting expectfailure at the test level rather than per-rule. Signed-off-by: Andrew Gregory <andrew.gregory.8@gmail.com> --- test/pacman/pactest.py | 5 ++-- test/pacman/pmdb.py | 5 ++-- test/pacman/pmenv.py | 65 +++++++++++++++++++++++--------------------------- test/pacman/pmrule.py | 15 ++++++------ test/pacman/pmtest.py | 22 ++++++++--------- test/pacman/tap.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++ test/pacman/util.py | 4 +++- 7 files changed, 121 insertions(+), 59 deletions(-) create mode 100644 test/pacman/tap.py diff --git a/test/pacman/pactest.py b/test/pacman/pactest.py index 2b1dee6..fe04c2b 100755 --- a/test/pacman/pactest.py +++ b/test/pacman/pactest.py @@ -26,6 +26,7 @@ import tempfile import pmenv +import tap import util __author__ = "Aurelien FORET" @@ -110,7 +111,7 @@ def create_parser(): env.pacman["ldconfig"] = opts.ldconfig if opts.testcases is None or len(opts.testcases) == 0: - print "no tests defined, nothing to do" + tap.bail("no tests defined, nothing to do") os.rmdir(root_path) sys.exit(2) @@ -124,7 +125,7 @@ def create_parser(): if not opts.keeproot: shutil.rmtree(root_path) else: - print "pacman testing root saved: %s" % root_path + tap.diag("pacman testing root saved: %s" % root_path) if env.failed > 0: sys.exit(1) diff --git a/test/pacman/pmdb.py b/test/pacman/pmdb.py index 3e9d305..b7b3522 100644 --- a/test/pacman/pmdb.py +++ b/test/pacman/pmdb.py @@ -23,6 +23,7 @@ import tarfile import pmpkg +import tap import util def _getsection(fd): @@ -105,7 +106,7 @@ def db_read(self, name): # desc filename = os.path.join(path, "desc") if not os.path.isfile(filename): - print "invalid db entry found (desc missing) for pkg", pkgname + tap.bail("invalid db entry found (desc missing) for pkg " + pkgname) return None fd = open(filename, "r") while 1: @@ -160,7 +161,7 @@ def db_read(self, name): # files filename = os.path.join(path, "files") if not os.path.isfile(filename): - print "invalid db entry found (files missing) for pkg", pkgname + tap.bail("invalid db entry found (files missing) for pkg " + pkgname) return None fd = open(filename, "r") while 1: diff --git a/test/pacman/pmenv.py b/test/pacman/pmenv.py index 5eaa473..a3a8f54 100644 --- a/test/pacman/pmenv.py +++ b/test/pacman/pmenv.py @@ -20,6 +20,7 @@ import os import pmtest +import tap class pmenv(object): @@ -58,26 +59,18 @@ def addtest(self, testcase): def run(self): """ """ - + tap.plan(len(self.testcases)) for t in self.testcases: - print "=========="*8 - print "Running '%s'" % t.testname + tap.diag("==========" * 8) + tap.diag("Running '%s'" % t.testname) t.load() - print t.description - print "----------"*8 - t.generate(self.pacman) - t.run(self.pacman) - t.check() - print "==> Test result" - if t.result["fail"] == 0: - print "\tPASS" - else: - print "\tFAIL" - print + tap.diag("==> Checking rules") + tap.todo = t.expectfailure + tap.subtest(lambda: t.check(), t.description) def results(self): """ @@ -109,40 +102,42 @@ def _printtest(t): result = "[PASS]" else: result = "[FAIL]" - print result, - print "%s Rules: OK = %2u FAIL = %2u" \ - % (t.testname.ljust(34), success, fail) + tap.diag("%s %s Rules: OK = %2u FAIL = %2u" \ + % (result, t.testname.ljust(34), success, fail)) if fail != 0: # print test description if test failed - print " ", t.description + tap.diag(" " + t.description) - print "=========="*8 - print "Results" - print "----------"*8 - print " Passed:" + tap.diag("==========" * 8) + tap.diag("Results") + tap.diag("----------" * 8) + tap.diag(" Passed:") for test in tpassed: _printtest(test) - print "----------"*8 - print " Expected Failures:" + tap.diag("----------" * 8) + tap.diag(" Expected Failures:") for test in texpectedfail: _printtest(test) - print "----------"*8 - print " Unexpected Passes:" + tap.diag("----------" * 8) + tap.diag(" Unexpected Passes:") for test in tunexpectedpass: _printtest(test) - print "----------"*8 - print " Failed:" + tap.diag("----------" * 8) + tap.diag(" Failed:") for test in tfailed: _printtest(test) - print "----------"*8 + tap.diag("----------" * 8) total = len(self.testcases) - print "Total = %3u" % total + tap.diag("Total = %3u" % total) if total: - print "Pass = %3u (%6.2f%%)" % (self.passed, float(self.passed) * 100 / total) - print "Expected Fail = %3u (%6.2f%%)" % (self.expectedfail, float(self.expectedfail) * 100 / total) - print "Unexpected Pass = %3u (%6.2f%%)" % (self.unexpectedpass, float(self.unexpectedpass) * 100 / total) - print "Fail = %3u (%6.2f%%)" % (self.failed, float(self.failed) * 100 / total) - print "" + tap.diag("Pass = %3u (%6.2f%%)" % (self.passed, + float(self.passed) * 100 / total)) + tap.diag("Expected Fail = %3u (%6.2f%%)" % (self.expectedfail, + float(self.expectedfail) * 100 / total)) + tap.diag("Unexpected Pass = %3u (%6.2f%%)" % (self.unexpectedpass, + float(self.unexpectedpass) * 100 / total)) + tap.diag("Fail = %3u (%6.2f%%)" % (self.failed, + float(self.failed) * 100 / total)) # vim: set ts=4 sw=4 et: diff --git a/test/pacman/pmrule.py b/test/pacman/pmrule.py index 3d38b85..c97a158 100644 --- a/test/pacman/pmrule.py +++ b/test/pacman/pmrule.py @@ -19,6 +19,7 @@ import os import stat +import tap import util class pmrule(object): @@ -57,12 +58,12 @@ def check(self, test): elif case == "OUTPUT": logfile = os.path.join(test.root, util.LOGFILE) if not os.access(logfile, os.F_OK): - print "LOGFILE not found, cannot validate 'OUTPUT' rule" + tap.diag("LOGFILE not found, cannot validate 'OUTPUT' rule") success = 0 elif not util.grep(logfile, key): success = 0 else: - print "PACMAN rule '%s' not found" % case + tap.diag("PACMAN rule '%s' not found" % case) success = -1 elif kind == "PKG": localdb = test.db["local"] @@ -108,7 +109,7 @@ def check(self, test): if not found: success = 0 else: - print "PKG rule '%s' not found" % case + tap.diag("PKG rule '%s' not found" % case) success = -1 elif kind == "FILE": filename = os.path.join(test.root, key) @@ -148,7 +149,7 @@ def check(self, test): if not os.path.isfile("%s.pacsave" % filename): success = 0 else: - print "FILE rule '%s' not found" % case + tap.diag("FILE rule '%s' not found" % case) success = -1 elif kind == "DIR": filename = os.path.join(test.root, key) @@ -156,7 +157,7 @@ def check(self, test): if not os.path.isdir(filename): success = 0 else: - print "DIR rule '%s' not found" % case + tap.diag("DIR rule '%s' not found" % case) success = -1 elif kind == "LINK": filename = os.path.join(test.root, key) @@ -164,7 +165,7 @@ def check(self, test): if not os.path.islink(filename): success = 0 else: - print "LINK rule '%s' not found" % case + tap.diag("LINK rule '%s' not found" % case) success = -1 elif kind == "CACHE": cachedir = os.path.join(test.root, util.PM_CACHEDIR) @@ -174,7 +175,7 @@ def check(self, test): os.path.join(cachedir, pkg.filename())): success = 0 else: - print "Rule kind '%s' not found" % kind + tap.diag("Rule kind '%s' not found" % kind) success = -1 if self.false and success != -1: diff --git a/test/pacman/pmtest.py b/test/pacman/pmtest.py index cea584d..b343d55 100644 --- a/test/pacman/pmtest.py +++ b/test/pacman/pmtest.py @@ -27,6 +27,7 @@ import pmrule import pmdb import pmfile +import tap import util from util import vprint @@ -104,7 +105,7 @@ def load(self): raise IOError("file %s does not exist!" % self.name) def generate(self, pacman): - print "==> Generating test environment" + tap.diag("==> Generating test environment") # Cleanup leftover files from a previous test session if os.path.isdir(self.root): @@ -192,23 +193,23 @@ def generate(self, pacman): def run(self, pacman): if os.path.isfile(util.PM_LOCK): - print "\tERROR: another pacman session is on-going -- skipping" + tap.bail("\tERROR: another pacman session is on-going -- skipping") return - print "==> Running test" + tap.diag("==> Running test") vprint("\tpacman %s" % self.args) cmd = [] if os.geteuid() != 0: fakeroot = util.which("fakeroot") if not fakeroot: - print "WARNING: fakeroot not found!" + tap.diag("WARNING: fakeroot not found!") else: cmd.append("fakeroot") fakechroot = util.which("fakechroot") if not fakechroot: - print "WARNING: fakechroot not found!" + tap.diag("WARNING: fakechroot not found!") else: cmd.append("fakechroot") @@ -252,23 +253,20 @@ def run(self, pacman): # Check if the lock is still there if os.path.isfile(util.PM_LOCK): - print "\tERROR: %s not removed" % util.PM_LOCK + tap.diag("\tERROR: %s not removed" % util.PM_LOCK) os.unlink(util.PM_LOCK) # Look for a core file if os.path.isfile(os.path.join(self.root, util.TMPDIR, "core")): - print "\tERROR: pacman dumped a core file" + tap.diag("\tERROR: pacman dumped a core file") def check(self): - print "==> Checking rules" - + tap.plan(len(self.rules)) for i in self.rules: success = i.check(self) if success == 1: - msg = " OK " self.result["success"] += 1 else: - msg = "FAIL" self.result["fail"] += 1 - print "\t[%s] %s" % (msg, i) + tap.ok(success, i) # vim: set ts=4 sw=4 et: diff --git a/test/pacman/tap.py b/test/pacman/tap.py new file mode 100644 index 0000000..c70a535 --- /dev/null +++ b/test/pacman/tap.py @@ -0,0 +1,64 @@ +# Copyright (c) 2013 Pacman Development Team <pacman-dev@archlinux.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +todo = None +count = 0 +level = 0 +failed = 0 + +def _output(msg): + print("%s%s" % (" "*level, msg)) + +def ok(ok, description=""): + global count, failed + count += 1 + if not ok: + failed += 1 + directive = " # TODO" if todo else "" + _output("%s %d - %s%s" % ("ok" if ok else "not ok", count, + description, directive)) + +def plan(count): + _output("1..%d" % (count)) + +def diag(msg): + _output("# %s" % (msg)) + +def bail(reason=""): + _output("Bail out! %s" % (reason)) + +def subtest(func, description=""): + global todo, count, level, failed + + save_todo = todo + save_count = count + save_level = level + save_failed = failed + + todo = None + count = 0 + level += 1 + failed = 0 + + func() + + subtest_ok = not failed + + todo = save_todo + count = save_count + level = save_level + failed = save_failed + + ok(subtest_ok, description) diff --git a/test/pacman/util.py b/test/pacman/util.py index 65540ed..14035d7 100644 --- a/test/pacman/util.py +++ b/test/pacman/util.py @@ -21,6 +21,8 @@ import re import hashlib +import tap + SELFPATH = os.path.abspath(os.path.dirname(__file__)) # ALPM @@ -43,7 +45,7 @@ def vprint(msg): if verbose: - print msg + tap.diag(msg) # # Methods to generate files -- 1.8.3.4