This will enable us to mark tests we know currently fail to differentiate them from those that we know should pass. Regressions should be easier to spot this way. Signed-off-by: Dan McGee <dan@archlinux.org> --- pactest/pmenv.py | 31 ++++++++++++++++++++++++++----- pactest/pmtest.py | 1 + pactest/tests/fileconflict001.py | 2 ++ pactest/tests/fileconflict002.py | 2 ++ pactest/tests/fileconflict004.py | 2 ++ pactest/tests/sync403.py | 2 ++ pactest/tests/trans001.py | 2 ++ pactest/tests/upgrade046.py | 2 ++ pactest/tests/upgrade051.py | 2 ++ 9 files changed, 41 insertions(+), 5 deletions(-) diff --git a/pactest/pmenv.py b/pactest/pmenv.py index f2327f9..b86abe9 100755 --- a/pactest/pmenv.py +++ b/pactest/pmenv.py @@ -86,13 +86,25 @@ class pmenv: """ passed = 0 tpassed = [] + failed = 0 tfailed = [] + expectedfail = 0 + texpectedfail = [] + unexpectedpass = 0 + tunexpectedpass = [] for test in self.testcases: fail = test.result["fail"] - if fail == 0: + if fail == 0 and not test.expectfailure: passed += 1 tpassed.append(test) + elif fail != 0 and test.expectfailure: + expectedfail += 1 + texpectedfail.append(test) + elif fail == 0: # and not test.expectfail + unexpectedpass += 1 + tunexpectedpass.append(test) else: + failed += 1 tfailed.append(test) def _printtest(t): @@ -114,17 +126,26 @@ class pmenv: print "=========="*8 print "Results" print "----------"*8 + print " Passed:" for test in tpassed: _printtest(test) print "----------"*8 + print " Expected Failures:" + for test in texpectedfail: _printtest(test) + print "----------"*8 + print " Unexpected Passes:" + for test in tunexpectedpass: _printtest(test) + print "----------"*8 + print " Failed:" for test in tfailed: _printtest(test) print "----------"*8 total = len(self.testcases) - failed = total - passed - print "TOTAL = %3u" % total + print "Total = %3u" % total if total: - print "PASS = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total) - print "FAIL = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total) + print "Pass = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total) + print "Expected Fail = %3u (%6.2f%%)" % (expectedfail, float(expectedfail) * 100 / total) + print "Unexpected Pass = %3u (%6.2f%%)" % (unexpectedpass, float(unexpectedpass) * 100 / total) + print "Fail = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total) print "" if __name__ == "__main__": diff --git a/pactest/pmtest.py b/pactest/pmtest.py index e8f6fa8..f31563b 100755 --- a/pactest/pmtest.py +++ b/pactest/pmtest.py @@ -90,6 +90,7 @@ class pmtest: # Test rules self.rules = [] self.files = [] + self.expectfailure = False if os.path.isfile(self.name): execfile(self.name) diff --git a/pactest/tests/fileconflict001.py b/pactest/tests/fileconflict001.py index 4c2069e..8c13911 100644 --- a/pactest/tests/fileconflict001.py +++ b/pactest/tests/fileconflict001.py @@ -18,3 +18,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2]) self.addrule("PACMAN_RETCODE=1") self.addrule("!PKG_EXIST=pkg1") self.addrule("!PKG_EXIST=pkg2") + +self.expectfailure = True diff --git a/pactest/tests/fileconflict002.py b/pactest/tests/fileconflict002.py index c54f6da..f70873b 100644 --- a/pactest/tests/fileconflict002.py +++ b/pactest/tests/fileconflict002.py @@ -14,3 +14,5 @@ self.args = "-U %s" % " ".join([p.filename() for p in p1, p2]) self.addrule("PACMAN_RETCODE=1") self.addrule("!PKG_EXIST=pkg1") self.addrule("!PKG_EXIST=pkg2") + +self.expectfailure = True diff --git a/pactest/tests/fileconflict004.py b/pactest/tests/fileconflict004.py index a5347cc..2396ced 100644 --- a/pactest/tests/fileconflict004.py +++ b/pactest/tests/fileconflict004.py @@ -17,3 +17,5 @@ self.addrule("PACMAN_RETCODE=0") self.addrule("PKG_EXIST=pkg1") self.addrule("PKG_VERSION=pkg1|2.0-1") self.addrule("FILE_TYPE=test|link") + +self.expectfailure = True diff --git a/pactest/tests/sync403.py b/pactest/tests/sync403.py index b8d0101..d8ab763 100644 --- a/pactest/tests/sync403.py +++ b/pactest/tests/sync403.py @@ -19,3 +19,5 @@ self.addrule("PACMAN_RETCODE=0") self.addrule("PKG_EXIST=pkg1") self.addrule("!PKG_EXIST=pkg2") self.addrule("PKG_EXIST=pkg3") + +self.expectfailure = True diff --git a/pactest/tests/trans001.py b/pactest/tests/trans001.py index b9889b1..b3d7883 100644 --- a/pactest/tests/trans001.py +++ b/pactest/tests/trans001.py @@ -18,3 +18,5 @@ self.addrule("PACMAN_RETCODE=1") self.addrule("!PKG_EXIST=pkg1") self.addrule("PKG_EXIST=pkg2") self.addrule("PKG_EXIST=pkg3") + +self.expectfailure = True diff --git a/pactest/tests/upgrade046.py b/pactest/tests/upgrade046.py index 60164b7..1239064 100644 --- a/pactest/tests/upgrade046.py +++ b/pactest/tests/upgrade046.py @@ -29,3 +29,5 @@ self.addrule("FILE_MODIFIED=bin/dummy") self.addrule("FILE_MODIFIED=bin/foobar") self.addrule("FILE_EXIST=usr/share/file") self.addrule("FILE_MODIFIED=usr/share/file") + +self.expectfailure = True diff --git a/pactest/tests/upgrade051.py b/pactest/tests/upgrade051.py index e8c69eb..4346c24 100644 --- a/pactest/tests/upgrade051.py +++ b/pactest/tests/upgrade051.py @@ -13,3 +13,5 @@ self.args = "-U %s" % p.filename() self.addrule("PACMAN_RETCODE=0") self.addrule("!PKG_EXIST=pkg1") self.addrule("PKG_EXIST=pkg2") + +self.expectfailure = True -- 1.5.6.4