On Wed, Jul 23, 2008 at 3:18 AM, Xavier <shiningxc@gmail.com> wrote:
On Wed, Jul 23, 2008 at 3:01 AM, Dan McGee <dpmcgee@gmail.com> wrote:
Here is a bisect script I have used with pacman in the past to do it automatically, I can't remember exactly which bug we were trying to track down but the ideas here are solid. You need to build, setup your test environment (if you are using pactest we really don't have to do much), run the test, and issue the correct exit code.
So what about directly issuing the correct exit code in the pactest python script? Sounds like it would be a nice and helpful behavior, and it should be rather easy to do.
It makes 'make check' (and with that, 'make distcheck', etc.) fail if the pactest script returns anything but zero. I've had this change in a local branch for ages, but we need something else- a way to indicate known failure in pactest so we can (at least temporarily) continue on when our 5 or 6 known failures happen. -Dan c0f892be819a72370778c7b8a8604a268f43dd30 pactest/pactest.py | 9 ++++++- pactest/pmenv.py | 53 +++++++++++++++++++++++---------------------------- pactest/pmtest.py | 6 ++-- pactest/util.py | 5 +--- 4 files changed, 35 insertions(+), 38 deletions(-) diff --git a/pactest/pactest.py b/pactest/pactest.py index 7cda255..b812c64 100755 --- a/pactest/pactest.py +++ b/pactest/pactest.py @@ -93,13 +93,18 @@ if __name__ == "__main__": env.pacman["valgrind"] = opts.valgrind env.pacman["manual-confirm"] = opts.manualconfirm - if len(opts.testcases) == 0: + if opts.testcases is None or len(opts.testcases) == 0: print "no tests defined, nothing to do" + sys.exit(2) else: - for i in opts.testcases: env.addtest(i) + for i in opts.testcases: + env.addtest(i) # run tests and print overall results env.run() env.results() + if len(env.failed) > 0: + sys.exit(1) + # vim: set ts=4 sw=4 et: diff --git a/pactest/pmenv.py b/pactest/pmenv.py old mode 100755 new mode 100644 index f2327f9..5960c0a --- a/pactest/pmenv.py +++ b/pactest/pmenv.py @@ -27,16 +27,19 @@ class pmenv: """Environment object """ + pacman = { + "bin": "pacman", + "debug": 0, + "gdb": 0, + "valgrind": 0, + "nolog": 0 + } + testcases = [] + passed = [] + failed = [] + def __init__(self, root = "root"): self.root = os.path.abspath(root) - self.pacman = { - "bin": "pacman", - "debug": 0, - "gdb": 0, - "valgrind": 0, - "nolog": 0 - } - self.testcases = [] def __str__(self): return "root = %s\n" \ @@ -47,8 +50,8 @@ class pmenv: """ """ if not os.path.isfile(testcase): - err("file %s not found" % testcase) - return + raise IOError("test file %s not found" % testcase) + test = pmtest.pmtest(testcase, self.root) self.testcases.append(test) @@ -76,24 +79,16 @@ class pmenv: t.check() print "==> Test result" if t.result["fail"] == 0: + self.passed.append(t) print "\tPASS" else: + self.failed.append(t) print "\tFAIL" print def results(self): """ """ - passed = 0 - tpassed = [] - tfailed = [] - for test in self.testcases: - fail = test.result["fail"] - if fail == 0: - passed += 1 - tpassed.append(test) - else: - tfailed.append(test) def _printtest(t): success = test.result["success"] @@ -114,20 +109,20 @@ class pmenv: print "=========="*8 print "Results" print "----------"*8 - for test in tpassed: _printtest(test) + for test in self.passed: _printtest(test) print "----------"*8 - for test in tfailed: _printtest(test) + for test in self.failed: _printtest(test) print "----------"*8 - total = len(self.testcases) - failed = total - passed + passes = len(self.passed) + fails = len(self.failed) + total = passes + fails print "TOTAL = %3u" % total - if total: - print "PASS = %3u (%6.2f%%)" % (passed, float(passed) * 100 / total) - print "FAIL = %3u (%6.2f%%)" % (failed, float(failed) * 100 / total) + print "PASS = %3u (%6.2f%%)" % (passes, float(passes) * 100 / total) + print "FAIL = %3u (%6.2f%%)" % (fails, float(fails) * 100 / total) print "" if __name__ == "__main__": - env = pmenv("/tmp") - print env + pass + # vim: set ts=4 sw=4 et: diff --git a/pactest/pmtest.py b/pactest/pmtest.py old mode 100755 new mode 100644 index e8f6fa8..a34efda --- a/pactest/pmtest.py +++ b/pactest/pmtest.py @@ -94,7 +94,7 @@ class pmtest: if os.path.isfile(self.name): execfile(self.name) else: - err("file %s does not exist!" % self.name) + raise IOError("file %s does not exist!" % self.name) def generate(self): """ @@ -267,6 +267,6 @@ class pmtest: if __name__ == "__main__": - test = pmtest("test1", "./root") - print test + pass + # vim: set ts=4 sw=4 et: diff --git a/pactest/util.py b/pactest/util.py old mode 100755 new mode 100644 index c6d5a59..db9560f --- a/pactest/util.py +++ b/pactest/util.py @@ -43,10 +43,6 @@ LOGFILE = "var/log/pactest.log" verbose = 0 -def err(msg): - print "error: " + msg - sys.exit(1) - def vprint(msg): if verbose: print msg @@ -273,4 +269,5 @@ def mkdir(dir): if __name__ == "__main__": pass + # vim: set ts=4 sw=4 et: