This change causes expected fail tests to actually fail by eliding the '# TODO' from the test plan. In turn, we can now properly use 'should_fail' in the meson test() rule and see these expected fail tests in the output: Before: ... 320/332 upgrade077.py OK 0.12679290771484375 s 321/332 upgrade078.py OK 0.12620115280151367 s 322/332 upgrade080.py OK 0.1252129077911377 s ... Ok: 332 Expected Fail: 0 Fail: 0 Unexpected Pass: 0 Skipped: 0 Timeout: 0 After: ... 320/332 upgrade077.py OK 0.12679290771484375 s 321/332 upgrade078.py EXPECTEDFAIL0.12620115280151367 s 322/332 upgrade080.py OK 0.1252129077911377 s ... Ok: 326 Expected Fail: 6 Fail: 0 Unexpected Pass: 0 Skipped: 0 Timeout: 0 --- test/pacman/meson.build | 15 ++++++++++++++- test/pacman/pmenv.py | 7 ++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/test/pacman/meson.build b/test/pacman/meson.build index 9de4f5a1..fd0dbf90 100644 --- a/test/pacman/meson.build +++ b/test/pacman/meson.build @@ -330,6 +330,17 @@ pacman_tests = [ 'tests/xfercommand001.py', ] +xfail_tests = { + 'tests/deprange001.py': true, + # expect failure on 32 bit machines + 'tests/query006.py': cc.sizeof('ssize_t') < 8, + 'tests/replace110.py': true, + 'tests/sync-update-package-removing-required-provides.py': true, + 'tests/sync403.py': true, + 'tests/sync406.py': true, + 'tests/upgrade078.py': true, +} + foreach input : pacman_tests test_name = input.split('/')[1] args = [ @@ -351,7 +362,9 @@ foreach input : pacman_tests test_name, PYTHON, protocol : 'tap', + env : ['RUNNING_UNDER_MESON=1'], args : args, - timeout: 120, + timeout : 120, + should_fail : xfail_tests.get(input, false), depends : [pacman_bin]) endforeach diff --git a/test/pacman/pmenv.py b/test/pacman/pmenv.py index 24437b61..d87193f2 100644 --- a/test/pacman/pmenv.py +++ b/test/pacman/pmenv.py @@ -70,5 +70,10 @@ def run(self): t.run(self.pacman) tap.diag("==> Checking rules") - tap.todo = t.expectfailure + # When running under meson, we don't emit 'todo' in the plan and instead + # handle expected failures in the test() objects. This really should be + # fixed in meson: + # https://github.com/mesonbuild/meson/issues/2923#issuecomment-614647076 + tap.todo = (t.expectfailure and + not 'RUNNING_UNDER_MESON' in os.environ) tap.subtest(lambda: t.check(), t.description) -- 2.26.2