Re: [PATCH 020/143] meson: add testsuite Makefile generator

[Alex Bennée]

Paolo Bonzini <pbonzini@redhat.com> writes:

> Rules to execute tests are generated by a simple Python program
> that integrates into the existing "make check" mechanism.  This
> provides familiarity for developers, and also allows piecewise
> conversion of the testsuite Makefiles to meson.

Hmm not sure why check-tcg has broken then:

  11:44:37 [alex.bennee@hackbox2:~/l/q/b/all] review/meson-for-5.2|✔ + make 
check-tcg
  make[1]: Entering directory '/home/alex.bennee/lsrc/qemu.git/slirp'
  make[1]: Nothing to be done for 'all'.
  make[1]: Leaving directory '/home/alex.bennee/lsrc/qemu.git/slirp'
  make: *** No rule to make target 'run-tcg-tests-aarch64-softmmu', needed by 
'check-tcg'.  Stop.


>
> The generated rules are based on QEMU's existing test harness
> Makefile and TAP parser.
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> ---
>  Makefile               |   4 ++
>  scripts/mtest2make.py  | 102 
> +++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/Makefile.include |   1 -
>  3 files changed, 106 insertions(+), 1 deletion(-)
>  create mode 100644 scripts/mtest2make.py
>
> diff --git a/Makefile b/Makefile
> index 9a75047..6248fd0 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -58,6 +58,10 @@ ninjatool: ninjatool.stamp
>  ninjatool.stamp: $(SRC_PATH)/scripts/ninjatool.py config-host.mak
>       $(MESON) setup --reconfigure . $(SRC_PATH) && touch $@
>  
> +Makefile.mtest: build.ninja scripts/mtest2make.py
> +     $(MESON) introspect --tests | $(PYTHON) scripts/mtest2make.py > $@
> +-include Makefile.mtest
> +
>  .git-submodule-status: git-submodule-update config-host.mak
>  
>  # Check that we're not trying to do an out-of-tree build from
> diff --git a/scripts/mtest2make.py b/scripts/mtest2make.py
> new file mode 100644
> index 0000000..e978303
> --- /dev/null
> +++ b/scripts/mtest2make.py
> @@ -0,0 +1,102 @@
> +#! /usr/bin/env python3
> +
> +# Create Makefile targets to run tests, from Meson's test introspection data.
> +#
> +# Author: Paolo Bonzini <pbonzini@redhat.com>
> +
> +from collections import defaultdict
> +import json
> +import os
> +import shlex
> +import sys
> +
> +class Suite(object):
> +    def __init__(self):
> +        self.tests = list()
> +        self.slow_tests = list()
> +        self.executables = set()
> +
> +print('''
> +SPEED = quick
> +
> +# $1 = test command, $2 = test name
> +.test-human-tap = $1 < /dev/null | ./scripts/tap-driver.pl --test-name="$2" 
> $(if $(V),, --show-failures-only)
> +.test-human-exitcode = $1 < /dev/null
> +.test-tap-tap = $1 < /dev/null | sed "s/^[a-z][a-z]* [0-9]*/& $2/" || true
> +.test-tap-exitcode = printf "%s\\n" 1..1 "`$1 < /dev/null > /dev/null || 
> echo "not "`ok 1 $2"
> +.test.print = echo $(if $(V),'$1','Running test $2') >&3
> +.test.env = MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$(( $${RANDOM:-0} % 255 + 
> 1))}
> +
> +# $1 = test name, $2 = test target (human or tap)
> +.test.run = $(call .test.print,$(.test.cmd.$1),$(.test.name.$1)) && $(call 
> .test-$2-$(.test.driver.$1),$(.test.cmd.$1),$(.test.name.$1))
> +
> +define .test.human_k
> +        @exec 3>&1; rc=0; $(foreach TEST, $1, $(call 
> .test.run,$(TEST),human) || rc=$$?;) \\
> +              exit $$rc
> +endef
> +define .test.human_no_k
> +        $(foreach TEST, $1, @exec 3>&1; $(call .test.run,$(TEST),human)
> +)
> +endef
> +.test.human = \\
> +        $(if $(findstring k, $(MAKEFLAGS)), $(.test.human_k), 
> $(.test.human_no_k))
> +
> +define .test.tap
> +        @exec 3>&1; { $(foreach TEST, $1, $(call .test.run,$(TEST),tap); ) } 
> \\
> +              | ./scripts/tap-merge.pl | tee "$@" \\
> +              | ./scripts/tap-driver.pl $(if $(V),, --show-failures-only)
> +endef
> +''')
> +
> +suites = defaultdict(Suite)
> +i = 0
> +for test in json.load(sys.stdin):
> +    env = ' '.join(('%s=%s' % (shlex.quote(k), shlex.quote(v))
> +                    for k, v in test['env'].items()))
> +    executable = os.path.relpath(test['cmd'][0])
> +    if test['workdir'] is not None:
> +        test['cmd'][0] = os.path.relpath(test['cmd'][0], test['workdir'])
> +    else:
> +        test['cmd'][0] = executable
> +    cmd = '$(.test.env) %s %s' % (env, ' '.join((shlex.quote(x) for x in 
> test['cmd'])))
> +    if test['workdir'] is not None:
> +        cmd = '(cd %s && %s)' % (shlex.quote(test['workdir']), cmd)
> +    driver = test['protocol'] if 'protocol' in test else 'exitcode'
> +
> +    i += 1
> +    print('.test.name.%d := %s' % (i, test['name']))
> +    print('.test.driver.%d := %s' % (i, driver))
> +    print('.test.cmd.%d := %s' % (i, cmd))
> +
> +    test_suites = test['suite'] or ['default']
> +    is_slow = any(s.endswith('-slow') for s in test_suites)
> +    for s in test_suites:
> +        # The suite name in the introspection info is "PROJECT:SUITE"
> +        s = s.split(':')[1]
> +        if s.endswith('-slow'):
> +            s = s[:-5]
> +        if is_slow:
> +            suites[s].slow_tests.append(i)
> +        else:
> +            suites[s].tests.append(i)
> +        suites[s].executables.add(executable)
> +
> +print('.PHONY: check check-report.tap')
> +print('check:')
> +print('check-report.tap:')
> +print('\t@cat $^ | scripts/tap-merge.pl >$@')
> +for name, suite in suites.items():
> +    executables = ' '.join(suite.executables)
> +    slow_test_numbers = ' '.join((str(x) for x in suite.slow_tests))
> +    test_numbers = ' '.join((str(x) for x in suite.tests))
> +    print('.test.suite-quick.%s := %s' % (name, test_numbers))
> +    print('.test.suite-slow.%s := $(.test.suite-quick.%s) %s' % (name, name, 
> slow_test_numbers))
> +    print('check-build: %s' % executables)
> +    print('.PHONY: check-%s' % name)
> +    print('.PHONY: check-report-%s.tap' % name)
> +    print('check: check-%s' % name)
> +    print('check-%s: %s' % (name, executables))
> +    print('\t$(call .test.human, $(.test.suite-$(SPEED).%s))' % (name, ))
> +    print('check-report.tap: check-report-%s.tap' % name)
> +    print('check-report-%s.tap: %s' % (name, executables))
> +    print('\t$(call .test.tap, $(.test.suite-$(SPEED).%s))' % (name, ))
> diff --git a/tests/Makefile.include b/tests/Makefile.include
> index 985cd14..5e9dff9 100644
> --- a/tests/Makefile.include
> +++ b/tests/Makefile.include
> @@ -674,7 +674,6 @@ check-report-unit.tap: $(check-unit-y)
>  # Reports and overall runs
>  
>  check-report.tap: $(patsubst %,check-report-qtest-%.tap, $(QTEST_TARGETS)) 
> check-report-unit.tap
> -     $(call quiet-command, cat $^ | scripts/tap-merge.pl >$@,"GEN","$@")
>  
>  # FPU Emulation tests (aka softfloat)
>  #


-- 
Alex Bennée