[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Parallel test execution: new option `parallel-tests': [1/4]
|
From: |
Ralf Wildenhues |
|
Subject: |
Parallel test execution: new option `parallel-tests': [1/4] |
|
Date: |
Sun, 12 Oct 2008 22:34:02 +0200 |
|
User-agent: |
Mutt/1.5.18 (2008-05-17) |
Parallel test execution: new option `parallel-tests'.
2008-07-05 Akim Demaille <address@hidden>
Jim Meyering <address@hidden>
Benoit Sigoure <address@hidden>
Ralf Wildenhues <address@hidden>
* automake.in (handle_tests): Set new conditional PARALLEL_TESTS
when reading check.am. Handle option `parallel-tests' and all
its new semantics. Define macros TEST_SUFFIXES, TEST_SUITE_LOG,
TEST_SUITE_HTML, TEST_LOGS, TEST_LOGS_TMP, suffix rules if
applicable, and per-target rules for other tests. Add all log
files to %clean_files at the `MOSTLY_CLEAN' level.
* lib/Automake/Options.pm (_process_option_list): Accept
`parallel-tests'.
* lib/am/check.am [!PARALLEL_TESTS] (check-TESTS): Move existing
testsuite driver under this new conditional.
[PARALLEL_TESTS]
(am__rst_title, am__rst_section, am__text_box am__sh_e_setup)
(am__check_pre, am__check_post): New internal macros.
($(TEST_SUITE_LOG), check-TESTS, .log.html, check-html): New
rules.
* lib/am/check2.am: New file.
* lib/am/Makefile.am (dist_am_DATA): Add check2.am.
* tests/Makefile.am (AUTOMAKE_OPTIONS): Use `parallel-tests'.
(clean-local): Renamed from distclean-local.
* tests/defs.in: Drop VERBOSE handling, not needed here any more.
diff --git a/automake.in b/automake.in
index 99f6400..b4e0312 100755
--- a/automake.in
+++ b/automake.in
@@ -4738,13 +4738,95 @@ sub handle_tests
{
push (@check_tests, 'check-TESTS');
$output_rules .= &file_contents ('check', new Automake::Location,
- COLOR => !! option 'color-tests');
+ COLOR => !! option 'color-tests',
+ PARALLEL_TESTS => !! option
'parallel-tests');
# Tests that are known programs should have $(EXEEXT) appended.
# For matching purposes, we need to adjust XFAIL_TESTS as well.
append_exeext { exists $known_programs{$_[0]} } 'TESTS';
append_exeext { exists $known_programs{$_[0]} } 'XFAIL_TESTS'
if (var ('XFAIL_TESTS'));
+
+ if (option 'parallel-tests')
+ {
+ define_variable ('TEST_SUITE_LOG', 'test-suite.log', INTERNAL);
+ define_variable ('TEST_SUITE_HTML', '$(TEST_SUITE_LOG:.log=.html)',
INTERNAL);
+ my $suff = '.test';
+ my $at_exeext = '';
+ if (exists $configure_vars{'EXEEXT'})
+ {
+ $at_exeext = subst ('EXEEXT');
+ $suff = $at_exeext . ' ' . $suff;
+ }
+ define_variable ('TEST_SUFFIXES', $suff, INTERNAL);
+ # FIXME: this mishandles conditions.
+ my @test_suffixes = (var 'TEST_SUFFIXES')->value_as_list_recursive;
+ if (exists $configure_vars{'EXEEXT'})
+ {
+ unshift (@test_suffixes, $at_exeext)
+ unless $test_suffixes[0] eq $at_exeext;
+ }
+ unshift (@test_suffixes, '');
+
+ transform_variable_recursively
+ ('TESTS', 'TEST_LOGS', 'am__testlogs', 1, INTERNAL,
+ sub {
+ my ($subvar, $val, $cond, $full_cond) = @_;
+ my $obj = $val;
+ return $obj
+ if $val =~ /address@hidden@$/;
+ $obj =~ s/\$\(EXEEXT\)$//o;
+ foreach my $test_suffix (@test_suffixes)
+ {
+ next
+ if $test_suffix eq $at_exeext || $test_suffix eq '';
+ return substr ($obj, 0, length ($obj) - length
($test_suffix)) . '.log'
+ if substr ($obj, - length ($test_suffix)) eq $test_suffix;
+ }
+ $obj .= '.log';
+ $output_rules .= file_contents ('check2', new
Automake::Location,
+ GENERIC => 0,
+ OBJ => $obj,
+ SOURCE => $val,
+ EXT => '');
+ return $obj;
+ });
+
+ my $nhelper=1;
+ my $prev = 'TESTS';
+ my $post = '';
+ my $last_suffix = $test_suffixes[$#test_suffixes];
+ my $cur = '';
+ foreach my $test_suffix (@test_suffixes)
+ {
+ if ($test_suffix eq $last_suffix)
+ {
+ $cur = 'TEST_LOGS';
+ }
+ else
+ {
+ $cur = 'am__test_logs' . $nhelper;
+ }
+ define_variable ($cur,
+ '$(' . $prev . ':' . $test_suffix . $post . '=.log)', INTERNAL);
+ $post = '.log';
+ $prev = $cur;
+ $nhelper++;
+ $output_rules .= file_contents ('check2', new Automake::Location,
+ GENERIC => 1,
+ OBJ => '',
+ SOURCE => '$<',
+ EXT => $test_suffix)
+ if $test_suffix ne $at_exeext && $test_suffix ne '';
+ }
+
+ define_variable ('TEST_LOGS_TMP', '$(TEST_LOGS:.log=.log-t)',
INTERNAL);
+
+ $clean_files{'$(TEST_LOGS_TMP)'} = MOSTLY_CLEAN;
+ $clean_files{'$(TEST_LOGS)'} = MOSTLY_CLEAN;
+ $clean_files{'$(TEST_SUITE_LOG)'} = MOSTLY_CLEAN;
+ $clean_files{'$(TEST_SUITE_HTML)'} = MOSTLY_CLEAN;
+ }
}
}
diff --git a/lib/Automake/Options.pm b/lib/Automake/Options.pm
index 1705981..df08e05 100644
--- a/lib/Automake/Options.pm
+++ b/lib/Automake/Options.pm
@@ -266,7 +266,7 @@ sub _process_option_list (\%$@)
|| $_ eq 'subdir-objects' || $_ eq 'nostdinc'
|| $_ eq 'no-exeext' || $_ eq 'no-define'
|| $_ eq 'std-options'
- || $_ eq 'color-tests'
+ || $_ eq 'color-tests' || $_ eq 'parallel-tests'
|| $_ eq 'cygnus' || $_ eq 'no-dependencies')
{
# Explicitly recognize these.
diff --git a/lib/am/Makefile.am b/lib/am/Makefile.am
index f6fdafa..1ab2f31 100644
--- a/lib/am/Makefile.am
+++ b/lib/am/Makefile.am
@@ -2,7 +2,7 @@
## Makefile for Automake lib/am.
-## Copyright (C) 1995, 1996, 1997, 1998, 1999, 2001, 2003, 2004
+## Copyright (C) 1995, 1996, 1997, 1998, 1999, 2001, 2003, 2004, 2008
## Free Software Foundation, Inc.
## This program is free software; you can redistribute it and/or modify
@@ -23,6 +23,7 @@ amdir = $(pkgvdatadir)/am
dist_am_DATA = \
ansi2knr.am \
check.am \
+check2.am \
clean-hdr.am \
clean.am \
compile.am \
diff --git a/lib/am/check.am b/lib/am/check.am
index ca400fe..129f2d1 100644
--- a/lib/am/check.am
+++ b/lib/am/check.am
@@ -38,6 +38,219 @@ endif !%?COLOR%
.PHONY: check-TESTS
+if %?PARALLEL_TESTS%
+
+include inst-vars.am
+
+## New parallel test driver.
+##
+## This provides special support for "unit tests", that is to
+## say, tests that (once run) no longer need to be re-compiled and
+## re-run at each "make check", unless their sources changed. To
+## enable unit-test supports, define LAZY_TEST_SUITE. In such a
+## setting, that heavily relies on correct dependencies, its users may
+## prefer to define EXTRA_PROGRAMS instead of check_PROGRAMS, because
+## it allows intertwined compilation and execution of the tests.
+## Sometimes this helps catching errors earlier (you don't have to
+## wait for all the tests to be compiled).
+##
+## Define TEST_SUITE_LOG to be the name of the global log to create.
+## Define TEST_LOGS to the set of logs to include in it. It defaults
+## to $(TESTS), with `.test' and address@hidden@' removed, and `'.log'
+## appended.
+##
+## In addition to the magic "exit 77 means SKIP" feature (which was
+## imported from automake), there is a magic "exit 99 means FAIL" feature
+## which is useful if you need to issue a hard error no matter whether the
+## test is XFAIL or not. You can disable this feature by setting the
+## variable DISABLE_HARD_ERRORS to `yes'.
+
+# Restructured Text title and section.
+am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
+am__rst_section = sed 'p;s/./=/g;p;g'
+
+# Put stdin (possibly several lines separated by ". ") in a box.
+am__text_box = sed 's/\. /\n/g' | sed '/^$$/d' | \
+$(AWK) '{ if (final) final = final "\n" $$0; else final = $$0; }\
+max < length($$0) { max = length($$0); } \
+END { \
+ for (i = 0; i < max; ++i) line = line "="; \
+ print line; print final; print line; \
+}'
+
+# Solaris 10 'make', and several other traditional 'make' implementations,
+# pass "-e" to $(SHELL). This contradicts POSIX. Work around the problem
+# by disabling -e (using the XSI extension "set +e") if it's set.
+am__sh_e_setup = case $$- in *e*) set +e;; esac
+
+# To be inserted before the command running the test. Creates the
+# directory for the log if needed. Stores in $dir the directory
+# containing $f, in $tst the test, in $log the log, and passes
+# TESTS_ENVIRONMENT. Save and restore TERM around use of
+# TESTS_ENVIRONMENT, in case that unsets it.
+am__check_pre = \
+$(am__sh_e_setup); \
+$(am__vpath_adj_setup) $(am__vpath_adj) \
+srcdir=$(srcdir); export srcdir; \
+rm -f address@hidden; \
+trap 'st=$$?; rm -f '\''$(abs_builddir)/address@hidden'\''; (exit $$st); exit
$$st' \
+ 1 2 13 15; \
+am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`; \
+test "x$$am__odir" = x. || $(MKDIR_P) "$$am__odir" || exit $$?; \
+if test -f "./$$f"; then dir=./; \
+elif test -f "$$f"; then dir=; \
+else dir="$(srcdir)/"; fi; \
+tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \
+$(TESTS_ENVIRONMENT)
+
+# To be appended to the command running the test. Handle the stdout
+# and stderr redirection, and catch the exit status.
+am__check_post = \
+>address@hidden 2>&1; \
+estatus=$$?; \
+TERM=$$__SAVED_TERM; export TERM; \
+$(am__tty_colors); \
+xfailed=PASS; \
+for xfail in : $(XFAIL_TESTS); do \
+ case $$f in \
+ $$xfail | $(srcdir)/$$xfail) xfailed=XFAIL; break; \
+ esac; \
+done; \
+case $$estatus:$$xfailed in \
+ 0:XFAIL) col=$$red; res=XPASS;; \
+ 0:*) col=$$grn; res=PASS ;; \
+ 77:*) col=$$blu; res=SKIP ;; \
+ 99:*) col=$$red; res=FAIL ;; \
+ *:XFAIL) col=$$lgn; res=XFAIL;; \
+ *:*) col=$$red; res=FAIL ;; \
+esac; \
+echo "$${col}$$res$${std}: $$f"; \
+echo "$$res: $$f (exit: $$estatus)" | \
+ $(am__rst_section) >$@; \
+cat address@hidden >>$@; \
+rm -f address@hidden; \
+if test -z '$(DISABLE_HARD_ERRORS)' \
+ && test $$estatus -eq 99; then \
+ exit $$estatus; else :; \
+fi
+
+$(TEST_SUITE_LOG): $(TEST_LOGS)
+ @$(am__sh_e_setup); \
+ list='$(TEST_LOGS)'; \
+ results=`for f in $$list; do \
+ read line < $$f && echo "$$line" || echo FAIL; \
+ done`; \
+ all=`echo "$$results" | wc -l | sed -e 's/^[ ]*//'`; \
+ fail=`echo "$$results" | grep -c '^FAIL'`; \
+ pass=`echo "$$results" | grep -c '^PASS'`; \
+ skip=`echo "$$results" | grep -c '^SKIP'`; \
+ xfail=`echo "$$results" | grep -c '^XFAIL'`; \
+ xpass=`echo "$$results" | grep -c '^XPASS'`; \
+ failures=`expr $$fail + $$xpass`; \
+ all=`expr $$all - $$skip`; \
+ if test "$$all" -eq 1; then tests=test; All=; \
+ else tests=tests; All="All "; fi; \
+ case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
+ fail=0:xpass=0:xfail=0) \
+ msg="$$All$$all $$tests passed. "; \
+ exit=true;; \
+ fail=0:xpass=0:xfail=*) \
+ msg="$$All$$all $$tests behaved as expected"; \
+ if test "$$xfail" -eq 1; then xfailures=failure; \
+ else xfailures=failures; fi; \
+ msg="$$msg ($$xfail expected $$xfailures). "; \
+ exit=true;; \
+ fail=*:xpass=0:xfail=*) \
+ msg="$$fail of $$all $$tests failed. "; \
+ exit=false;; \
+ fail=*:xpass=*:xfail=*) \
+ msg="$$failures of $$all $$tests did not behave as expected"; \
+ if test "$$xpass" -eq 1; then xpasses=pass; \
+ else xpasses=passes; fi; \
+ msg="$$msg ($$xpass unexpected $$xpasses). "; \
+ exit=false;; \
+ *) \
+ echo >&2 "incorrect case"; exit 4;; \
+ esac; \
+ if test "$$skip" -ne 0; then \
+ if test "$$skip" -eq 1; then \
+ msg="$$msg($$skip test was not run). "; \
+ else \
+ msg="$$msg($$skip tests were not run). "; \
+ fi; \
+ fi; \
+ if test "$$failures" -ne 0; then \
+ { \
+ echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
+ $(am__rst_title); \
+ echo "$$msg"; \
+ echo; \
+ echo ".. contents:: :depth: 2"; \
+ echo; \
+ for f in $$list; do \
+ read line < $$f; \
+ case $$line in \
+ SKIP:*|PASS:*|XFAIL:*);; \
+ *) echo; cat $$f;; \
+ esac; \
+ done; \
+ } >$(TEST_SUITE_LOG).tmp; \
+ mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
+ msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
+ fi; \
+ fi; \
+ $(am__tty_colors); \
+ if $$exit; then \
+ echo $(ECHO_N) "$$grn$(ECHO_C)"; \
+ else \
+ echo $(ECHO_N) "$$red$(ECHO_C)"; \
+ fi; \
+ echo "$$msg" | $(am__text_box); \
+ echo $(ECHO_N) "$$std$(ECHO_C)"; \
+ test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
+ $$exit
+
+# Run all the tests.
+check-TESTS:
+ @if test -z '$(LAZY_TEST_SUITE)' \
+ && test -n "$(TEST_SUITE_LOG)$(TEST_LOGS)"; then \
+ rm -f $(TEST_SUITE_LOG) $(TEST_LOGS); \
+ fi
+ @$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG)
+
+
+## -------------- ##
+## Produce HTML. ##
+## -------------- ##
+
+.log.html:
+ @list='$(RST2HTML) $$RST2HTML rst2html rst2html.py'; \
+ for r2h in $$list; do \
+ if ($$r2h --version) >/dev/null 2>&1; then \
+ R2H=$$r2h; \
+ fi; \
+ done; \
+ if test -z "$$R2H"; then \
+ echo >&2 "cannot find rst2html, cannot create $@"; \
+ exit 2; \
+ fi; \
+ $$R2H $< >address@hidden
+ @mv address@hidden $@
+
+# Be sure to run check-TESTS first, and then to convert the result.
+# Beware of concurrent executions. And expect check-TESTS to fail.
+check-html:
+ @if $(MAKE) $(AM_MAKEFLAGS) check-TESTS; then :; else \
+ rv=$$?; \
+ $(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML); \
+ exit $$rv; \
+ fi
+.PHONY: check-html
+
+else !%?PARALLEL_TESTS%
+
check-TESTS: $(TESTS)
@failed=0; all=0; xfail=0; xpass=0; skip=0; \
srcdir=$(srcdir); export srcdir; \
@@ -138,3 +351,5 @@ check-TESTS: $(TESTS)
echo "$$dashes$$std"; \
test "$$failed" -eq 0; \
else :; fi
+
+endif !%?PARALLEL_TESTS%
diff --git a/lib/am/check2.am b/lib/am/check2.am
new file mode 100644
index 0000000..58ca9e6
--- /dev/null
+++ b/lib/am/check2.am
@@ -0,0 +1,20 @@
+## automake - create Makefile.in from Makefile.am
+## Copyright (C) 2008 Free Software Foundation, Inc.
+
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3, or (at your option)
+## any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+## From a test file to a log file.
+?GENERIC?%EXT%.log:
+?!GENERIC?%OBJ%: %SOURCE%
+ @p='%SOURCE%'; $(am__check_pre) "$$tst" $(am__check_post)
diff --git a/tests/.gitignore b/tests/.gitignore
index f1c3bff..a6827fd 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -2,3 +2,5 @@ aclocal-*
automake-*
defs
*.dir
+*.log
+*.log-t
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 32cefa5..9cc6dbe 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1,4 +1,5 @@
## Process this file with automake to create Makefile.in
+AUTOMAKE_OPTIONS = parallel-tests
XFAIL_TESTS = \
all.test \
@@ -670,7 +671,7 @@ EXTRA_DIST = ChangeLog-old $(TESTS)
# Each test case depends on defs, aclocal, and automake.
check_SCRIPTS = defs aclocal-$(APIVERSION) automake-$(APIVERSION)
-distclean-local: check-clean-local
+clean-local: check-clean-local
check-clean-local:
-chmod -R u+rwx *.dir
diff --git a/tests/defs.in b/tests/defs.in
index fb4e0e0..063ce58 100644
--- a/tests/defs.in
+++ b/tests/defs.in
@@ -40,9 +40,7 @@ test -f ./defs || {
exit 1
}
-# If srcdir is not set, then we are not running from `make check', be verbose.
if test -z "$srcdir"; then
- VERBOSE=x
# compute $srcdir.
srcdir=`echo "$0" | sed -e 's,/[^\\/]*$,,'`
test "$srcdir" = $0 && srcdir=.
@@ -56,12 +54,6 @@ test -f "$srcdir/defs.in" || {
me=`echo "$0" | sed -e 's,.*[\\/],,;s/\.test$//'`
-# See how redirections should work. User can set VERBOSE to see all
-# output.
-test -z "$VERBOSE" && {
- exec > /dev/null 2>&1
-}
-
# Make sure we override the user shell.
SHELL='@SHELL@'
export SHELL
@@ -402,10 +394,7 @@ AUTOMAKE_fails ()
AUTOMAKE_run 1 ${1+"$@"}
}
-# Turn on shell traces when VERBOSE is set.
-if test -n "$VERBOSE"; then
- set -x
-else
- :
-fi
+# Turn on shell traces.
+set -x
+
pwd