parallel-tests: Ensure backward-compatible semantics. [2/4]

[Ralf Wildenhues]

    parallel-tests: Ensure backward-compatible semantics.
    
    For each test in Automake's test suite that uses TESTS, generate
    an identical one that uses the `parallel-tests' option, for
    coverage of backward-compatible functionality.
    
    * tests/gen-parallel-tests: New file, generates distributed
    Makefile.am snippet tests/parallel-tests.am to list all tests
    that use the TESTS interface but not yet the `parallel-tests'
    option, with names mangled to use suffix `-p.test', in ...
    (parallel_tests): ... this new make macro.
    * tests/Makefile.am ($(srcdir)/parallel-tests.am)
    ($(parallel_tests), defs-p): New rules.
    (TESTS): Add $(parallel_tests).
    (check_SCRIPTS): Add defs-p, $(parallel_tests).
    (check-clean-local): Remove `defs-p'.
    (EXTRA_DIST): Distribute gen-parallel-tests.
    * bootstrap: Generate parallel-tests.am.
    * tests/check8.test: Check for circular dependencies in rules.
    * tests/check11.test: New test, check that SKIPs are not counted
    as passed tests.
    * tests/defs.in: Unset DISABLE_HARD_ERRORS, LAZY_TEST_SUITE,
    VERBOSE, so the tests are not influenced by the way our test
    suite is invoked.

diff --git a/bootstrap b/bootstrap
index aa50f5d..1abde1d 100755
--- a/bootstrap
+++ b/bootstrap
@@ -122,6 +122,11 @@ dosubst m4/amversion.in m4/amversion.m4
 # Create temporary replacement for automake
 dosubst automake.in automake.tmp
 
+# Create tests/parallel-tests.am.
+cd tests
+$BOOTSTRAP_SHELL ./gen-parallel-tests > parallel-tests.am
+cd ..
+
 # Run the autotools.
 $PERL ./aclocal.tmp -I m4
 autoconf
diff --git a/tests/.gitignore b/tests/.gitignore
index a6827fd..cfd4565 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1,6 +1,9 @@
 aclocal-*
 automake-*
 defs
+defs-p
+parallel-tests.am
 *.dir
 *.log
 *.log-t
+*-p.test
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 9cc6dbe..b56e7e0 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -7,6 +7,19 @@ auxdir2.test                                   \
 cond17.test                                    \
 txinfo5.test
 
+include $(srcdir)/parallel-tests.am
+
+$(srcdir)/parallel-tests.am: gen-parallel-tests Makefile.am
+       (cd $(srcdir) && $(SHELL) ./gen-parallel-tests) >$@
+
+$(parallel_tests): $(parallel_tests:-p.test=.test) Makefile.am
+       input=`echo $@ | sed 's,.*/,,; s,-p.test$$,.test,'`; \
+       sed 's,^\. \./defs,. ./defs-p,' < $(srcdir)/$$input > $@
+       chmod a+rx $@
+
+defs-p: defs Makefile.am
+       sed 's,^AM_INIT_AUTOMAKE$$,&([parallel-tests]),' < defs >$@
+
 TESTS =        \
 aclibobj.test \
 aclocal.test \
@@ -98,6 +111,7 @@ check7.test \
 check8.test \
 check9.test \
 check10.test \
+check11.test \
 checkall.test \
 clean.test \
 clean2.test \
@@ -664,15 +678,16 @@ yacc6.test \
 yacc7.test \
 yacc8.test \
 yaccpp.test \
-yaccvpath.test
+yaccvpath.test \
+$(parallel_tests)
 
-EXTRA_DIST = ChangeLog-old $(TESTS)
+EXTRA_DIST = ChangeLog-old gen-parallel-tests $(TESTS)
 
 # Each test case depends on defs, aclocal, and automake.
-check_SCRIPTS = defs aclocal-$(APIVERSION) automake-$(APIVERSION)
+check_SCRIPTS = defs defs-p aclocal-$(APIVERSION) automake-$(APIVERSION) 
$(parallel_tests)
 
 clean-local: check-clean-local
 
 check-clean-local:
        -chmod -R u+rwx *.dir
-       -rm -rf *.dir
+       -rm -rf defs-p *.dir
diff --git a/tests/check11.test b/tests/check11.test
new file mode 100755
index 0000000..d4f6edc
--- /dev/null
+++ b/tests/check11.test
@@ -0,0 +1,53 @@
+#! /bin/sh
+# Copyright (C) 2008  Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check skip summary.
+
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = skip skip2
+END
+
+cat >>skip <<'END'
+#! /bin/sh
+exit 77
+END
+chmod a+x skip
+cp skip skip2
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+unset TESTS || :
+
+./configure
+env TESTS=skip $MAKE -e check >stdout
+cat stdout
+grep '1.*passed' stdout && Exit 1
+
+env TESTS="skip skip2" $MAKE -e check >stdout
+cat stdout
+grep '2.*passed' stdout && Exit 1
+
+:
diff --git a/tests/check8.test b/tests/check8.test
index 242be56..3e38ccf 100755
--- a/tests/check8.test
+++ b/tests/check8.test
@@ -67,14 +67,20 @@ $AUTOMAKE -a
 unset TESTS || :
 
 ./configure
-AM_COLOR_TESTS=always $MAKE -e check >stdout && { cat stdout; Exit 1; }
+AM_COLOR_TESTS=always $MAKE -e check >stdout 2>stderr &&
+  { cat stdout; cat stderr >&2; Exit 1; }
 cat stdout
+cat stderr >&2
 grep 'XPASS.* foo$' stdout
 grep '^[^X]*PASS.* sub/foo$' stdout
 grep '^[^X]*PASS.* bar$' stdout
 grep '^[^X]*PASS.* sub/bar$' stdout
 grep '^[^X]*FAIL.* baz$' stdout
 grep 'XFAIL.* sub/baz$' stdout
+# parallel-tests should not add circular dependencies.
+# Look for known warnings from a couple of `make' implementations.
+grep -i 'circular.*dependency' stderr && Exit 1
+grep -i 'graph cycles' stderr && Exit 1
 
 $MAKE distclean
 
diff --git a/tests/defs.in b/tests/defs.in
index 063ce58..1e91183 100644
--- a/tests/defs.in
+++ b/tests/defs.in
@@ -290,6 +290,13 @@ unset MFLAGS
 unset MAKEFLAGS
 unset MAKELEVEL
 unset DESTDIR
+# Also unset variables that control our test driver.  While not
+# conceptually independent, they cause some changed semantics we
+# need to control (and test for) in some of the tests to ensure
+# backward-compatible behavior.
+unset DISABLE_HARD_ERRORS
+unset LAZY_TEST_SUITE
+unset VERBOSE
 
 echo "=== Running test $0"
 
diff --git a/tests/gen-parallel-tests b/tests/gen-parallel-tests
new file mode 100755
index 0000000..4a71a73
--- /dev/null
+++ b/tests/gen-parallel-tests
@@ -0,0 +1,26 @@
+#! /bin/sh
+# Generate parallel-tests.am.
+#
+# For each test in the TESTS list in this Makefile.am file, that itself
+# tests features of the TESTS automake interface, generate a sibling
+# test that does likewise, but with the option `parallel-tests' enabled.
+
+set -e
+
+tests=`sed -n '/^TESTS =/,/^$/s/\(.*\.test\).*/\1/p' Makefile.am`
+{
+  grep -l '^TESTS ' $tests
+  grep -l ' TESTS ' $tests
+} |
+grep -v '.-p\.test' |
+LC_ALL=C sort -u |
+while read tst; do
+  if grep '^[^#]*parallel-tests' $tst >/dev/null \
+     || grep '^\. \./defs-p' $tst >/dev/null
+  then :; else echo $tst; fi;
+done |
+{
+  echo "## Generated by gen-parallel-tests.  Edit Makefile.am instead of this."
+  echo "parallel_tests = \\"
+  sed 's,\.test$,-p.test,; $!s,$, \\,'
+}