[Automake-commit] [SCM] GNU Automake branch, testsuite-work, updated. v1.11-989-g6623ea6

[Stefano Lattarini]

This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU Automake".

http://git.sv.gnu.org/gitweb/?p=automake.git;a=commitdiff;h=6623ea6120320829ba8eb40dd08535f89ef47793

The branch, testsuite-work has been updated
       via  6623ea6120320829ba8eb40dd08535f89ef47793 (commit)
       via  b1109e2f9164e850dea0fea1f009d3265e51e5c8 (commit)
       via  c9e39919f0afea01e55795050d4978f530e086e2 (commit)
       via  9667c39a2cd9c1af4be172169031975febe78657 (commit)
       via  9b61494445c234e505a2efea8e1c0384591edd2f (commit)
       via  e8bcc4800b17c3fd64b951e0136e7769693edbc8 (commit)
       via  4e9474e6e9b18723d3fb02858824682dbd0e03f0 (commit)
       via  e4a70cff310e29066e388aa18aeb07242e003eec (commit)
       via  f31a38aed9c8ffbd33d797aa2df5b5864860bb17 (commit)
       via  5c237a6208c74eb445613e5237b4749d62f18b28 (commit)
       via  a9eef973b5ea47cc3495f1a8307d4f7b85aea46f (commit)
       via  42a465249b6d3425efbf23df8f2b62fc0e088cfb (commit)
      from  cc9a67f2fd5e0c422fb0bae819cb70708253b030 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
commit 6623ea6120320829ba8eb40dd08535f89ef47793
Merge: cc9a67f b1109e2
Author: Stefano Lattarini <address@hidden>
Date:   Tue Jun 28 12:01:04 2011 +0200

    Merge branch 'master' into testsuite-work
    
    * master:
      Revert "docs: parallel-tests is not experimental anymore"
      parallel-tests: stricter checks on DISABLE_HARD_ERRORS support
      docs: parallel-tests is not experimental anymore
      check: document and test $(TEST_SUITE_LOG) overriding

-----------------------------------------------------------------------

Summary of changes:
 ChangeLog                                          |   20 ++++
 doc/automake.texi                                  |   17 +++
 tests/Makefile.am                                  |    4 +
 tests/Makefile.in                                  |    4 +
 tests/parallel-tests-harderror.test                |   98 +++++++++++++++++
 tests/parallel-tests-log-override-1.test           |  112 ++++++++++++++++++++
 tests/parallel-tests-log-override-2.test           |   88 +++++++++++++++
 ...st => parallel-tests-log-override-recheck.test} |   77 +++++++------
 tests/parallel-tests.test                          |   18 +---
 9 files changed, 389 insertions(+), 49 deletions(-)
 create mode 100755 tests/parallel-tests-harderror.test
 create mode 100755 tests/parallel-tests-log-override-1.test
 create mode 100755 tests/parallel-tests-log-override-2.test
 copy tests/{parallel-tests9.test => parallel-tests-log-override-recheck.test} 
(51%)

diff --git a/ChangeLog b/ChangeLog
index 3c79637..525654b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -825,6 +825,15 @@
        ($AUTORECONF): New variable, defaulting to address@hidden@'.
        * doc/Makefile.am ($(srcdir)/amhello-1.0.tar.gz): Update.
 
+2011-05-22  Stefano Lattarini  <address@hidden>
+
+       parallel-tests: stricter checks on DISABLE_HARD_ERRORS support
+       * tests/parallel-tests-harderror.test: New test, doing more
+       in-depth checks on DISABLE_HARD_ERRORS.
+       * tests/parallel-tests.test: Remove tests on DISABLE_HARD_ERRORS,
+       now redundant.
+       * tests/Makefile.am (TESTS): Update.
+
 2011-05-22  Ralf Wildenhues  <address@hidden>
            Stefano Lattarini  <address@hidden>
 
@@ -1460,6 +1469,17 @@
        have `.o' suffix and executables have no default suffix; instead,
        use `$(OBJEXT)' and `$(EXEEXT)'.
 
+2011-05-11  Stefano Lattarini  <address@hidden>
+
+       check: document and test $(TEST_SUITE_LOG) overriding
+       * doc/automake.texi (Simple Tests using parallel-tests): Explain
+       how and why TEST_SUITE_LOG can be overridden at runtime.
+       * tests/parallel-tests-log-override-1.test: New test, check that
+       the newly documented idiom and similar usages are truly supported.
+       * tests/parallel-tests-log-override-2.test: Likewise.
+       * tests/parallel-tests-log-override-recheck.test: Likewise.
+       * tests/Makefile.am (TESTS): Update.
+
 2011-05-07  Stefano Lattarini  <address@hidden>
 
        tests: fix spurious failure of extradep.test on FreeBSD
diff --git a/doc/automake.texi b/doc/automake.texi
index afb7b33..9f40855 100644
--- a/doc/automake.texi
+++ b/doc/automake.texi
@@ -8864,6 +8864,20 @@ use a command like this to run only a subset of the 
tests:
 env TESTS="foo.test bar.test" make -e check
 @end example
 
+Note however that the command above will unconditionally overwrite the
address@hidden file, thus clobbering the recorded results
+of any previous testsuite run.  This might be undesirable for packages
+whose testsuite takes long time to execute.  Luckily, this problem can
+easily be avoided by overriding also @code{TEST_SUITE_LOG} at runtime;
+for example,
+
address@hidden
+env TEST_SUITE_LOG=partial.log TESTS="..." make -e check
address@hidden example
+
+will write the result of the partial testsuite runs to the
address@hidden, without touching @file{test-suite.log}.
+
 @item
 You can set the @code{TEST_LOGS} variable.  By default, this variable is
 computed at @command{make} run time from the value of @code{TESTS} as
@@ -8874,6 +8888,9 @@ set x subset*.log; shift
 env TEST_LOGS="foo.log $*" make -e check
 @end example
 
+The comments made above about @code{TEST_SUITE_LOG} overriding applies
+here too.
+
 @item
 @vindex RECHECK_LOGS
 @cindex lazy test execution
diff --git a/tests/Makefile.am b/tests/Makefile.am
index e1487e2..2815932 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -766,6 +766,10 @@ parallel-tests-unreadable-log.test \
 parallel-tests-subdir.test \
 parallel-tests-interrupt.test \
 parallel-tests-reset-term.test \
+parallel-tests-harderror.test \
+parallel-tests-log-override-1.test \
+parallel-tests-log-override-2.test \
+parallel-tests-log-override-recheck.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 8555866..ac0e64d 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -1054,6 +1054,10 @@ parallel-tests-unreadable-log.test \
 parallel-tests-subdir.test \
 parallel-tests-interrupt.test \
 parallel-tests-reset-term.test \
+parallel-tests-harderror.test \
+parallel-tests-log-override-1.test \
+parallel-tests-log-override-2.test \
+parallel-tests-log-override-recheck.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/parallel-tests-harderror.test 
b/tests/parallel-tests-harderror.test
new file mode 100755
index 0000000..8680e29
--- /dev/null
+++ b/tests/parallel-tests-harderror.test
@@ -0,0 +1,98 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: DISABLE_HARD_ERRORS
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_CONFIG_FILES([sub/Makefile])
+AC_OUTPUT
+END
+
+mkdir sub
+
+cat > Makefile.am << 'END'
+SUBDIRS = . sub
+TESTS = foo.test
+XFAIL_TESTS = foo.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > sub/Makefile.am << 'END'
+TESTS = bar.test
+XFAIL_TESTS = $(TESTS)
+EXTRA_DIST = $(TESTS)
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+exit 99
+END
+chmod a+x foo.test
+cp foo.test sub/bar.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+# DISABLE_HARD_ERRORS is not defined as a make variable, so that it
+# should be possible to define either from the environment or from
+# command-line, also when recursive make invocations are involved,
+# and also without using the `-e' make flag; this should work also
+# for non-GNU make. Moreover, it shouldn't be necessary to define
+# DISABLE_HARD_ERRORS to "yes" to really disable hard errors: any
+# non-empty value should do.
+$MAKE check DISABLE_HARD_ERRORS=yes
+DISABLE_HARD_ERRORS=x $MAKE check
+
+# But an empty values for DISABLE_HARD_ERRORS means that hard errors
+# are not to be counted like normal failures.
+
+$MAKE check DISABLE_HARD_ERRORS='' && Exit 1
+cat test-suite.log
+grep '^FAIL: foo\.test .*exit.*99' test-suite.log
+
+cd sub
+# The `-e' is wanted here.
+DISABLE_HARD_ERRORS='' $MAKE -e check && Exit 1
+cat test-suite.log
+grep '^FAIL: bar\.test .*exit.*99' test-suite.log
+cd ..
+
+# Check the distributions.
+$MAKE DISABLE_HARD_ERRORS=y distcheck
+
+# Finally, DISABLE_HARD_ERRORS should work also when the developer
+# sets it directly in Makefile.am or Makefile.  And its effects
+# should remain local to that specific Makefile, obviously.
+
+echo 'DISABLE_HARD_ERRORS = yes' >> Makefile.am
+$AUTOMAKE Makefile
+./config.status Makefile
+VERBOSE=yes $MAKE check && Exit 1
+grep '^FAIL' test-suite.log && Exit 1
+grep '^FAIL: bar\.test .*exit.*99' sub/test-suite.log
+
+echo 'DISABLE_HARD_ERRORS = zardoz' >> sub/Makefile
+VERBOSE=yes $MAKE check
+
+:
diff --git a/tests/parallel-tests-log-override-1.test 
b/tests/parallel-tests-log-override-1.test
new file mode 100755
index 0000000..6e98380
--- /dev/null
+++ b/tests/parallel-tests-log-override-1.test
@@ -0,0 +1,112 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: runtime redefinition of $(TEST_SUITE_LOG).
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = pass.test skip.test xfail.test
+XFAIL_TESTS = xfail.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > pass.test <<'END'
+#! /bin/sh
+exit 0
+END
+
+cat > skip.test <<'END'
+#! /bin/sh
+echo "% test skipped %"
+exit 77
+END
+
+cat > xfail.test <<'END'
+#! /bin/sh
+echo "# expected failure #"
+exit 1
+END
+
+chmod a+x *.test
+
+test_log_edit ()
+{
+  sed -e "s|^  *$me 1\.0:.*$|  $me 1.0: ???|" \
+      -e "s|^=====*|=======================|" $*
+}
+
+test_log_expected ()
+{
+  test_log_edit orig > exp
+  test_log_edit $1   > got
+  diff exp got || Exit 1
+  rm -f exp got
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check
+ls -l
+cat test-suite.log
+cp test-suite.log orig
+
+$MAKE clean
+test -f test-suite.log && Exit 99 # Sanity check.
+
+# Check that we can override the testsuite log file at runtime.
+TEST_SUITE_LOG=zardoz.log $MAKE -e check
+ls -l
+test ! -f test-suite.log
+cat zardoz.log
+test_log_expected zardoz.log
+# Sanity check the distribution too (this also does minimal checks on
+# VPATH support).
+TEST_SUITE_LOG=zardoz.log $MAKE -e distcheck
+
+# Check that cleanup rules remove the correct file even when
+# user overrides are in place.
+cp orig test-suite.log
+TEST_SUITE_LOG=zardoz.log $MAKE -e clean
+ls -l
+test ! -f zardoz.log
+diff orig test-suite.log
+
+# Check that the default testsuite log doesn't get unduly modified.
+# Also check that the testsuite log file doesn't need to be named
+# accordingly to the `*.log' pattern.
+chmod a-w test-suite.log
+TEST_SUITE_LOG=TheLogFile $MAKE -e check
+ls -l
+diff orig test-suite.log
+test_log_expected TheLogFile
+TEST_SUITE_LOG=TheLogFile $MAKE -e clean
+ls -l
+test ! -f TheLogFile
+diff orig test-suite.log
+
+:
diff --git a/tests/parallel-tests-log-override-2.test 
b/tests/parallel-tests-log-override-2.test
new file mode 100755
index 0000000..649360c
--- /dev/null
+++ b/tests/parallel-tests-log-override-2.test
@@ -0,0 +1,88 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: runtime redefinition of:
+#  - $(TEST_SUITE_LOG) and $(TESTS)
+#  - $(TEST_SUITE_LOG) and $(TEST_LOGS)
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = pass.test pass2.test skip.test skip2.test fail.test
+END
+
+cat > pass.test <<'END'
+#! /bin/sh
+exit 0
+END
+
+cp pass.test pass2.test
+
+cat > skip.test <<'END'
+#! /bin/sh
+echo "% skipped test %"
+exit 77
+END
+
+cp skip.test skip2.test
+
+cat > fail.test <<'END'
+#! /bin/sh
+exit 1
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+for test_list_override in \
+  'TESTS=pass.test skip.test' \
+  'TEST_LOGS=pass.log skip.log'
+do
+  env TEST_SUITE_LOG=partial.log "$test_list_override" \
+    $MAKE -e check >stdout || { cat stdout; Exit 1; }
+  cat stdout
+  ls -l
+  cat pass.log
+  cat skip.log
+  cat partial.log
+  test ! -f test-suite.log
+  test ! -f pass2.log
+  test ! -f skip2.log
+  test ! -f fail.log
+  grep '^PASS: .*pass\.test' stdout
+  grep '^SKIP: .*skip\.test' stdout
+  $FGREP 'skip.test' partial.log
+  $FGREP '% skipped test %' partial.log
+  for t in pass2 skip2 fail; do
+    $FGREP "$t.test" stdout && Exit 1
+    $FGREP "$t.test" partial.log && Exit 1
+  done
+  rm -f *.log
+done
+
+:
diff --git a/tests/parallel-tests9.test 
b/tests/parallel-tests-log-override-recheck.test
similarity index 51%
copy from tests/parallel-tests9.test
copy to tests/parallel-tests-log-override-recheck.test
index b06d364..8a50d29 100755
--- a/tests/parallel-tests9.test
+++ b/tests/parallel-tests-log-override-recheck.test
@@ -1,5 +1,5 @@
 #! /bin/sh
-# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+# Copyright (C) 2011 Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,42 +14,43 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Check parallel-tests features:
-# - recheck
+# Check parallel-tests features: runtime redefinition of $(TEST_SUITE_LOG)
+# for the recheck target.
 
 parallel_tests=yes
 . ./defs || Exit 1
 
-cat >> configure.in << 'END'
+set -e
+
+cat >> configure.in <<'END'
 AC_OUTPUT
 END
 
-cat > Makefile.am << 'END'
-TEST_SUITE_LOG = mylog.log
+cat > Makefile.am <<'END'
 TESTS = foo.test bar.test baz.test
-check_SCRIPTS = bla
-CLEANFILES = bla
-bla:
-       echo bla > $@
 END
 
 cat > foo.test <<'END'
 #! /bin/sh
 echo "this is $0"
-test -f bla || exit 1
 exit 0
 END
+
 cat > bar.test <<'END'
 #! /bin/sh
 echo "this is $0"
 exit 99
 END
+
 cat > baz.test <<'END'
 #! /bin/sh
 echo "this is $0"
-exit 1
+exit ${BAZ_EXIT_STATUS-1}
 END
-chmod a+x foo.test bar.test baz.test
+
+chmod a+x *.test
+
+unset BAZ_EXIT_STATUS || :
 
 $ACLOCAL
 $AUTOCONF
@@ -59,29 +60,33 @@ $AUTOMAKE -a
 $MAKE check >stdout && { cat stdout; Exit 1; }
 cat stdout
 
-$MAKE recheck >stdout && { cat stdout; Exit 1; }
+chmod a-rw test-suite.log
+TEST_SUITE_LOG=my.log $MAKE -e recheck >stdout \
+  && { cat stdout; Exit 1; }
+cat stdout
+ls -l
+grep '^2 of 2 .*failed' stdout
+for x in stdout my.log; do
+  $FGREP foo.test $x && Exit 1
+  $FGREP bar.test $x
+  $FGREP baz.test $x
+done
+
+chmod a-rw my.log
+BAZ_EXIT_STATUS=0 TEST_SUITE_LOG=my2.log $MAKE -e recheck >stdout \
+  && { cat stdout; Exit 1; }
 cat stdout
-grep foo.test stdout && Exit 1
-grep bar.test stdout || Exit 1
-grep baz.test stdout || Exit 1
-grep '2 of 2.*failed' stdout
-
-# If we cannot read the log file, then redo it as well.
-chmod a-r foo.log
-if test ! -r foo.log; then
-   $MAKE recheck >stdout && { cat stdout; Exit 1; }
-   cat stdout
-   grep foo.test stdout || Exit 1
-fi
-
-# Ensure that recheck builds check_SCRIPTS, and that
-# recheck reruns nothing if check has not been run.
-$MAKE clean
-$MAKE recheck
-test -f bla
-test ! -f foo.log
-test ! -f bar.log
-test ! -f baz.log
-test -f mylog.log
+ls -l
+grep '^1 of 2 .*failed' stdout
+$FGREP foo.test stdout && Exit 1
+$FGREP bar.test stdout
+$FGREP baz.test stdout
+$FGREP foo.test my2.log && Exit 1
+$FGREP bar.test my2.log
+$FGREP baz.test my2.log && Exit 1
+
+chmod u+r test-suite.log my.log
+$FGREP baz.test test-suite.log
+$FGREP baz.test my.log
 
 :
diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test
index ea92ae7..b1e157c 100755
--- a/tests/parallel-tests.test
+++ b/tests/parallel-tests.test
@@ -19,7 +19,6 @@
 # - clean
 # - TEST_SUITE_LOG
 # - dependencies between tests
-# - DISABLE_HARD_ERRORS
 # - TESTS
 # - TEST_LOGS
 # - RECHECK_LOGS
@@ -66,12 +65,13 @@ $AUTOCONF
 $AUTOMAKE -a
 
 ./configure
-# No hard errors: all tests should be run, there should be one failure.
-env DISABLE_HARD_ERRORS=yes $MAKE -e check >stdout && { cat stdout; Exit 1; }
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
 cat stdout
-test `grep -c '^FAIL' stdout` -eq 1
+# There should be two errors: bar.test is a hard error.
+test `grep -c '^FAIL' stdout` -eq 2
 test -f mylog.log
-test `grep -c '^FAIL' mylog.log` -eq 1
+test `grep -c '^FAIL' mylog.log` -eq 2
 test -f baz.log
 test -f bar.log
 test -f foo.log
@@ -87,19 +87,11 @@ test ! -f foo.log
 test ! -f mylog.log
 test -f unrelated.log
 
-$MAKE clean
-$MAKE check >stdout && { cat stdout; Exit 1; }
-cat stdout
-# Now, there should be two errors: bar.test is a hard error.
-test `grep -c '^FAIL' stdout` -eq 2
-test `grep -c '^FAIL' mylog.log` -eq 2
-
 # Check dependencies: baz.test needs to run before bar.test,
 # but foo.test is not needed.
 # Note that this usage has a problem: the summary will only
 # take bar.log into account, because the $(TEST_SUITE_LOG) rule
 # does not "see" baz.log.  Hmm.
-$MAKE clean
 env TESTS='bar.test' $MAKE -e check && Exit 1
 test -f baz.log
 test -f bar.log


hooks/post-receive
-- 
GNU Automake