[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Automake-commit] [SCM] GNU Automake branch, test-protocols, updated. v1
|
From: |
Stefano Lattarini |
|
Subject: |
[Automake-commit] [SCM] GNU Automake branch, test-protocols, updated. v1.11-957-g9a3268ca7 |
|
Date: |
Wed, 03 Aug 2011 16:14:00 +0000 |
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU Automake".
http://git.sv.gnu.org/gitweb/?p=automake.git;a=commitdiff;h=9a3268ca7a4ec184f014a322ee1ffcbf7055dfd1
The branch, test-protocols has been updated
via 9a3268ca7a4ec184f014a322ee1ffcbf7055dfd1 (commit)
via f5c022ededc54e53eb3f8a6702c757015f87e322 (commit)
via 2815a73e40df4a98c9693c63b071cdf0d8d13d2a (commit)
via f4e516e6d4ac0fae8919b1997432443ce90320ac (commit)
via a5b34b4c6aa1f0d66537c0b4683b6d7dc39a7ffc (commit)
via 05088171d107f4b904e41c308be493efa4e08670 (commit)
via dac4d254d1b8d79bd197d4aece6b58b75ad81247 (commit)
via b9fc781cf7ddeab662e331c12d664d4b6ec2e8ad (commit)
via 9cddc4f56b14502c110b0b1459fa9890c91997ec (commit)
via 5d4dc886c0863ed2a4fdec933a1bded31402094b (commit)
via b1a5d2e3e5255e5ff5abda272c985a5d54073ed9 (commit)
via c349db06721d376cb70be78450e78d30a476910d (commit)
via 25903324b11ea930bf6f6107e119158d494a0603 (commit)
via a2e67e599367ae14bb69fd99202a136db0ec4762 (commit)
via d7fc560de1c1eb43cbdcf68f6a755dbe2ce42adf (commit)
via bd4c823784524b079fc6f0b709d80900e62106af (commit)
via 6d85101e57f1b6a57e317a415b5c46abd35509e4 (commit)
via a017bc080e3c36daa7ee0f3e26820d5f8540fe74 (commit)
via 9fb15f19427bb194feb87c59c52169b2fb9a04d5 (commit)
via 79ddf6800c051f0b75b0b4115568644c156351c5 (commit)
via 6483ac07dfcca5ddd282113f23c3771a8027767c (commit)
via 0c81b43f711fb861f04227ced8dba889596d9c43 (commit)
via 4014b502e2180c1c1ea501ef77e5f97c186d4cbe (commit)
via 24dd0449267486b3268ad812c5fda8b60dad1981 (commit)
via 50c32b61b305cb8c2965470927522ed2f226b917 (commit)
via b9e9d546e8ab45139be6d8e939602cd712c36c8c (commit)
via 3af7ec13fa085bbb532384d6bb7fd01fbd62391d (commit)
via 2fdeb6bb98b502373fbd7af66fd304724ddc94b6 (commit)
via 7858f2bbb985536317fe2b5285ea332db96161c9 (commit)
via ba5fce66ab2e1892ee7f92cc71cf3f1517f6b5bc (commit)
via 58c182ae59f585c5687104e0112d163274c6c732 (commit)
via 1bb5974d3d955590eb771e547a2858430284f477 (commit)
via 6b3aabe1ad4acfb83ad68d9768a5e7523886fd37 (commit)
via 40ef9a0b95e9f15eb53f21db2e07f82342beaad9 (commit)
via 3680f7deb922ec3128eb1a69788392984b856860 (commit)
via b6a40fa2d2c08f71f9aeced063a36a27f657da6b (commit)
via 4e0ccf0f434fc0e83d2dd46d2d14791ee86df06b (commit)
via 9201a314dbbb4793023a069d8530bc37c34b09b7 (commit)
via eaac33fe13811cf104f45a09a6ece911c135fa83 (commit)
via 15400bca303910d059f1a68f94b25cffb1a8ac2f (commit)
via 2b0687bb9db3b34ccb0de7bebd4769f635ae1fd1 (commit)
via 158e1d9b450600f15226eefc5b7a026478e1500b (commit)
via 7450206489bea8b4150d564a70413c541c08596b (commit)
via edee35da30481a0fa26800bd514bb177416f7e83 (commit)
via 9d1a3a28e52edf568f5edd598446d00c6deebec4 (commit)
via b01fc0873d749b73fcd54378772d36c05ce81bf2 (commit)
via 76cfe59c148e8ba4299fbd2ee9f29ede6198698e (commit)
via 8285d525bfa39018781f5c49b5caa3a44960b8b1 (commit)
via 839b1e34dce6a7038573157a10a4000488926f8d (commit)
via 2b1eeec164e79a573ed9b92a1599c0d2a9b7f694 (commit)
via fae45c77d9dfd2b566730bf80fc17d91d61a6931 (commit)
via e58e5f493fa0856d6782549d978c3f1d5f24de8f (commit)
via 61b2bb0d647ea5f192e49f5f5885009ea83968be (commit)
via 8c7c3b592e8fe01e263ade96eddbc85f2619943c (commit)
via 044035ce85d089a9bbdfd6fa99c31dd22b74e7db (commit)
via f8cb365330f9b00aa821b5d64239c94d7b4284c1 (commit)
via 12803f06863ba563237ab2404e28897c07f8b757 (commit)
via a44667dce47afd200d990910f8e8a966276be756 (commit)
via 1b5e0246957acb4987e12b48d6ffbe9166cb8f82 (commit)
via cd8f7420f78b3d9a9005cd425f6f27545d0f7aaa (commit)
via e7a6300b022c251e067dd75229d5c02b658c49d2 (commit)
from 1402741313cf6c2fe8611eb294f9a8111c994a32 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
-----------------------------------------------------------------------
Summary of changes:
ChangeLog | 784 +++++++++++++++-
HACKING | 7 +-
INSTALL | 9 +-
Makefile.am | 16 +-
Makefile.in | 7 +-
NEWS | 25 +
automake.in | 4 +
doc/automake.texi | 1030 ++++++++++++++------
lib/Automake/tests/.gitignore | 1 +
lib/Automake/tests/Makefile.in | 227 +++--
lib/INSTALL | 9 +-
lib/Makefile.am | 2 +-
lib/Makefile.in | 2 +-
lib/am/check.am | 281 +++---
lib/am/check2.am | 16 +-
lib/am/configure.am | 7 +-
lib/config-ml.in | 15 +-
lib/config.guess | 217 +++--
lib/config.sub | 111 ++-
lib/tap-driver | 426 ++++++++
lib/test-driver | 35 +-
lib/texinfo.tex | 536 +++++++---
tests/.gitignore | 1 +
tests/Makefile.am | 142 +++-
tests/Makefile.in | 396 +++++---
tests/README | 7 +
tests/ac-output-old.tap | 179 ++++
tests/acoutbs.test | 42 -
tests/acoutbs2.test | 47 -
tests/acoutnoq.test | 35 -
tests/acoutpt.test | 32 -
tests/acoutpt2.test | 44 -
tests/acoutqnl.test | 32 -
tests/am-tests-environment.test | 93 ++
tests/amhello-binpkg.test | 50 +
tests/amhello-cflags.test | 51 +
tests/amhello-cross-compile.test | 54 +
tests/check-fd-redirect.test | 110 +++
tests/check10.test | 2 +
tests/check11.test | 15 +-
tests/color.test | 105 ++-
tests/color2.test | 112 ++-
tests/defs | 68 +-
tests/defs-static.in | 11 +-
tests/extract-testsuite-summary | 15 +
tests/interp.test | 10 +-
tests/parallel-tests-am_tests_environment.test | 97 --
tests/parallel-tests-cmdline-override.test | 84 ++
tests/parallel-tests-console-output.test | 104 ++
...all.test => parallel-tests-driver-install.test} | 0
tests/parallel-tests-dry-run.test | 112 +++
tests/parallel-tests-empty-testlogs.test | 2 +-
tests/parallel-tests-exit-statuses.test | 108 ++
tests/parallel-tests-extra-programs.test | 174 ++++
tests/parallel-tests-fd-redirect.test | 109 ++
tests/parallel-tests-harderror.test | 6 +-
tests/parallel-tests-log-compiler-example.test | 71 ++
tests/parallel-tests-log-override-2.test | 10 +-
tests/parallel-tests-log-override-recheck.test | 4 +-
tests/parallel-tests-no-color-in-log.test | 63 ++
tests/parallel-tests-no-spurious-summary.test | 68 ++
tests/parallel-tests-once.test | 48 +
tests/parallel-tests-unreadable-log.test | 60 --
tests/parallel-tests-unreadable.test | 75 ++
tests/parallel-tests.test | 62 +-
tests/parallel-tests3.test | 2 +-
tests/parallel-tests6.test | 4 +-
tests/parallel-tests9.test | 17 +-
tests/plain-functions.sh | 39 +
tests/remake-subdir-long-time.test | 116 +++
tests/self-check-env-sanitize.test | 14 +-
tests/self-check-is_newest.test | 33 +
tests/self-check-tap.test | 34 +
tests/specflg8.test | 33 +-
tests/tap-autonumber.test | 75 ++
tests/tap-bad-prog.tap | 86 ++
tests/tap-bailout.test | 138 +++
tests/tap-basic.test | 177 ++++
tests/tap-color.test | 165 ++++
tests/tap-common-setup.test | 47 +
tests/tap-deps.test | 91 ++
tests/tap-diagnostic.test | 124 +++
tests/tap-doc.test | 92 ++
tests/tap-doc2.test | 139 +++
tests/tap-empty-diagnostic.test | 47 +
tests/tap-empty.test | 55 +
tests/tap-escape-directive.test | 45 +
tests/tap-exit.test | 54 +
tests/tap-fancy.test | 133 +++
tests/tap-fancy2.test | 144 +++
tests/tap-functions.sh | 239 +++++
tests/tap-global-log.test | 122 +++
tests/tap-global-result.test | 104 ++
tests/tap-html.test | 80 ++
tests/tap-log.test | 164 ++++
tests/tap-merge-stdout-stderr.test | 73 ++
tests/tap-message-0.test | 66 ++
tests/tap-more.test | 164 ++++
tests/tap-more2.test | 93 ++
tests/tap-no-disable-hard-error.test | 50 +
tests/tap-no-merge-stdout-stderr.test | 53 +
tests/tap-no-spurious-summary.test | 47 +
tests/tap-no-spurious.test | 102 ++
tests/tap-not-ok-skip.test | 39 +
tests/tap-numeric-description.test | 69 ++
tests/tap-out-of-order.test | 88 ++
tests/tap-passthrough-exit.test | 66 ++
tests/tap-passthrough.test | 166 ++++
tests/tap-plan-corner.test | 63 ++
tests/tap-plan-corner2.test | 49 +
tests/tap-plan-errors.test | 92 ++
tests/tap-plan.test | 51 +
tests/tap-realtime.test | 101 ++
tests/tap-recheck-logs.test | 119 +++
tests/tap-recheck.test | 217 ++++
tests/tap-setup.sh | 48 +
tests/tap-signal.test | 49 +
tests/tap-skip-whole-whitespace.test | 48 +
tests/tap-skip-whole.test | 83 ++
tests/tap-summary-aux.sh | 361 +++++++
tests/tap-summary-color.test | 23 +
tests/tap-summary.test | 23 +
tests/tap-todo-skip-together.test | 39 +
tests/tap-todo-skip-whitespace.test | 100 ++
tests/tap-todo-skip.test | 91 ++
tests/tap-unplanned.test | 159 +++
tests/tap-whitespace-normalization.test | 165 ++++
tests/tap-with-and-without-number.test | 50 +
tests/tap-xfail-tests.test | 68 ++
tests/test-driver-create-log-dir.test | 19 +-
tests/test-driver-custom-multitest-recheck.test | 35 +-
tests/test-driver-custom-multitest-recheck2.test | 66 +-
tests/test-driver-custom-multitest.test | 55 +-
tests/test-driver-custom-no-html.test | 10 +-
tests/test-driver-custom-xfail-tests.test | 36 +-
tests/test-driver-custom.test | 4 +
tests/test-driver-global-log.test | 110 ---
tests/test-driver-recheck.test | 126 ---
tests/test-driver-strip-vpath.test | 2 +
tests/test-driver-trs-suffix-registered.test | 58 ++
tests/test-log.test | 168 ++++
tests/test-metadata-global-log.test | 147 +++
tests/test-metadata-recheck.test | 163 +++
tests/test-metadata-results.test | 169 ++++
tests/test-missing.test | 72 ++
tests/test-missing2.test | 54 +
tests/test-trs-basic.test | 141 +++
tests/test-trs-recover.test | 167 ++++
tests/test-trs-recover2.test | 133 +++
tests/tests-environment-backcompat.test | 61 ++
...sts_environment.test => tests-environment.test} | 0
tests/testsuite-summary-checks.sh | 106 ++
tests/testsuite-summary-color.test | 199 ++++
tests/testsuite-summary-count-many.test | 105 ++
tests/testsuite-summary-count.test | 176 ++++
tests/testsuite-summary-reference-log.test | 88 ++
tests/trivial-test-driver | 81 +-
tests/txinfo21.test | 4 +-
158 files changed, 12983 insertions(+), 1937 deletions(-)
create mode 100755 lib/tap-driver
create mode 100755 tests/ac-output-old.tap
delete mode 100755 tests/acoutbs.test
delete mode 100755 tests/acoutbs2.test
delete mode 100755 tests/acoutnoq.test
delete mode 100755 tests/acoutpt.test
delete mode 100755 tests/acoutpt2.test
delete mode 100755 tests/acoutqnl.test
create mode 100755 tests/am-tests-environment.test
create mode 100755 tests/amhello-binpkg.test
create mode 100755 tests/amhello-cflags.test
create mode 100755 tests/amhello-cross-compile.test
create mode 100755 tests/check-fd-redirect.test
create mode 100644 tests/extract-testsuite-summary
delete mode 100755 tests/parallel-tests-am_tests_environment.test
create mode 100755 tests/parallel-tests-cmdline-override.test
create mode 100755 tests/parallel-tests-console-output.test
rename tests/{parallel-test-driver-install.test =>
parallel-tests-driver-install.test} (100%)
create mode 100755 tests/parallel-tests-dry-run.test
create mode 100755 tests/parallel-tests-exit-statuses.test
create mode 100755 tests/parallel-tests-extra-programs.test
create mode 100755 tests/parallel-tests-fd-redirect.test
create mode 100755 tests/parallel-tests-log-compiler-example.test
create mode 100755 tests/parallel-tests-no-color-in-log.test
create mode 100755 tests/parallel-tests-no-spurious-summary.test
create mode 100755 tests/parallel-tests-once.test
delete mode 100755 tests/parallel-tests-unreadable-log.test
create mode 100755 tests/parallel-tests-unreadable.test
create mode 100644 tests/plain-functions.sh
create mode 100755 tests/remake-subdir-long-time.test
create mode 100755 tests/self-check-tap.test
create mode 100755 tests/tap-autonumber.test
create mode 100755 tests/tap-bad-prog.tap
create mode 100755 tests/tap-bailout.test
create mode 100755 tests/tap-basic.test
create mode 100755 tests/tap-color.test
create mode 100755 tests/tap-common-setup.test
create mode 100755 tests/tap-deps.test
create mode 100755 tests/tap-diagnostic.test
create mode 100755 tests/tap-doc.test
create mode 100755 tests/tap-doc2.test
create mode 100755 tests/tap-empty-diagnostic.test
create mode 100755 tests/tap-empty.test
create mode 100755 tests/tap-escape-directive.test
create mode 100755 tests/tap-exit.test
create mode 100755 tests/tap-fancy.test
create mode 100755 tests/tap-fancy2.test
create mode 100644 tests/tap-functions.sh
create mode 100755 tests/tap-global-log.test
create mode 100755 tests/tap-global-result.test
create mode 100755 tests/tap-html.test
create mode 100755 tests/tap-log.test
create mode 100755 tests/tap-merge-stdout-stderr.test
create mode 100755 tests/tap-message-0.test
create mode 100755 tests/tap-more.test
create mode 100755 tests/tap-more2.test
create mode 100755 tests/tap-no-disable-hard-error.test
create mode 100755 tests/tap-no-merge-stdout-stderr.test
create mode 100755 tests/tap-no-spurious-summary.test
create mode 100755 tests/tap-no-spurious.test
create mode 100755 tests/tap-not-ok-skip.test
create mode 100755 tests/tap-numeric-description.test
create mode 100755 tests/tap-out-of-order.test
create mode 100755 tests/tap-passthrough-exit.test
create mode 100755 tests/tap-passthrough.test
create mode 100755 tests/tap-plan-corner.test
create mode 100755 tests/tap-plan-corner2.test
create mode 100755 tests/tap-plan-errors.test
create mode 100755 tests/tap-plan.test
create mode 100755 tests/tap-realtime.test
create mode 100755 tests/tap-recheck-logs.test
create mode 100755 tests/tap-recheck.test
create mode 100755 tests/tap-setup.sh
create mode 100755 tests/tap-signal.test
create mode 100755 tests/tap-skip-whole-whitespace.test
create mode 100755 tests/tap-skip-whole.test
create mode 100755 tests/tap-summary-aux.sh
create mode 100755 tests/tap-summary-color.test
create mode 100755 tests/tap-summary.test
create mode 100755 tests/tap-todo-skip-together.test
create mode 100755 tests/tap-todo-skip-whitespace.test
create mode 100755 tests/tap-todo-skip.test
create mode 100755 tests/tap-unplanned.test
create mode 100755 tests/tap-whitespace-normalization.test
create mode 100755 tests/tap-with-and-without-number.test
create mode 100755 tests/tap-xfail-tests.test
delete mode 100755 tests/test-driver-global-log.test
delete mode 100755 tests/test-driver-recheck.test
create mode 100755 tests/test-driver-trs-suffix-registered.test
create mode 100755 tests/test-log.test
create mode 100755 tests/test-metadata-global-log.test
create mode 100755 tests/test-metadata-recheck.test
create mode 100755 tests/test-metadata-results.test
create mode 100755 tests/test-missing.test
create mode 100755 tests/test-missing2.test
create mode 100755 tests/test-trs-basic.test
create mode 100755 tests/test-trs-recover.test
create mode 100755 tests/test-trs-recover2.test
create mode 100755 tests/tests-environment-backcompat.test
rename tests/{check-tests_environment.test => tests-environment.test} (100%)
create mode 100755 tests/testsuite-summary-checks.sh
create mode 100755 tests/testsuite-summary-color.test
create mode 100755 tests/testsuite-summary-count-many.test
create mode 100755 tests/testsuite-summary-count.test
create mode 100755 tests/testsuite-summary-reference-log.test
diff --git a/ChangeLog b/ChangeLog
index 6203f89..2d51a28 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,779 @@
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ maintcheck: fix more maintainer-check failures
+ * tests/test-driver-custom-no-html.test (no-rst): Use `EOF',
+ not `EoL', as the here-document delimiter.
+ * tests/test-trs-basic.test: Use `cat + here-doc' rather
+ than `echo' when creating the dummy test scripts, to please
+ maintainer-check.
+ * tests/test-trs-recover.test: Use creative quoting where
+ needed, to please maintainer-check.
+ * tests/parallel-tests-no-color-in-log.test: Likewise.
+ * tests/parallel-tests-dry-run.test: Likewise.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ maintcheck: fix maintainer-check failures, both real and spurious
+ * tests/amhello-binpkg.test: Use "$MAKE", not bare "make".
+ * Makefile.am (sc_perl_local): Also allow perl special variable
+ `$~' to be localized. And be slightly laxer in the regexp, to
+ allow for usages like "local $_ = $foo;".
+ (sc_tests_overriding_macros_on_cmdline): Also allow for command
+ line overriding of the `DISABLE_HARD_ERRORS' make variable. Try
+ to avoid false positives for usages like "$MAKE || st=$?".
+
+2011-08-03 Bruno Haible <address@hidden>
+
+ docs: how to use '-I' option in AM_CPPFLAGS for best VPATH support
+ * doc/automake.texi (Program Variables): Recommend -I options to
+ both the build directory and the source directory when needed.
+
+2011-07-23 Stefano Lattarini <address@hidden>
+
+ tests: relax a test on amhello examples to cater to Solaris tar
+ * tests/amhello-binpkg.test: When the tar implementation in use
+ is not GNU tar, relax the tests on tar output, to avoid spurious
+ failures. For example, "tar cvf ..." with GNU tar can output
+ lines like "./usr/bin/hello" on the standard output, while with
+ Solaris tar it can output lines like "a ./usr/bin/hello 8K" on
+ standard output, and with Heirloom tar it can output lines like
+ "a ./usr/bin/hello 15 tape blocks" on standard error.
+
+2011-07-23 Stefano Lattarini <address@hidden>
+
+ test defs: function 'is_newest' now works also with directories
+ * tests/defs (is_newest): Call `find' with the `-prune' option,
+ so that it won't descend in the directories (which could cause
+ spurious results).
+ * tests/self-check-is_newest: Extend accordingly.
+ From a report by Jim Meyering, see automake bug#9147.
+
+2011-07-15 Benoit Sigoure <address@hidden>
+
+ docs: add references between the 2 sections on java support
+ * doc/automake.texi (Java Support, Java): Add cross-references.
+
+2011-07-20 Ralf Wildenhues <address@hidden>
+
+ Sync auxiliary files from upstream.
+ * INSTALL, lib/INSTALL, lib/config.guess, lib/config.sub,
+ lib/texinfo.tex: Sync from upstream.
+
+2011-07-08 Stefano Lattarini <address@hidden>
+
+ tests: fix weakness in 'tests-environment-backcompat.test'
+ * tests/tests-environment-backcompat.test: Do not override the
+ content of xfailing test `baz.test' with a "weaker" version that
+ fails unconditionally: the test must fail only when the 'strict'
+ pragma is in use, in order not to reduce coverage.
+
+2011-07-08 Stefano Lattarini <address@hidden>
+
+ docs, tests: synchronize examples on silent-rules from config.site
+ * doc/automake.texi (Automake silent-rules Option): Reference test
+ 'silent-configsite.test' in comments.
+
+2011-07-08 Stefano Lattarini <address@hidden>
+
+ tests: portability fixes in tests on amhello examples
+ * tests/amhello-binpkg.test: Don't use "tar xzf too.tag.gz" to
+ extract a gzip-compressed tarball, that's unportable to some
+ tar implementations; use the "gzip -dc fo.tar.gz | tar xf -"
+ idiom instead.
+ * tests/amhello-cflags.test: Likewise.
+ * tests/amhello-cross-compile.test: Likewise.
+ Suggestion from Ralf Wildenhues.
+
+2011-07-04 Stefano Lattarini <address@hidden>
+
+ remake: add test ensuring that slower remakes don't hang
+ * tests/remake-subdir-long-time.test: New test.
+ * tests/Makefile.am (TESTS): Update.
+ Suggestion by Ralf Wildenhues.
+
+2011-07-04 Stefano Lattarini <address@hidden>
+
+ remake: fix outdated comment in configure.am
+ * lib/am/configure.am: Fix comment falsified by changes in
+ commit `v1.11-366-gbee9871'.
+ Suggestion by Ralf Wildenhues.
+
+2011-07-01 Stefano Lattarini <address@hidden>
+
+ docs, tests: synchronize examples from docs to tests
+ * tests/README (Writing test cases): Give suggestions on how to
+ keep test cases and examples in the documentation synchronized.
+ * doc/automake.texi: Improve or fix existing testcase-referencing
+ comments, and add many new ones.
+ * HACKING (Administrivia): Suggest to test complex examples and
+ idioms from the manual.
+ * tests/specflg8.test: Improve synchronization with the example
+ in the manual.
+ * tests/output11.test:Likewise.
+ * tests/txinfo21.test:Likewise.
+ * tests/interp.test: Likewise.
+ * tests/amhello-cflags.test: New test.
+ * tests/amhello-cross-compile.test: Likewise.
+ * tests/amhello-binpkg.test: Likewise.
+ * tests/tests-environment-backcompat.test: Likewise.
+ * tests/parallel-tests-log-compiler-example.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-06-30 Stefano Lattarini <address@hidden>
+
+ coverage: new test on parallel-tests TESTS runtime overriding
+ * tests/parallel-tests-cmdline-override.test: New test, check that
+ we can use indirections when overriding TESTS and TEST_LOGS from
+ the command line.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ gitignore: ignore `*.trs' files in lib/Automake/tests
+ * lib/Automake/tests/.gitignore: Add `*.trs' pattern.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: list another "forgotten" test script in Makefile.am
+ * tests/Makefile.am (TESTS): Also list test script
+ `test-driver-trs-suffix-registered.test'; it had been introduced
+ in commit `v1.11-910-g0c81b43', but by mistake it wasn't added
+ to the list of tests in the Makefile back then.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: list "forgotten" test script in Makefile.am
+ * tests/Makefile.am (TESTS): Also list the test script
+ `parallel-tests-no-color-in-log.test'; it had been introduced in
+ commit `v1.11-899-geaac33f', but by mistake it wasn't added to
+ the list of tests in the Makefile back then.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: remove duplicated mention of a test in Makefile.am
+ * tests/Makefile.am (tap_other_tests): Remove `tap-empty.test',
+ it is already listed in `$(tap_with_common_setup_tests)'.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: one more use of TAP in our own tests
+ * tests/tap-bad-prog.test, tests/tap-bad-prog2.test: Merged
+ into ...
+ * tests/tap-bad-prog.tap: ... this TAP-generating test.
+ * tests/Makefile.am (tap_other_tests): Remove plan-bad-prog.test
+ and plan-bad-prog2.test.
+ (XFAIL_TESTS): Remove plan-bad-prog2.test.
+ (plan-bad-prog2.log): Remove.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: scaffolding to allow use of TAP in our own tests
+ Now that Automake has initial support for the TAP test protocol,
+ we can start "eating our own dog food" and rewrite some tests to
+ use TAP; this should allow for better granularity, especially in
+ the control of skips and expected failures. With this change, we
+ set up the initial scaffolding required by the planned TAP tests,
+ and convert few older tests to use TAP, mostly in order to verify
+ that there are no obvious errors. The tests in our testsuite that
+ use the TAP protocol will be marked by the new `.tap' extension.
+ * tests/plain-functions.sh: New file containing definition of
+ helper shell functions used by the "protocol-less" tests of the
+ Automake testsuite.
+ * tests/tap-functions.sh: New file containing definition of
+ helper shell functions used by the TAP-producing tests of the
+ Automake testsuite.
+ * tests/defs: Remove definitions of some functions that are
+ now defined in `plain-functions.sh'. Instead, source one of
+ `plain-functions.sh' or `tap-functions.sh', depending on whether
+ `$use_tap' is set to "no" or "yes".
+ (exit trap): Call `late_plan_' if `$use_tap' is true. Also,
+ unset shell traces before issuing the latest commands, to avoid
+ confusing the tap driver with spurious output.
+ * tests/defs-static.in ($use_tap): New variable, by default set
+ to "yes" if the calling test script has a `.tap' suffix, and to
+ "no" otherwise. The individual scripts can override it though.
+ In code sanity-checking the environment, verify that `$use_tap'
+ is not exported.
+ * tests/self-check-env-sanitize.test: Update, and small related
+ reformatting.
+ * tests/self-check-tap.test: New very minimal self test.
+ * tests/acoutbs.test, tests/acoutbs2.test, tests/acoutnoq.test,
+ tests/acoutpt.test, tests/acoutpt2.test, tests/acoutqnl.test:
+ Removed, merged into ...
+ * tests/ac-output-old.tap: ... this new test, that uses TAP.
+ * tests/Makefile.am (TAP_LOG_DRIVER): Define to invoke our own
+ `tap-driver' script.
+ (TAP_LOG_DRIVER_FLAGS): Define to `--merge', so that the stdout
+ and stderr of the test scripts remains synced.
+ (TEST_EXTENSIONS): Define, to add the `.tap' suffix; also list
+ the `.test' suffix explicitly.
+ (EXTRA_DIST): Distribute the new files `plain-functions.sh' and
+ `tap-functions.sh'.
+ ($(TEST_LOGS)): Depends on them.
+ (AM_TESTS_ENVIRONMENT): Ensure that `use_tap' is not exported in
+ the environment of the tests.
+ (TESTS): Update.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ * NEWS: Fix typo, and related reformatting.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ * NEWS: add mention of new experimental TAP support
+
+2011-08-02 Stefano Lattarini <address@hidden>
+
+ testsuite: rename some test scripts
+ * tests/parallel-tests-am_tests_environment.test: Renamed ...
+ * tests/am-tests-environment.test: ... to this.
+ * tests/check-tests_environment.test: Renamed ...
+ * tests/tests-environment.test: ... to this.
+ * tests/parallel-test-driver-install.test: Renamed ...
+ * tests/parallel-tests-driver-install.test: ... to this.
+ * tests/parallel-tests-make-n.test: Renamed ...
+ * tests/parallel-tests-dry-run.test: ... to this.
+ * tests/Makefile.am (TESTS): Updated.
+
+2011-08-02 Stefano Lattarini <address@hidden>
+
+ coverage: parallel-tests and lazy dependencies on EXTRA_PROGRAMS
+ * tests/parallel-tests-extra-programs.test: New test.
+ * tests/Makefile.am (TESTS): Update.
+ * doc/automake.texi (Parallel Test Harness): Add a comment
+ pointing to the new test.
+
+2011-08-02 Stefano Lattarini <address@hidden>
+
+ docs: improve, extend and fix documentation on TAP support
+ * doc/automake.texi ("Using the TAP test protocol"): Divide this
+ section into ...
+ ("Introduction to TAP", "Use TAP with the Automake test harness",
+ "Incompatibilities with other TAP parsers and drivers", "Links
+ and external resources"): ... these subsections, extend them by
+ adding more information and examples, and improve them by removing
+ incomplete and/or temporary wordings and TODO items.
+ ("Script-based Testsuites", "Parallel Test Harness"): Add a couple
+ of anchors to improve the granularity of cross-references.
+ * tests/tap-doc2.test: New test, verifying the correctness of the
+ new examples given in the manual.
+ * tests/Makefile.am (tap_other_tests): Add the new test.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ testsuite: separate the only failing check of an xfailing test
+ * tests/plan-bad-prog.test: Move the only failing check of this
+ test (i.e., the one about the total number of "ERROR" outcomes)
+ into ...
+ * tests/plan-bad-prog2.test: ... this new test.
+ * tests/Makefile.am (XFAIL_TESTS): Remove `plan-bad-prog.test',
+ add `plan-bad-prog2.test'.
+ (tap_other_tests): Add `plan-bad-prog2.test'.
+ (plan-bad-prog2.log): Depend on `plan-bad-prog.test'.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ testsuite: fix spurious errors in an xfailing test on TAP support
+ * tests/plan-bad-prog.test: Fix typo in the name of the test
+ being made unreadable. Correct the wording of the potential
+ skip message, and simplify the condition under which the test
+ is to be skipped. Escape literal dots in grep regexps.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ testsuite: better granularity in a couple of tests on TAP support
+ * tests/tap-summary-aux.sh: New auxiliary script, filled with code
+ moved out from ...
+ * tests/tap-summary.test: ... this test, from which the checks on
+ colored testsuite have further been moved into ...
+ * tests/tap-summary-color.test: ... this new test, which in turn
+ * tests/Makefile.am (tap_other_tests): Add `tap-summary-color.test'.
+ (EXTRA_DIST): Distribute `tap-summary-aux.sh'
+ (tap-summary.log): Depend on `tap-summary-aux.sh'.
+ (tap-summary-color.log): Likewise.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ testsuite: optimize tests on TAP for speed and against duplication
+ The creation and configuration of common files and data used by
+ many tests on TAP is, with this change, factored out into a new
+ dedicated auxiliary test, from which those tests will depend upon.
+ This reduces code duplication in tests and, more importantly,
+ offers a noticeable speedup in the involved tests (30-40%).
+ * tests/tap-common-setup.test: New test, setting up the common
+ files and data used by various tests on TAP support.
+ * tests/tap-setup.sh: New file, to be sourced by tests wanting to
+ bring in data generated by `tap-common-setup.test'.
+ * tests/tap-autonumber.test: Update to use the pre-computed data
+ files.
+ * tests/tap-bailout.test: Likewise.
+ * tests/tap-color.test: Likewise.
+ * tests/tap-deps.test: Likewise.
+ * tests/tap-diagnostic.test: Likewise.
+ * tests/tap-empty-diagnostic.test: Likewise.
+ * tests/tap-empty.test: Likewise.
+ * tests/tap-escape-directive.test: Likewise.
+ * tests/tap-exit.test: Likewise.
+ * tests/tap-signal.test: Likewise.
+ * tests/tap-fancy.test: Likewise.
+ * tests/tap-fancy2.test: Likewise.
+ * tests/tap-global-log.test: Likewise.
+ * tests/tap-global-result.test: Likewise.
+ * tests/tap-html.test: Likewise.
+ * tests/tap-log.test: Likewise.
+ * tests/tap-merge-stdout-stderr.test: Likewise.
+ * tests/tap-no-merge-stdout-stderr.test: Likewise.
+ * tests/tap-message-0.test: Likewise.
+ * tests/tap-no-disable-hard-error.test: Likewise.
+ * tests/tap-no-spurious-summary.test: Likewise.
+ * tests/tap-no-spurious.test: Likewise.
+ * tests/tap-not-ok-skip.test: Likewise.
+ * tests/tap-numeric-description.test: Likewise.
+ * tests/tap-out-of-order.test: Likewise.
+ * tests/tap-passthrough.test: Likewise.
+ * tests/tap-passthrough-exit.test: Likewise.
+ * tests/tap-plan.test: Likewise.
+ * tests/tap-plan-corner.test: Likewise.
+ * tests/tap-plan-corner2.test: Likewise.
+ * tests/tap-plan-errors.test: Likewise.
+ * tests/tap-realtime.test: Likewise.
+ * tests/tap-recheck-logs.test: Likewise.
+ * tests/tap-skip-whole.test: Likewise.
+ * tests/tap-todo-skip-together.test: Likewise.
+ * tests/tap-todo-skip-whitespace.test: Likewise.
+ * tests/tap-todo-skip.test: Likewise.
+ * tests/tap-unplanned.test: Likewise.
+ * tests/tap-with-and-without-number.test: Likewise.
+ * tests/tap-xfail-tests.test: Likewise.
+ * tests/tap-skip-whole-whitespace.test: Likewise, and remove
+ redundant definitions of `$sp' and `$tab' (they are already
+ defined in `tests/defs').
+ * tests/tap-whitespace-normalization.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+ (EXTRA_DIST): Distribute `tap-setup.sh'.
+ (tap_with_common_setup_tests, tap_with_common_setup_logs)): New
+ variables, holding respectively the list of tests using the files
+ pre-computed by `tap-common-setup.test', and the list of their
+ corresponding log files.
+ (tap_other_tests): New variable, holding the list of other tests
+ on TAP support.
+ ($(tap_with_common_setup_logs)): Depend on `tap-common-setup.log'
+ and `tap-setup.sh'.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ test harness: use new `.trs' files to hold test metadata
+ With this change, the test harness will keep test metadata in
+ dedicated `.trs' files, instead of having them embedded into the
+ `.log' files. This allows for easier forward-compatibility and
+ extension of test metadata, and for more flexibility in the
+ format of the `.log' files. Note that this change makes the
+ `:end-metadata:' field obsolete.
+ * doc/automake.texi (Parallel Test Harness, Log files generation
+ and test results recording): Document the new APIs and behaviour;
+ some related minor rewordings and fixlets.
+ * NEWS: Update.
+ * automake.in (handle_tests): When bringing in the content of
+ `check2.am', substitute %BASE% with the basename of the `.log'
+ file being created by a rule. Add the generated `.trs' files
+ to the list of files to be cleaned by "make mostlyclean".
+ * lib/am/check.am (am__test_driver_flags): Rename ...
+ (am__common_driver_flags): ... to this, and remove the flags
+ `--test-name' and `--log-file' from it: they are now define in
+ the proper rules in `check2.am'.
+ (am__TEST_BASES): New internal variable, holding the names of
+ the tests, with any registered extension removed.
+ (am__stealth_MAKE): New internal variable, can be used instead of
+ $(MAKE) in recipes requiring a recursive call to make, but which
+ are not intended to be executed by "make -n".
+ (.log.trs): New suffix rule, to recover from deletion of `.trs'
+ files.
+ ($(TEST_SUITE_LOG)): Almost completely rewritten to follow the
+ new API of "test logs in `.log' files, test metadata in `.trs'
+ files". It goes to some length to work correctly in face of
+ unreadable or missing `.log' and `.trs' files, and to error out
+ with proper error messages when this is not possible.
+ [%?PARALLEL_TESTS%] (check-TESTS): Also remove relevant "stale"
+ `.trs' files (in addition to `.log files) before remaking the
+ $(TEST_SUITE_LOG).
+ (recheck, recheck-html): Look for the `:recheck:' field in the
+ `.trs' files, not in the `.log' files.
+ * lib/am/check2.am (?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Adjust
+ the call to the test driver, in particularly passing the new
+ option `--trs-file'.
+ [%am__EXEEXT%] (?GENERIC?%EXT%$(EXEEXT).log): Likewise.
+ * lib/tap-driver ($USAGE): Adjust the help screen.
+ (Getopt::Long::GetOptions): Handle the `--trs-file' option,
+ through the use of ...
+ ($trs_file): ... this new global variable.
+ (finish): Write metadata for the test run to `$trs_file' rather
+ then to `$log_file', through the use of ...
+ (write_test_results): ... this new function.
+ * lib/test-driver (print_usage): Update the help screen.
+ (Option parsing): Handle the `--trs-file' option, through the
+ use of ...
+ ($resfile): ... this new global variable.
+ (Main code): Write metadata for the test run to `$trsfile' rather
+ than to `$logfile'.
+ Minor related adjustments to comments.
+ * tests/.gitignore: Ignore `*.trs' files.
+ * tests/parallel-tests-unreadable-log.test: Moved ...
+ * tests/parallel-tests-unreadable.test: ... to this, and extended
+ to also check the semantics for unreadable `.trs' files.
+ * tests/test-driver-end-metadata.test: Deleted as obsolete.
+ * tests/test-driver-metadata-no-leading-space.test: Likewise.
+ * tests/test-driver-global-log.test: Renamed ...
+ * tests/test-metadata-global-log.test: ... to this, and modified
+ as to verify the new APIs and semantics.
+ * tests/test-driver-recheck: Renamed ...
+ * tests/test-metadata-recheck.test: ... to this, and modified
+ likewise.
+ * tests/parallel-tests-once.test: New test.
+ * tests/parallel-tests-make-n.test: Likewise.
+ * test-metadata-results.test: Likewise.
+ * test-missing.test: Likewise.
+ * test-missing2.test: Likewise.
+ * test-trs-basic.test: Likewise.
+ * test-trs-recover.test: Likewise.
+ * test-trs-recover2.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-27 Stefano Lattarini <address@hidden>
+
+ test harness: allow more metadata in log files
+ This change reworks and improves the parallel test harness to use
+ more specialized reStructuredText fields in the log files (instead
+ of relying on specially-placed of "magic lines" and more indirect
+ semantical formatting); the new fields are the following:
+ - ":recheck:": tell whether the associated test will have to be
+ re-run by "make recheck";
+ - ":copy-in-global-log:": tell whether the content of the log
+ file should be copied in the "global log" `test-suite.log';
+ - ":end-metadata:", which inhibits the scanning of the rest of
+ the log file (for what concerns test metadata).
+ Also, the special `:test-result:' value "END" has been removed,
+ superseded by the new `:end-metadata:' field.
+ * doc/automake.texi (Log files generation and test results
+ recording): Document the new API and semantics. Remove or fix
+ some obsolete comments.
+ * lib/am/check.am ($(TEST_SUITE_LOG), recheck, recheck-html):
+ Adjust comments and code.
+ * lib/tap-driver (finish): Adjust, with the help of ...
+ (must_recheck, copy_in_global_log): ... these new functions.
+ * lib/test-driver (Main code): Adjust, with the help of ...
+ ($recheck, $gcopy): ... these new variables.
+ * tests/trivial-test-driver: Update to obey the new APIs.
+ * tests/test-driver-recheck.test: Likewise.
+ * tests/test-driver-global-log.test: Likewise.
+ * tests/tap-passthrough.test: Relax the test, avoiding to check
+ what is written in `test-suite.log'; such check has been moved ...
+ * tests/tap-global-log.test: ... in this new test, and extended.
+ * tests/test-driver-metadata-no-leading-space.test: New test.
+ * tests/test-driver-end-test-results.test: Removed, it checked
+ the old APIs; superseded by ...
+ * tests/test-driver-end-metadata.test: ... this new test.
+ * tests/tap-log.test: Improve syncing with ...
+ * tests/test-log.test: ... this new test.
+ * tests/parallel-tests.test: Remove some duplication w.r.t. this
+ last new test. Updated heading comments.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-27 Stefano Lattarini <address@hidden>
+
+ testsuite: fix a spurious failure with non-bash shells
+ * tests/tap-fancy2.test: Remove an unportable use of backslashes
+ with the `echo' builtin, which was causing some shells (among them
+ bash and Debian /bin/ksh) to print a `\\' string, while other
+ shells (among them zsh, dash, and Solaris /bin/sh and /bin/ksh)
+ were unexpectedly printing a single `\' character. Since we are
+ at it, add a sanity check to ensure that this issue does not
+ resurface.
+
+2011-07-27 Stefano Lattarini <address@hidden>
+
+ testsuite: fix spurious failures with Solaris /bin/sh
+ * tests/tap-more.test: Use `echo > file', not `: > file', to
+ create empty files in the "for" loops; this is required since,
+ as documented in the autoconf manual, Solaris 10 /bin/sh
+ "optimizes" away the `:' command after the first iteration,
+ even if it is redirected.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/tap-numeric-description.test: Partial rewrite to avoid
+ using positional parameters from the 10th onward, which are
+ unportable to Solaris /bin/sh (using `${10}' causes the shell
+ to die with "bad substitution").
+
+2011-07-31 Stefano Lattarini <address@hidden>
+
+ simple tests: support developer-defined fd redirections
+ Motivated by coreutils bug#8846, and related discussions:
+ <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8846>
+ <http://lists.gnu.org/archive/html/bug-autoconf/2011-06/msg00002.html>
+ In those threads it has been shown how problematic it can be to do
+ portable file descriptor redirections in the testsuite when using
+ the Automake testsuite harness. This change should remedy to that
+ situation.
+ * lib/am/check2.am (?GENERIC?%EXT%$(EXEEXT).log,
+ ?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Append redirection defined
+ in $(AM_TESTS_FD_REDIRECT) to the command-line invocations of the
+ test scripts.
+ * lib/am/check.am [!%?PARALLEL_TESTS%] $(check-TESTS): Likewise.
+ * NEWS: Update.
+ * doc/automake.texi (Script-based Testsuites): Document the new
+ feature.
+ * tests/check-fd-redirect.test: New test.
+ * tests/parallel-tests-fd-redirect.test: Likewise.
+ * tests/parallel-tests-am_tests_environment.test: Remove checks
+ about the use of redirections in AM_TESTS_ENVIRONMENT: they would
+ check deprecated (if not undefined) behaviour now. Strengthen a
+ couple of still valid checks, to keep the test more in sync with
+ the documentation. Improve debugging information.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-22 Stefano Lattarini <address@hidden>
+
+ docs: rework and extend documentation on testsuites support
+ * doc/automake.texi (Generalities about Testing): New section.
+ (Simple Tests using parallel-tests): Section removed, merged
+ into ...
+ (Simple Tests): ... this one, which on the other hand has been
+ subdivided into ...
+ (Script-based Testsuites, Serial Test Harness, Parallel Test
+ Harness): ... these new subsection.
+ (DejaGnu Tests): Minor adjustments.
+ Other related typofixes and rewordings throughout the manual;
+ in particular, avoid to use the term "test driver" for three
+ different concepts (and use instead "test harness" and "test
+ runner" where appropriate).
+ * tests/tap-doc.test: New test.
+ * tests/tap-no-disable-hard-error.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-18 Stefano Lattarini <address@hidden>
+
+ tap: support colorization of testsuite progress output
+ * lib/tap-driver (%COLORS): New variable (definition extracted
+ from `lib/am/check.am:$(am__tty_colors)', with some obvious
+ adjustments.
+ (report): Adjust to colorize console output when required,
+ using ...
+ (decorate_result): ... this new function.
+ (colored): New function, used by the one above.
+ * tests/tap-summary.test: Also run the checks when `color-tests'
+ is in use.
+ * tests/Makefile.am (XFAIL_TESTS): Remove `tap-color.test'.
+
+2011-07-18 Stefano Lattarini <address@hidden>
+
+ tap: some preparatory refactoring (2)
+ This is a follow-up simplification.
+ * lib/tap-driver (console_output): Renamed ...
+ (report): ... to this, and extended to appropriately register
+ the test results when needed.
+ (testsuite_error, handle_tap_comment, handle_tap_test,
+ handle_tap_plan): Adjusted accordingly.
+
+2011-07-18 Stefano Lattarini <address@hidden>
+
+ tap: some preparatory refactoring (1)
+ This refactoring is aimed at simplifying the introduction of
+ colored console output for the TAP driver.
+ * lib/tap-driver (console_output): Now accept two arguments, the
+ first one indicating which kind of thing is to be displayed (for
+ now only a test result or a diagnostic comment), and the second
+ one (if present) the message associated to it.
+ (handle_tap_test, handle_tap_comment, handle_tap_plan,
+ testsuite_error): Adapt to the new `console_output' interface.
+
+2011-07-18 Stefano Lattarini <address@hidden>
+
+ tap: add experimental TAP-aware driver
+ * doc/automake.texi (Using the TAP test protocol): New section.
+ (Overview of Custom Test Drivers Support): Minor updates.
+ * lib/tap-driver: New script, TAP-aware test driver for Automake;
+ implemented in perl and based on TAP::Parser.
+ * lib/Makefile.am (dist_script_DATA): Add it.
+ * tests/tap-autonumber.test: New test.
+ * tests/tap-bailout.test: Likewise.
+ * tests/tap-basic.test: Likewise.
+ * tests/tap-deps.test: Likewise.
+ * tests/tap-diagnostic.test: Likewise.
+ * tests/tap-empty.test: Likewise.
+ * tests/tap-empty-diagnostic.test: Likewise.
+ * tests/tap-escape-directive.test: Likewise.
+ * tests/tap-exit.test: Likewise.
+ * tests/tap-fancy.test: Likewise.
+ * tests/tap-fancy2.test: Likewise.
+ * tests/tap-global-result.test: Likewise.
+ * tests/tap-html.test: Likewise.
+ * tests/tap-log.test: Likewise.
+ * tests/tap-merge-stdout-stderr.test: Likewise.
+ * tests/tap-more.test: Likewise.
+ * tests/tap-more2.test: Likewise.
+ * tests/tap-no-merge-stdout-stderr.test: Likewise.
+ * tests/tap-no-spurious-summary.test: Likewise.
+ * tests/tap-no-spurious.test: Likewise.
+ * tests/tap-not-ok-skip.test: Likewise.
+ * tests/tap-numeric-description.test: Likewise.
+ * tests/tap-out-of-order.test: Likewise.
+ * tests/tap-passthrough.test: Likewise.
+ * tests/tap-plan.test: Likewise.
+ * tests/tap-plan-errors.test: Likewise.
+ * tests/tap-plan-corner.test: Likewise.
+ * tests/tap-realtime.test: Likewise.
+ * tests/tap-recheck-logs.test: Likewise.
+ * tests/tap-recheck.test: Likewise.
+ * tests/tap-skip-whole.test: Likewise.
+ * tests/tap-summary.test: Likewise.
+ * tests/tap-todo-skip.test: Likewise.
+ * tests/tap-todo-skip-together.test: Likewise.
+ * tests/tap-todo-skip-whitespace.test: Likewise.
+ * tests/tap-skipall-whitespace.test: Likewise.
+ * tests/tap-unplanned.test: Likewise.
+ * tests/tap-whitespace-normalization.test: Likewise.
+ * tests/tap-with-and-without-number.test: Likewise.
+ * tests/tap-xfail-tests.test: Likewise.
+ * tests/tap-bad-prog.test: New xfailing test.
+ * tests/tap-color.test: Likewise.
+ * tests/tap-plan-corner2.test: Likewise.
+ * tests/tap-message-0.test: Likewise.
+ * tests/tap-signal.test: Likewise.
+ * tests/Makefile.am (TESTS, XFAIL_TESTS): Update.
+
+2011-07-07 Stefano Lattarini <address@hidden>
+
+ test defs: new auxiliary function 'count_test_results'
+ * tests/defs (count_test_results): New function.
+ * tests/check11.test: Use it.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-no-spurious-summary.test: Likewise, and
+ slightly improve debugging output.
+ * tests/parallel-tests.test: Make use of `count_test_results'.
+ Also, make grepping of "make check" output slightly stricter
+ * tests/parallel-tests9.test: Likewise.
+ * tests/parallel-tests-log-override-2.test: Likewise, and throw
+ in a small optimization.
+
+2011-07-07 Stefano Lattarini <address@hidden>
+
+ parallel-tests: simplify testsuite summary
+ Prefer a more deterministic, "tabular" format for the testsuite
+ summary, always listing the numbers of passed, failed, xfailed,
+ xpassed, skipped and errored tests, even when these numbers are
+ zero. This simplify the logic of testsuite summary creation,
+ makes it more easily machine-parseable, and will probably allow
+ for easier addition of new kinds of test results in the future.
+ * lib/am/check.am (am__tty_colors_dummy): New make variable, to
+ reduce code duplication. Extracted from previous versions of
+ $(am__tty_colors), and extended by defining two new variables
+ `$mgn' and `$brg'.
+ [%?COLOR%, %!?COLOR%] (am__tty_colors): Use that new variable.
+ (am__text_box): Delete, is not needed anymore.
+ ($(TEST_SUITE_LOG)): Rewrite associated rules to implement the
+ new testsuite summary format.
+ * NEWS: Update.
+ * tests/check10.test: Don't run with the parallel-tests harness
+ too, that makes no sense anymore.
+ * tests/color.test: Update and adjust.
+ * tests/color2.test: Likewise.
+ * tests/parallel-tests.test: Likewise.
+ * tests/parallel-tests3.test: Likewise.
+ * tests/parallel-tests6.test: Likewise.
+ * tests/parallel-tests9.test: Likewise.
+ * tests/parallel-tests-unreadable-log.test: Likewise.
+ * tests/parallel-tests-empty-testlogs.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-no-spurious-summary.test: Likewise.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-end-test-results.test: Likewise.
+ * tests/parallel-tests-no-color-in-log.test: New test.
+ * tests/testsuite-summary-color.test: Likewise.
+ * tests/testsuite-summary-count.test: Likewise.
+ * tests/testsuite-summary-count-many.test: Likewise.
+ * tests/testsuite-summary-reference-log.test: Likewise.
+ * tests/testsuite-summary-checks.sh: New auxiliary script, used
+ by the new tests above.
+ * tests/extract-testsuite-summary: Likewise.
+ * tests/trivial-test-driver: Optimize for speed when there are
+ lots of of tests.
+ * tests/Makefile.am (EXTRA_DIST): Distribute them.
+ (testsuite-summary-color.log, testsuite-summary-count.log): Depend
+ on them.
+ (testsuite-summary-count-many.log): Depend on the auxiliary scripts
+ 'trivial-test-driver' and 'extract-testsuite-summary'.
+ (TESTS): Update.
+
+2011-07-07 Stefano Lattarini <address@hidden>
+
+ parallel-tests: new recognized test result 'ERROR'
+ * lib/am/check.am ($(TEST_SUITE_LOG)): Recognize a new test result
+ `ERROR'. Use it when encountering unreadable test logs (previously
+ a simple `FAIL' was used in this situations).
+ * lib/test-driver: Set the global test result to `ERROR' when the
+ test exit status is 99. When doing colorized output, color `ERROR'
+ results in magenta.
+ * doc/automake.texi (Log files generation and test results
+ recording): Update by listing `ERROR' too among the list of valid
+ `:test-results:' arguments.
+ * NEWS: Update.
+ * tests/trivial-test-driver: Update.
+ * tests/parallel-tests.test: Likewise.
+ * tests/parallel-tests-harderror.test: Likewise.
+ * tests/parallel-tests-no-spurious-summary.test: Likewise.
+ * tests/test-driver-global-log.test: Likewise.
+ * tests/test-driver-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-custom-no-html.test: Likewise.
+ * tests/test-driver-end-test-results.test: Likewise.
+ * tests/color.test: Likewise. Also, make stricter, and also test
+ from VPATH.
+ * tests/color2.test: Likewise, and improve syncing with color.test.
+ * tests/parallel-tests-exit-statuses.test: New test.
+ * tests/parallel-tests-console-output.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-07 Stefano Lattarini <address@hidden>
+
+ parallel-tests: make parsing of test results safer
+ The new code for parsing the testsuite-generated `.log' files,
+ as introduced in commit `v1.11-872-gc96b881', considers each
+ `:test-result:' field anywhere in a `.log' file as a declaration
+ of a test result, and accounts for it as such in the testsuite
+ summary. Unfortunately this could easily cause spurious test
+ failures being reported in the testsuite summary. This happened
+ in practice with the Automake's own testsuite; for example:
+ $ make check TESTS='check12-p.test'; echo exit: $?
+ ...
+ PASS: check12-p.test
+ =====================================
+ 4 of 5 tests failed
+ See tests/test-suite.log
+ Please report to address@hidden
+ =====================================
+ make[2]: *** [test-suite.log] Error 1
+ make: *** [check-am] Error 2
+ exit: 2
+ This change introduces a new special `:test-result:' "END", that,
+ when seen, prevents the rest of the log file from being parsed.
+ For more information, refer to the thread:
+
<http://lists.gnu.org/archive/html/automake-patches/2011-06/msg00199.html>
+ * lib/am/check.am ($(TEST_SUITE_LOG)): Stop the parsing of a log
+ file as soon as the special ":test-result:END" directive is seen.
+ Related changes and enhancements.
+ * lib/test-driver: Protect the rest of the log after the result
+ lined with a ":test-result:END" directive.
+ * doc/automake.texi (Log files generation and test results
+ recording): Update, and related improvements.
+ * tests/parallel-tests-no-spurious-summary.test: New test.
+ * tests/test-driver-end-test-results.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
2011-06-29 Stefano Lattarini <address@hidden>
docs: document custom test drivers and protocols
@@ -39,14 +815,14 @@
example, simpler this time, but still inspired to real-world
usage (the GNU coreutils testsuite).
-2011-06-23 Stefano Lattarini <address@hidden>
+2011-06-23 Stefano Lattarini <address@hidden>
docs: avoid a footnote, some related rewordings and improvements
* doc/automake.texi (Dist): Reword the part about automatically
distributed files to avoid a footnote. Since we are at it, extend
a bit, and add an example and a reference to a relevant test case.
-2011-06-23 Stefano Lattarini <address@hidden>
+2011-06-23 Stefano Lattarini <address@hidden>
docs: minor cosmetic fixes
* doc/automake.texi: Break few overly long lines, throughout the
@@ -59,7 +835,7 @@
("Other things Automake recognizes" @item AM_C_PROTOTYPES): Use
@pxref instead of @ref.
-2011-06-23 Stefano Lattarini <address@hidden>
+2011-06-23 Stefano Lattarini <address@hidden>
help: improve text about automatically-distributed files
This change fixes automake bug#7819.
@@ -72,7 +848,7 @@
* tests/autodist.test: Likewise.
(configure.in): Remove useless call to AM_MAINTAINER_MODE.
-2011-06-23 Stefano Lattarini <address@hidden>
+2011-06-23 Stefano Lattarini <address@hidden>
refactor: split 'usage' subroutine in automake
This change is related to automake bug#7819.
diff --git a/HACKING b/HACKING
index dc727d3..231e3fc 100644
--- a/HACKING
+++ b/HACKING
@@ -20,6 +20,10 @@
* If somebody reports a new bug, mention his name in the ChangeLog entry
and in the test case you write. Put him into THANKS.
+* When documenting a non-trivial idiom or example in the manual, be
+ sure to add a test case for it, and to reference such test case from
+ a proper Texinfo comment.
+
* The correct response to most actual bugs is to write a new test case
which demonstrates the bug. Then fix the bug, re-run the test suite,
and check everything in.
@@ -243,7 +247,8 @@
-----
-Copyright (C) 2003, 2007, 2008, 2010 Free Software Foundation, Inc.
+Copyright (C) 2003, 2007, 2008, 2010, 2011 Free Software Foundation,
+Inc.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
diff --git a/INSTALL b/INSTALL
index 81fd332..a1e89e1 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,8 +1,8 @@
Installation Instructions
*************************
-Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
-2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
+Copyright (C) 1994-1996, 1999-2002, 2004-2011 Free Software Foundation,
+Inc.
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright
@@ -226,6 +226,11 @@ order to use an ANSI C compiler:
and if that doesn't work, install pre-built binaries of GCC for HP-UX.
+ HP-UX `make' updates targets which have the same time stamps as
+their prerequisites, which makes it generally unusable when shipped
+generated files such as `configure' are involved. Use GNU `make'
+instead.
+
On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot
parse its `<wchar.h>' header file. The option `-nodtk' can be used as
a workaround. If GNU CC is not installed, it is therefore recommended
diff --git a/Makefile.am b/Makefile.am
index 6f7554d..66c72ff 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -392,9 +392,9 @@ sc_perl_local_no_parens:
exit 1; \
fi
-## Allow only `local $_' in Automake.
+## Allow only few variables to be localized in Automake.
sc_perl_local:
- @if grep -v '^[ \t]*local \$$_;' $(srcdir)/automake.in | \
+ @if egrep -v '^[ \t]*local \$$[_~]( *=|;)' $(srcdir)/automake.in | \
grep '^[ \t]*local [^*]'; then \
echo "Please avoid \`local'." 1>&2; \
exit 1; \
@@ -522,17 +522,19 @@ sc_tests_required_after_defs:
## Overriding a Makefile macro on the command line is not portable when
## recursive targets are used. Better use an envvar. SHELL is an
-## exception, POSIX says it can't come from the environment. DESTDIR and
-## DISTCHECK_CONFIGURE_FLAGS and V are exceptions, too, as package authors
-## are urged not to initialize them anywhere.
+## exception, POSIX says it can't come from the environment. V, DESTDIR,
+## DISTCHECK_CONFIGURE_FLAGS and DISABLE_HARD_ERRORS are exceptions, too,
+## as package authors are urged not to initialize them anywhere.
sc_tests_overriding_macros_on_cmdline:
@if grep -E '\$$MAKE .*(SHELL=.*=|=.*SHELL=)' $(srcdir)/tests/*.test;
then \
echo 'Rewrite "$$MAKE foo=bar SHELL=$$SHELL" as "foo=bar $$MAKE -e
SHELL=$$SHELL"' 1>&2; \
echo ' in the above lines, it is more portable.' 1>&2; \
exit 1; \
fi
- @if sed -e 's/ DESTDIR=[^ ]*/ /' -e 's/ SHELL=[^ ]*/ /' \
- -e 's/ V=[^ ]*/ /' -e 's///' \
+## Also try to account for usages like "$MAKE || st=$?".
+ @if sed -e 's/ || .*//' -e 's/ && .*//' \
+ -e 's/ DESTDIR=[^ ]*/ /' -e 's/ SHELL=[^ ]*/ /' \
+ -e 's/ V=[^ ]*/ /' -e 's/ DISABLE_HARD_ERRORS=[^ ]*/ /' \
## DISTCHECK_CONFIGURE_FLAGS is allowed to contain whitespace in its
## definition, so the more complex substitutions below.
-e "s/ DISTCHECK_CONFIGURE_FLAGS='[^']*'/ /" \
diff --git a/Makefile.in b/Makefile.in
index a8c0c40..530f8cc 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1144,7 +1144,7 @@ sc_perl_local_no_parens:
fi
sc_perl_local:
- @if grep -v '^[ \t]*local \$$_;' $(srcdir)/automake.in | \
+ @if egrep -v '^[ \t]*local \$$[_~]( *=|;)' $(srcdir)/automake.in | \
grep '^[ \t]*local [^*]'; then \
echo "Please avoid \`local'." 1>&2; \
exit 1; \
@@ -1258,8 +1258,9 @@ sc_tests_overriding_macros_on_cmdline:
echo ' in the above lines, it is more portable.' 1>&2; \
exit 1; \
fi
- @if sed -e 's/ DESTDIR=[^ ]*/ /' -e 's/ SHELL=[^ ]*/ /' \
- -e 's/ V=[^ ]*/ /' -e 's///' \
+ @if sed -e 's/ || .*//' -e 's/ && .*//' \
+ -e 's/ DESTDIR=[^ ]*/ /' -e 's/ SHELL=[^ ]*/ /' \
+ -e 's/ V=[^ ]*/ /' -e 's/ DISABLE_HARD_ERRORS=[^ ]*/ /' \
-e "s/ DISTCHECK_CONFIGURE_FLAGS='[^']*'/ /" \
-e 's/ DISTCHECK_CONFIGURE_FLAGS="[^"]*"/ /' \
-e 's/ DISTCHECK_CONFIGURE_FLAGS=[^ ]/ /' \
diff --git a/NEWS b/NEWS
index 37af53a..1c2f102 100644
--- a/NEWS
+++ b/NEWS
@@ -13,6 +13,20 @@ New in 1.11a:
* Changes to Automake-generated testsuite harnesses:
+ - Test scripts that exit with status 99 to signal an "hard error" (e.g.,
+ and unexpected or internal error, or a failure to set up the test case
+ scenario) have their outcome reported as an 'ERROR' now. Previous
+ versions of automake reported such an outcome as a 'FAIL' (the only
+ difference with normal failures being that hard errors were counted
+ as failures even when the test originating them was listed in
+ XFAIL_TESTS).
+
+ - The testsuite summary displayed by the parallel-test harness has a
+ completely new format, that always list the numbers of passed, failed,
+ xfailed, xpassed, skipped and errored tests, even when these numbers
+ are zero (but using smart coloring when the color-tests option is in
+ effect).
+
- The default testsuite driver offered by the 'parallel-tests' option is
now implemented (partly at least) with the help of automake-provided
auxiliary scripts (e.g., `test-driver'), instead of relying entirely
@@ -58,6 +72,17 @@ New in 1.11a:
Consistently with the existing syntax, this can be done by defining
special makefile variables `LOG_DRIVER' and `<ext>_LOG_DRIVER'.
+ - A new developer-reserved variable `AM_TESTS_FD_REDIRECT' can be used
+ to redirect/define file descriptors used by the test scripts.
+
+ - The parallel-tests harness generates now, in addition the `.log' files
+ holding the output produced by the test scripts, a new set of `.trs'
+ files, holding "metadata" derived by the execution of the test scripts;
+ among such metadata are the outcomes of the test cases run by a script.
+
+ - Initial and still experimental support for the TAP test protocol is
+ now provided.
+
* WARNING: Future backward-incompatibilities!
- The Automake support for automatic de-ANSI-fication will be removed in
diff --git a/automake.in b/automake.in
index 2ad8dc2..ad58f06 100644
--- a/automake.in
+++ b/automake.in
@@ -5033,6 +5033,7 @@ sub handle_tests
return substr ($obj, 0, length ($obj) - length
($test_suffix)) . '.log'
if substr ($obj, - length ($test_suffix)) eq $test_suffix;
}
+ my $base = $obj;
$obj .= '.log';
# The "test driver" program, deputed to handle tests protocol
used by
# test scripts. By default, it's assumed that no protocol is
used,
@@ -5054,6 +5055,7 @@ sub handle_tests
$output_rules .= file_contents ('check2', new
Automake::Location,
GENERIC => 0,
OBJ => $obj,
+ BASE => $base,
SOURCE => $val,
DRIVER => $driver,
DRIVER_FLAGS => $driver_flags,
@@ -5107,6 +5109,7 @@ sub handle_tests
$output_rules .= file_contents ('check2', new
Automake::Location,
GENERIC => 1,
OBJ => '',
+ BASE => '$*',
SOURCE => '$<',
DRIVER => $driver,
DRIVER_FLAGS => $driver_flags,
@@ -5122,6 +5125,7 @@ sub handle_tests
$clean_files{'$(TEST_LOGS_TMP)'} = MOSTLY_CLEAN;
$clean_files{'$(TEST_LOGS)'} = MOSTLY_CLEAN;
+ $clean_files{'$(am__TEST_BASES:=.trs)'} = MOSTLY_CLEAN;
$clean_files{'$(TEST_SUITE_LOG)'} = MOSTLY_CLEAN;
$clean_files{'$(TEST_SUITE_HTML)'} = MOSTLY_CLEAN;
}
diff --git a/doc/automake.texi b/doc/automake.texi
index a26f2a5..5eb40d5 100644
--- a/doc/automake.texi
+++ b/doc/automake.texi
@@ -317,12 +317,26 @@ What Goes in a Distribution
Support for test suites
-* Simple Tests:: Listing programs and scripts in @code{TESTS}
-* Simple Tests using parallel-tests:: More powerful test driver
+* Generalities about Testing:: Generic concepts and terminology about testing
+* Simple Tests:: Listing test scripts in @code{TESTS}
* Custom Test Drivers:: Writing and using custom test drivers
-* DejaGnu Tests:: Interfacing with the external testing framework
+* Using the TAP test protocol:: Integrating test scripts that use the TAP
protocol
+* DejaGnu Tests:: Interfacing with the @command{dejagnu} testing
framework
* Install Tests:: Running tests on installed packages
+Simple Tests
+
+* Script-based Testsuites:: Automake-specific concepts and terminology
+* Serial Test Harness:: Older (and obsolescent) serial test harness
+* Parallel Test Harness:: Generic concurrent test harness
+
+Using the TAP test protocol
+
+* Introduction to TAP::
+* Use TAP with the Automake test harness::
+* Incompatibilities with other TAP parsers and drivers::
+* Links and external resources::
+
Custom Test Drivers
* Overview of Custom Test Drivers Support::
@@ -923,6 +937,7 @@ GNU Make Manual}).
VPATH builds have other interesting uses. One is to build the same
sources with multiple configurations. For instance:
address@hidden Keep in sync with amhello-cflags.test.
@example
~ % @kbd{tar zxf ~/amhello-1.0.tar.gz}
~ % @kbd{cd amhello-1.0}
@@ -1074,6 +1089,7 @@ instance my cross-compiler for MinGW32 has its binaries
called
Here is how we could build @code{amhello-1.0} for
@code{i586-mingw32msvc} on a GNU/Linux PC.
address@hidden Keep in sync with amhello-cross-compile.test.
@smallexample
~/amhello-1.0 % @kbd{./configure --build i686-pc-linux-gnu --host
i586-mingw32msvc}
checking for a BSD-compatible install... /usr/bin/install -c
@@ -1188,6 +1204,7 @@ by some means.
For instance here is how we could create a binary package containing a
snapshot of all the files to be installed.
address@hidden Keep in sync with amhello-binpkg.test.
@example
~/amhello-1.0 % @kbd{./configure --prefix /usr}
@dots{}
@@ -1867,6 +1884,7 @@ When examining a variable definition, Automake will
recursively examine
variables referenced in the definition. For example, if Automake is
looking at the content of @code{foo_SOURCES} in this snippet
address@hidden Keep in sync with interp.test.
@example
xs = a.c b.c
foo_SOURCES = c.c $(xs)
@@ -2047,6 +2065,7 @@ defined (e.g., @samp{zardir}).
For instance, the following snippet will install @file{file.xml} into
@samp{$(datadir)/xml}.
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@example
xmldir = $(datadir)/xml
xml_DATA = file.xml
@@ -2057,6 +2076,7 @@ performs to diagnose suspicious directory/primary couples
(in the
unlikely case these checks are undesirable, and you really know what
you're doing). For example, Automake would error out on this input:
address@hidden Should be tested in primary-prefix-invalid-couples.test.
@example
# Forbidden directory combinations, automake will error out on this.
pkglib_PROGRAMS = foo
@@ -2066,6 +2086,7 @@ doc_LIBRARIES = libquux.a
@noindent
but it will succeed with this:
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@example
# Work around forbidden directory combinations. Do not use this
# without a very good reason!
@@ -2146,6 +2167,7 @@ data_DATA = file1 @dots{} address@hidden address@hidden
@dots{} address@hidden
@noindent
may also be written as
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@example
data_DATA = file1 @dots{} address@hidden
data2dir = $(datadir)
@@ -2324,8 +2346,8 @@ multilibs (@pxref{Multilibs}). This file is maintained
in the GCC
tree at @url{http://gcc.gnu.org/svn.html}.
@item test-driver
-This implements the default testsuite driver offered by the
address@hidden testsuite harness.
+This implements the default test driver offered by the parallel
+testsuite harness.
@item texinfo.tex
Not a program, this file is required for @samp{make dvi}, @samp{make
@@ -2488,6 +2510,7 @@ example in the manual. However if you were to build
@code{true} and
@code{false} in real life, you would probably use per-program
compilation flags, like so:
address@hidden Keep in sync with specflg7.test and specflg8.test.
@example
bin_PROGRAMS = false true
@@ -2811,6 +2834,7 @@ literals. If part of the specification uses shell
variables,
@command{automake} will not be able to fulfill this setup, and you will
have to complete the missing bits by hand. For instance, on
address@hidden Keep in sync with output11.test.
@example
file=input
@dots{}
@@ -2826,6 +2850,7 @@ source file.)
Similarly
address@hidden Keep in sync with output11.test.
@example
file=output
file2=out:in
@@ -3465,6 +3490,7 @@ installs this macro so that @command{aclocal} will find
it.
A macro file's name should end in @file{.m4}. Such files should be
installed in @file{$(datadir)/aclocal}. This is as simple as writing:
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@example
aclocaldir = $(datadir)/aclocal
aclocal_DATA = mymacro.m4 myothermacro.m4
@@ -4305,9 +4331,7 @@ does not know the possible values of these variables. In
this case
@cindex @code{SUBDIRS} and @code{AM_CONDITIONAL}
@cindex @code{AM_CONDITIONAL} and @code{SUBDIRS}
address@hidden The test case for the setup described here is
address@hidden test/subdircond2.test
address@hidden Try to keep it in sync.
address@hidden Keep in sync with subcond2.test.
@file{configure} should output the @file{Makefile} for each directory
and define a condition into which @file{opt/} should be built.
@@ -4347,9 +4371,7 @@ automatically because it knows that @code{MAYBE_OPT} can
contain
@cindex @code{SUBDIRS} and @code{AC_SUBST}
@cindex @code{AC_SUBST} and @code{SUBDIRS}
address@hidden The test case for the setup described here is
address@hidden test/subdircond3.test
address@hidden Try to keep it in sync.
address@hidden Keep in sync with subcond3.test.
Another possibility is to define @code{MAYBE_OPT} from
@file{./configure} using @code{AC_SUBST}:
@@ -4507,6 +4529,7 @@ often be replaced by several variables, one for each
destination
directory (@pxref{Uniform}). For instance, the last example could be
rewritten as follows:
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@example
imagesdir = $(pkgdatadir)/images
soundsdir = $(pkgdatadir)/sounds
@@ -4962,6 +4985,7 @@ You can also use Automake conditionals
(@pxref{Conditionals}) to
select programs to be built. In this case you don't have to worry
about @samp{$(EXEEXT)} or @code{EXTRA_PROGRAMS}.
address@hidden Keep in sync with exeext.test.
@example
bin_PROGRAMS = cpio pax
if WANT_MT
@@ -5012,6 +5036,7 @@ Extra objects can be added to a library using the
@address@hidden variable. This should be used for objects
determined by @command{configure}. Again from @code{cpio}:
address@hidden Keep in sync with pr401c.test.
@example
libcpio_a_LIBADD = $(LIBOBJS) $(ALLOCA)
@end example
@@ -5215,6 +5240,7 @@ relates to @file{libfoo.la} or @file{libbar.la} at the
time it creates
the link rule for these two libraries. Therefore the @option{-rpath}
argument must be explicitly supplied.
address@hidden Keep in sync with ltcond.test.
@example
EXTRA_LTLIBRARIES = libfoo.la libbar.la
lib_LTLIBRARIES = $(WANTEDLIBS)
@@ -5230,6 +5256,7 @@ Automake is able to compute the @option{-rpath} setting
itself, because
it's clear that both libraries will end up in @samp{$(libdir)} if they
are installed.
address@hidden Keep in sync with ltcond.test.
@example
lib_LTLIBRARIES =
if WANT_LIBFOO
@@ -5256,6 +5283,7 @@ we could build a @file{libhello.la} library using either
@file{hello-linux.c} or @file{hello-generic.c} with the following
@file{Makefile.am}.
address@hidden Keep in sync with ltcond2.test.
@example
lib_LTLIBRARIES = libhello.la
libhello_la_SOURCES = hello-common.c
@@ -5270,6 +5298,7 @@ either @file{hello-linux.lo} or @address@hidden
Or we could simply use an Automake conditional as follows.
address@hidden Keep in sync with ltcond2.test.
@example
lib_LTLIBRARIES = libhello.la
libhello_la_SOURCES = hello-common.c
@@ -5309,6 +5338,7 @@ dependency anywhere it won't be built (this is why
Here is a sample setup merging libtool convenience libraries from
subdirectories into one main @file{libtop.la} library.
address@hidden Keep in sync with ltconv.test.
@example
# -- Top-level Makefile.am --
SUBDIRS = sub1 sub2 @dots{}
@@ -6078,6 +6108,18 @@ the C preprocessor. In particular it generates
@samp{-I.},
@code{AM_CONFIG_HEADER}). You can disable the default @option{-I}
options using the @option{nostdinc} option.
+When a file to be included is generated during the build and not part
+of a distribution tarball, its location is under @code{$(builddir)},
+not under @code{$(srcdir)}. This matters especially for packages that
+use header files placed in sub-directories and want to allow builds
+outside the source tree (@pxref{VPATH Builds}). In that case we
+recommend to use a pair of @option{-I} options, such as, e.g.,
address@hidden/subdir -I$(srcdir)/some/subdir} or
address@hidden(top_builddir)/some/subdir -I$(top_srcdir)/some/subdir}.
+Note that the reference to the build tree should come before the
+reference to the source tree, so that accidentally leftover generated
+files in the source directory are ignored.
+
@code{AM_CPPFLAGS} is ignored in preference to a per-executable (or
per-library) @code{_CPPFLAGS} variable if it is defined.
@@ -6707,8 +6749,10 @@ is as follows:
@cindex Java support
@cindex Support for Java
-Automake includes support for compiled Java, using @command{gcj}, the Java
-front end to the GNU Compiler Collection.
+Automake includes support for natively compiled Java, using @command{gcj},
+the Java front end to the GNU Compiler Collection (preliminary support
+for compiling Java to bytecode using the @command{javac} compiler is
+also present; @pxref{Java}).
Any package including Java code to be compiled must define the output
variable @code{GCJ} in @file{configure.ac}; the variable @code{GCJFLAGS}
@@ -7533,6 +7577,7 @@ variable explicitly prevents byte-compilation.
Since Automake 1.8, we now recommend using @code{lisp_DATA} instead:
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@example
lisp_DATA = file1.el file2.el
@end example
@@ -7575,8 +7620,9 @@ libtool, The Libtool Manual}) with the @code{LTLIBRARIES}
primary.
@cindex @code{JAVA} primary, defined
@cindex Primary variable, @code{JAVA}
-Automake provides some minimal support for Java compilation with the
address@hidden primary.
+Automake provides some minimal support for Java bytecode compilation with
+the @code{JAVA} primary (in addition to the support for compiling Java to
+native machine code; @pxref{Java Support}).
Any @file{.java} files listed in a @code{_JAVA} variable will be
compiled with @code{JAVAC} at build time. By default, @file{.java}
@@ -7586,6 +7632,7 @@ files are not included in the distribution, you should
use the
Here is a typical setup for distributing @file{.java} files and
installing the @file{.class} files resulting from their compilation.
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@example
javadir = $(datadir)/java
dist_java_JAVA = a.java b.java @dots{}
@@ -7659,6 +7706,7 @@ Automake ships with an Autoconf macro called
@code{AM_PATH_PYTHON}
that will determine some Python-related directory variables (see
below). If you have called @code{AM_PATH_PYTHON} from
@file{configure.ac}, then you may use the variables
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@code{python_PYTHON} or @code{pkgpython_PYTHON} to list Python source
files in your @file{Makefile.am}, depending on where you want your files
installed (see the definitions of @code{pythondir} and
@@ -7747,6 +7795,7 @@ This is the directory where Python extension modules
(shared libraries)
should be installed. An extension module written in C could be declared
as follows to Automake:
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
@example
pyexec_LTLIBRARIES = quaternion.la
quaternion_la_SOURCES = quaternion.c support.c support.h
@@ -7943,6 +7992,7 @@ passed to @code{makeinfo} when building @file{.info}
files; and
@samp{$(AM_MAKEINFOHTMLFLAGS)} is used when building @file{.html}
files.
address@hidden Keep in sync with txinfo21.test.
For instance, the following setting can be used to obtain one single
@file{.html} file per manual, without node separators.
@example
@@ -8168,10 +8218,11 @@ Variables using the standard directory prefixes
@samp{bin},
For instance, @code{data_DATA} files are installed by @code{install-data},
while @code{bin_PROGRAMS} files are installed by @code{install-exec}.
-Any variable using a user-defined directory prefix with @samp{exec} in
-the name (e.g., @code{myexecbin_PROGRAMS}) is installed by
address@hidden All other user-defined prefixes are installed by
address@hidden
+Any variable using a user-defined directory prefix with
address@hidden in the name (e.g.,
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
address@hidden) is installed by @code{install-exec}. All
+other user-defined prefixes are installed by @code{install-data}.
@node Extending Installation
@section Extending Installation
@@ -8356,7 +8407,7 @@ included if they are found in the current directory
(either physically,
or as the target of a @file{Makefile.am} rule); this list is printed by
@samp{automake --help}. Note that some files in this list are actually
distributed only if other certain conditions hold (for example,
address@hidden The following example is covered by autodist-config-headers.test.
address@hidden Keep in sync with autodist-config-headers.test.
the @file{config.h.top} and @file{config.h.bot} files are automatically
distributed only if, e.g., @samp{AC_CONFIG_HEADERS([config.h])} is used
in @file{configure.ac}). Also, files that are read by @command{configure}
@@ -8546,6 +8597,7 @@ If you want @code{distcleancheck} to ignore built files
that have not
been cleaned because they are also part of the distribution, add the
following definition instead:
address@hidden Keep in sync with distcleancheck.test.
@example
distcleancheck_listfiles = \
find . -type f -exec sh -c 'test -f $(srcdir)/$$1 || echo $$1' \
@@ -8649,41 +8701,185 @@ default, only the @code{dist-gzip} target is hooked to
@code{dist}.
@cindex @code{make check}
@trindex check
-Automake supports three forms of test suites, the first two of which
-are very similar.
+Automake can generate code to handle two kinds of test suites. One is
+based on integration with the @command{dejagnu} framework. The other
+(and most used) form is based on the use of generic test scripts, and
+its activation is triggered by the definition of the special @code{TESTS}
+variable. This second form allows for various degrees of sophistication
+and customization; in particular, it allows for concurrent execution
+of test scripts, use of established test protocols such as TAP, and
+definition of custom test drivers and test runners.
+
address@hidden
+In either case, the testsuite is invoked via @samp{make check}.
@menu
-* Simple Tests:: Listing programs and scripts in @code{TESTS}
-* Simple Tests using parallel-tests:: More powerful test driver
+* Generalities about Testing:: Concepts and terminology about testing
+* Simple Tests:: Listing test scripts in @code{TESTS}
* Custom Test Drivers:: Writing and using custom test drivers
-* DejaGnu Tests:: Interfacing with the external testing framework
+* Using the TAP test protocol:: Integrating test scripts that use the TAP
protocol
+* DejaGnu Tests:: Interfacing with the @command{dejagnu} testing
framework
* Install Tests:: Running tests on installed packages
@end menu
address@hidden Generalities about Testing
address@hidden Generalities about Testing
+
+The purpose of testing is to determine whether a program or system behaves
+as expected (e.g., known inputs produce the expected outputs, error
+conditions are correctly handled or reported, and older bugs do not
+resurface).
+
address@hidden test case
+The minimal unit of testing is usually called @emph{test case}, or simply
address@hidden How a test case is defined or delimited, and even what
+exactly @emph{constitutes} a test case, depends heavily on the testing
+paradigm and/or framework in use, so we won't attempt any more precise
+definition. The set of the test cases for a given program or system
+constitutes its @emph{testsuite}.
+
address@hidden test harness
address@hidden testsuite harness
+A @emph{test harness} (also @emph{testsuite harness}) is a program or
+software component that executes all (or part of) the defined test cases,
+analyzes their outcomes, and report or register these outcomes
+appropriately. Again, the details of how this is accomplished (and how
+the developer and user can influence it or interface with it) varies
+wildly, and we'll attempt no precise definition.
+
address@hidden test pass
address@hidden test failure
+A test is said to @emph{pass} when it can determine that the condition or
+behaviour it means to verify holds, and is said to @emph{fail} when it can
+determine that such condition of behaviour does @emph{not} hold.
+
address@hidden test skip
+Sometimes, tests can rely on non-portable tools or prerequisites, or
+simply make no sense on a given system (for example, a test checking a
+Windows-specific feature makes no sense on a GNU/Linux system). In this
+case, accordingly to the definition above, the tests can neither be
+considered passed nor failed; instead, they are @emph{skipped} -- i.e.,
+they are not run, or their result is anyway ignored for what concerns
+the count of failures an successes. Skips are usually explicitly
+reported though, so that the user will be aware that not all of the
+testsuite has really run.
+
address@hidden xfail
address@hidden expected failure
address@hidden expected test failure
address@hidden xpass
address@hidden unexpected pass
address@hidden unexpected test pass
+It's not uncommon, especially during early development stages, that some
+tests fail for known reasons, and that the developer doesn't want to
+tackle these failures immediately (this is especially true when the
+failing tests deal with corner cases). In this situation, the better
+policy is to declare that each of those failures is an @emph{expected
+failure} (or @emph{xfail}). In case a test that is expected to fail ends
+up passing instead, many testing environments will flag the result as a
+special kind of failure called @emph{unexpected pass} (or @emph{xpass}).
+
address@hidden hard error
address@hidden Distinction between errors and failures in testsuites
+Many testing environments and frameworks distinguish between test failures
+and hard errors. As we've seen, a test failure happens when some invariant
+or expected behaviour of the software under test is not met. An @emph{hard
+error} happens when e.g., the set-up of a test case scenario fails, or when
+some other unexpected or highly undesirable condition is encountered (for
+example, the program under test experiences a segmentation fault).
+
address@hidden: Links to other test harnesses (esp. those sharing our
+terminology)?
+
@node Simple Tests
@section Simple Tests
-If the variable @code{TESTS} is defined, its value is taken to be a
-list of programs or scripts to run in order to do the testing.
-Programs needing data files should look for them in @code{srcdir}
-(which is both an environment variable and a make variable) so they
-work when building in a separate directory (@pxref{Build Directories,
-, Build Directories , autoconf, The Autoconf Manual}), and in
-particular for the @code{distcheck} rule (@pxref{Checking the
-Distribution}).
-
-For each of the @code{TESTS}, the result of execution is printed along
-with the test name, where @code{PASS} denotes a successful test,
address@hidden denotes a failed test, @code{XFAIL} an expected failure,
address@hidden an unexpected pass for a test that is supposed to fail,
-and @code{SKIP} denotes a skipped test.
address@hidden
+* Script-based Testsuites:: Automake-specific concepts and terminology
+* Serial Test Harness:: Older (and obsolescent) serial test harness
+* Parallel Test Harness:: Generic concurrent test harness
address@hidden menu
+
address@hidden Script-based Testsuites
address@hidden Script-based Testsuites
+
+If the special variable @code{TESTS} is defined, its value is taken to be
+a list of programs or scripts to run in order to do the testing. Under
+the appropriate circumstances, it's possible for @code{TESTS} to list
+also data files to be passed to one or more test scripts defined by
+different means (the so-called ``log compilers'', @pxref{Parallel Test
+Harness}).
+
+Test scripts can be executed serially or concurrently. Automake
+supports both these kinds of test execution, with the serial test harness
+being the default (for backward-compatibility reasons only, as its use
+is nowadays discouraged). The concurrent test harness relies on the
+concurrence capabilities (if any) offered by the underlying @command{make}
+implementation, and can thus only be as good as those are.
+
+By default, only the exit statuses of the test scripts are considered when
+determining the testsuite outcome. But Automake allows also the use of
+more complex test protocols, either standard (@pxref{Using the TAP test
+protocol}) or custom (@pxref{Custom Test Drivers}). Note that you can
+enable such protocols only when the parallel harness is used: they won't
+work with the serial test harness. In the rest of this section we are
+going to concentrate mostly on protocol-less tests, since we'll have later
+a whole section devoted to the use of test protocols (again, @pxref{Custom
+Test Drivers}).
@cindex Exit status 77, special interpretation
address@hidden Exit status 99, special interpretation
+When no test protocol is in use, an exit status of 0 from a test script will
+denote a success, an exit status of 77 a skipped test, an exit status of 99
+an hard error, and any other exit status will denote a failure.
-The number of failures will be printed at the end of the run. If a
-given test program exits with a status of 77, then its result is ignored
-in the final count. This feature allows non-portable tests to be
-ignored in environments where they don't make sense.
address@hidden Tests, expected failure
address@hidden Expected test failure
address@hidden XFAIL_TESTS
address@hidden DISABLE_HARD_ERRORS
address@hidden Disabling hard errors
+You may define the variable @code{XFAIL_TESTS} to a list of tests
+(usually a subset of @code{TESTS}) that are expected to fail; this will
+effectively reverse the result of those tests (with the provision that
+skips and hard errors remain untouched). You may also instruct the
+testsuite harness to treat hard errors like simple failures, by defining
+the @code{DISABLE_HARD_ERRORS} make variable to a nonempty value.
+
+Note however that, for tests based on more complex test protocols,
+the exact effects of @code{XFAIL_TESTS} and @code{DISABLE_HARD_ERRORS}
+might change, or they might even have no effect at all (for example,
address@hidden Keep this in sync with tap-no-disable-hard-errors.test.
+in tests using TAP, there is not way to disable hard errors, and the
address@hidden variable has no effect on them).
+
address@hidden progress on console}
address@hidden Testsuite progress on console
+The result of each test case run by the scripts in @code{TESTS} will be
+printed on standard output, along with the test name. For test protocols
+that allow more test cases per test script (such as TAP), a number,
+identifier and/or brief description specific for the single test case is
+expected to be printed in addition to the name of the test script. The
+possible results (whose meanings should be clear from the previous
address@hidden about Testing}) are @code{PASS}, @code{FAIL},
address@hidden, @code{XFAIL}, @code{XPASS} and @code{ERROR}. Here is an
+example of output from an hypothetical testsuite that uses both plain
+and TAP tests:
address@hidden Keep in sync with tap-doc.test.
address@hidden
+PASS: foo.sh
+PASS: zardoz.tap 1 - Daemon started
+PASS: zardoz.tap 2 - Daemon responding
+SKIP: zardoz.tap 3 - Daemon uses /proc # SKIP /proc is not mounted
+PASS: zardoz.tap 4 - Daemon stopped
+SKIP: bar.sh
+PASS: mu.tap 1
+XFAIL: mu.tap 2 # TODO frobnication not yet implemented
address@hidden example
+
address@hidden
+A testsuite summary (expected to report at least the number of run,
+skipped and failed tests) will be printed at the end of the testsuite
+run.
@anchor{Simple tests and color-tests}
@vindex AM_COLOR_TESTS
@@ -8693,52 +8889,77 @@ and standard output is connected to a capable terminal,
then the test
results and the summary are colored appropriately. The user can disable
colored output by setting the @command{make} variable
@samp{AM_COLOR_TESTS=no}, or force colored output even without a connecting
-terminal with @samp{AM_COLOR_TESTS=always}.
-
-Note that the semantics of some @command{make} implementations when used
-in parallel mode (@pxref{Parallel make,,, autoconf, The Autoconf Manual})
-can cause the automatic detection of a connection to a capable terminal
-to fail. In that case, you can still resort to the use of
address@hidden
+terminal with @samp{AM_COLOR_TESTS=always}. It's also worth noting that
+some @command{make} implementations, when used in parallel mode, have
+slightly different semantics (@pxref{Parallel make,,, autoconf,
+The Autoconf Manual}), which can break the automatic detection of a
+connection to a capable terminal. If this is the case, you'll have to
+resort to the use of @samp{AM_COLOR_TESTS=always} in order to have the
+testsuite output colorized.
+
+Test programs that need data files should look for them in @code{srcdir}
+(which is both a make variable and an environment variable made available
+to the tests), so that they work when building in a separate directory
+(@pxref{Build Directories, , Build Directories , autoconf,
+The Autoconf Manual}), and in particular for the @code{distcheck} rule
+(@pxref{Checking the Distribution}).
@vindex TESTS
@vindex TESTS_ENVIRONMENT
-The variable @code{TESTS_ENVIRONMENT} can be used to set environment
-variables for the test run; the environment variable @env{srcdir} is
-set in the rule. If all your test programs are scripts, you can also
-set @code{TESTS_ENVIRONMENT} to an invocation of the shell (e.g.
address@hidden(SHELL) -x} can be useful for debugging the tests), or any other
-interpreter. For instance, the following setup may be used to run tests
-with Perl:
address@hidden AM_TESTS_ENVIRONMENT
+The @code{AM_TESTS_ENVIRONMENT} and @code{TESTS_ENVIRONMENT} variables can
+be used to run initialization code and set environment variables for the
+test scripts. The former variable is developer-reserved, and can be
+defined in the @file{Makefile.am}, while the latter is reserved for the
+user, which can employ it to extend or override the settings in the
+former; for this to work portably, however, the contents of a non-empty
address@hidden @emph{must} be terminated by a semicolon.
address@hidden AM_TESTS_FD_REDIRECT
+The @code{AM_TESTS_FD_REDIRECT} variable can be used to define file
+descriptor redirections for the test scripts. One might think that
address@hidden could be used for this purpose, but experience
+has shown that doing so portably is practically impossible. The main
+hurdle is constituted by Korn shells, which usually set the close-on-exec
+flag on file descriptors opened with the @command{exec} builtin, thus
+rendering an idiom like @code{AM_TESTS_ENVIRONMENT = exec 9>&2;}
+ineffectual. This issue also affects some Bourne shells, such as the
+HP-UX's @command{/bin/sh},
address@hidden FIXME: should we offer a link to the relevant discussions on the
address@hidden bug-autoconf list?
+
address@hidden Keep in sync with tests-environment-backcompat.test.
@example
-TESTS_ENVIRONMENT = $(PERL) -Mstrict -w
-TESTS = foo.pl bar.pl baz.pl
+AM_TESTS_ENVIRONMENT = \
+## Some environment initializations are kept in a separate shell file
+## `tests-env.sh', which can make it easier to also run tests from
+## the command line.
+ . $(srcdir)/tests-env.sh; \
+## On Solaris, prefer more POSIX-compliant versions of the standard
+## tools by default.
+ if test -d /usr/xpg4/bin; then \
+ PATH=/usr/xpg4/bin:$$PATH; export PATH; \
+ fi;
address@hidden $$ restore font-lock
+## With this, the test scripts will be able to print diagnostic messages
+## to the original standard error stream, even if the test driver
+## redirects the stderr of the test scripts to a log file before executing
+## them.
+AM_TESTS_FD_REDIRECT = 9>&2
@end example
-It's important to note that this last use of @code{TESTS_ENVIRONMENT}
-is invalid when the @option{parallel-tests} option is active; however,
-the @option{parallel-tests} driver provides a more elegant way to
-achieve the same effect, with the further benefit of freeing the
address@hidden variable for the user
-(@pxref{Simple Tests using parallel-tests}).
-
-
address@hidden Tests, expected failure
address@hidden Expected test failure
-
address@hidden XFAIL_TESTS
-You may define the variable @code{XFAIL_TESTS} to a list of tests
-(usually a subset of @code{TESTS}) that are expected to fail. This will
-reverse the result of those tests.
address@hidden
+Note however that @code{AM_TESTS_ENVIRONMENT} is, for historical and
+implementation reasons, @emph{not} supported by the serial harness
+(@pxref{Serial Test Harness}).
Automake ensures that each file listed in @code{TESTS} is built before
-any tests are run; you can list both source and derived programs (or
-scripts) in @code{TESTS}; the generated rule will look both in
address@hidden and @file{.}. For instance, you might want to run a C
-program as a test. To do this you would list its name in @code{TESTS}
-and also in @code{check_PROGRAMS}, and then specify it as you would
-any other program.
+it is run; you can list both source and derived programs (or scripts)
+in @code{TESTS}; the generated rule will look both in @code{srcdir} and
address@hidden For instance, you might want to run a C program as a test.
+To do this you would list its name in @code{TESTS} and also in
address@hidden, and then specify it as you would any other
+program.
Programs listed in @code{check_PROGRAMS} (and @code{check_LIBRARIES},
@code{check_LTLIBRARIES}...) are only built during @code{make check},
@@ -8749,56 +8970,100 @@ that @code{check_PROGRAMS} are @emph{not}
automatically added to
by the tests, not the tests themselves. Of course you can set
@code{TESTS = $(check_PROGRAMS)} if all your programs are test cases.
address@hidden Serial Test Harness
address@hidden Serial Test Harness
+
address@hidden:} This harness, while still being the default one, is
+obsolescent, and kept mostly for backward-compatibility reasons.
+The user is advised to use the parallel test harness instead
+(@pxref{Parallel Test Harness}).
+
+The serial harness operates by simply running the tests serially, one at
+the time, without any I/O redirection. It's up to the user to implement
+logging of tests' output, if that's requited or desired.
address@hidden TODO: give an example of how this can be done.
address@hidden Simple Tests using parallel-tests
address@hidden Simple Tests using @samp{parallel-tests}
+For historical and implementation reasons, the @code{AM_TESTS_ENVIRONMENT}
+variable is @emph{not} supported by this harness (it will be silently
+ignored if defined); only @code{TESTS_ENVIRONMENT} is, and it is to be
+considered a developer-reserved variable. This is done so that, when
+using the serial harness, @code{TESTS_ENVIRONMENT} can be defined to an
+invocation of an interpreter through which the tests are to be run.
+For instance, the following setup may be used to run tests with Perl:
+
address@hidden
+TESTS_ENVIRONMENT = $(PERL) -Mstrict -w
+TESTS = foo.pl bar.pl baz.pl
address@hidden example
+
address@hidden
+It's important to note that the use of @code{TESTS_ENVIRONMENT} endorsed
+here would be @emph{invalid} with the parallel harness. That harness
+provides a more elegant way to achieve the same effect, with the further
+benefit of freeing the @code{TESTS_ENVIRONMENT} variable for the user
+(@pxref{Parallel Test Harness}).
+
+Another, less serious limit of the serial harness is that it doesn't
+really distinguish between simple failures and hard errors; this is
+due to historical reasons only, and might be fixed in future Automake
+versions.
+
address@hidden Parallel Test Harness
address@hidden Parallel Test Harness
@cindex @option{parallel-tests}, Using
-The option @option{parallel-tests} (@pxref{Options}) enables a test
-suite driver that is mostly compatible to the simple test driver described
-in the previous section, but provides a few more features and slightly
different
-semantics. It features concurrent execution of tests with @code{make -j},
-allows to specify inter-test dependencies, lazy reruns of tests that
-have not completed in a prior run, summary and verbose output in
address@hidden (reStructuredText) and @samp{HTML} format, and hard errors
-for exceptional failures. Similar to the simple test driver,
address@hidden, @code{XFAIL_TESTS}, and
-the @code{check_*} variables are honored, and the environment variable
address@hidden is set during test execution. Also, @code{TESTS_ENVIRONMENT}
-is still honored, but is complemented by a new developer-reserved variable
address@hidden (described below), and its allowed uses are
-somewhat restricted (other features of the @option{parallel-tests} driver
-compensate for this, though).
-
-This test driver is still experimental and may undergo changes in order
-to satisfy additional portability requirements.
+The parallel (or concurrent) test harness is enabled by the Automake option
address@hidden . It features concurrent execution of tests with
address@hidden -j}, allows to specify inter-test dependencies, lazy reruns of
+tests that have not completed in a prior run, summary and verbose output in
address@hidden (reStructuredText) and @samp{HTML} format.
+
+This harness is still somewhat experimental and may undergo changes in
+order to satisfy additional portability requirements.
address@hidden of test metadata}
@vindex TEST_SUITE_LOG
@vindex TESTS
-The driver operates by defining a set of @command{make} rules to create
-a summary log file, @code{TEST_SUITE_LOG}, which defaults to
address@hidden and requires a @file{.log} suffix. This file
-depends upon log files created for each single test program listed in
address@hidden, which in turn contain all output produced by the
-corresponding tests.
address@hidden @file{.log} files
address@hidden @file{.trs} files
address@hidden test metadata
+The parallel test harness operates by defining a set of @command{make}
+rules that run the test scripts listed in @code{TESTS}, and, for each
+such script, save its output in a corresponding @file{.log} file and
+its results (and other ``metadata'', @pxref{API for Custom Test Drivers})
+in a corresponding @file{.trs} (as in @b{T}est @address@hidden) file.
address@hidden We choose the `.trs' extension also because, at the time of
writing,
address@hidden it isn't already used for other significant purposes; see e.g.:
address@hidden - http://filext.com/file-extension/trs
address@hidden - http://www.file-extensions.org/search/?searchstring=trs
+The @file{.log} file will contain all the output emitted by the test on
+its standard output and its standard error. The @file{.trs} file will
+contain, among the other things, the results of the test cases run by
+the script.
+
+The parallel test harness will also create a summary log file,
address@hidden, which defaults to @file{test-suite.log} and requires
+a @file{.log} suffix. This file depends upon all the @file{.log} and
address@hidden files created for the test scripts listed in @code{TESTS}.
@vindex VERBOSE
-As with the simple driver above, by default one status line is printed
+As with the serial harness above, by default one status line is printed
per completed test, and a short summary after the suite has completed.
However, standard output and standard error of the test are redirected
to a per-test log file, so that parallel execution does not produce
intermingled output. The output from failed tests is collected in the
@file{test-suite.log} file. If the variable @samp{VERBOSE} is set, this
-file is output after the summary. For best results, the tests should be
-verbose by default now.
+file is output after the summary.
address@hidden FIXME: we should be clearer about what we mean exactly here ...
+For best results, the tests should be verbose by default now.
@vindex TEST_EXTENSIONS
@vindex TEST_LOGS
-Each log file is created when the corresponding test has completed.
-The set of log files is listed in the read-only variable
address@hidden, and defaults to @code{TESTS}, with the executable
-extension if any (@pxref{EXEEXT}), as well as any suffix listed in
address@hidden removed, and @file{.log} appended.
+Each couple of @file{.log} and @file{.trs} files is created when the
+corresponding test has completed. The set of log files is listed in
+the read-only variable @code{TEST_LOGS}, and defaults to @code{TESTS},
+with the executable extension if any (@pxref{EXEEXT}), as well as any
+suffix listed in @code{TEST_EXTENSIONS} removed, and @file{.log} appended.
@code{TEST_EXTENSIONS} defaults to @file{.test}. Results are undefined
if a test file name ends in several concatenated suffixes.
@@ -8814,14 +9079,15 @@ if a test file name ends in several concatenated
suffixes.
@vindex address@hidden
@vindex AM_LOG_FLAGS
For tests that match an extension @address@hidden listed in
address@hidden, you can provide a test driver using the variable
address@hidden@var{ext}_LOG_COMPILER} (note the upper-case extension) and pass
-options in @address@hidden and allow the user to pass
-options in @address@hidden It will cause all tests with
-this extension to be called with this driver. For all tests without a
-registered extension, the variables @code{LOG_COMPILER},
address@hidden, you can provide a custom ``test runner'' using
+the variable @address@hidden (note the upper-case
+extension) and pass options in @address@hidden and allow
+the user to pass options in @address@hidden It will cause
+all tests with this extension to be called with this runner. For all
+tests without a registered extension, the variables @code{LOG_COMPILER},
@code{AM_LOG_FLAGS}, and @code{LOG_FLAGS} may be used. For example,
address@hidden Keep in sync with parallel-tests-log-compiler-example.test.
@example
TESTS = foo.pl bar.py baz
TEST_EXTENSIONS = .pl .py
@@ -8836,37 +9102,14 @@ AM_LOG_FLAGS = -d
@noindent
will invoke @samp{$(PERL) -w foo.pl}, @samp{$(PYTHON) -v bar.py},
and @samp{./wrapper-script -d baz} to produce @file{foo.log},
address@hidden, and @file{baz.log}, respectively. The
address@hidden variable is still expanded before the driver,
-but should be reserved for the user.
-
address@hidden AM_TESTS_ENVIRONMENT
-The @code{AM_TESTS_ENVIRONMENT} and @code{TESTS_ENVIRONMENT} variables can
-be used to run initialization code and set environment variables for the
-test scripts. The former variable is developer-reserved, and can be
-defined in the @file{Makefile.am}, while the latter is reserved for the
-user, which can employ it to extend or override the settings in the
-former; for this to work portably, however, the contents of a non-empty
address@hidden @emph{must} be terminated by a semicolon.
-
address@hidden
-AM_TESTS_ENVIRONMENT = \
-## Some environment initializations are kept in a separate shell file
-## `tests-env.sh', which can make it easier to also run tests from
-## the command line.
- . $(srcdir)/tests-env.sh; \
-## On Solaris, prefer more POSIX-compliant versions of the standard
-## tools by default.
- if test -d /usr/xpg4/bin; then \
- PATH=/usr/xpg4/bin:$$PATH; export PATH; \
- fi;
address@hidden $$ restore font-lock
address@hidden example
address@hidden, and @file{baz.log}, respectively. The @file{foo.trs},
address@hidden and @file{baz.trs} files will be automatically produced
+as a side-effect.
It's important to note that, differently from what we've seen for the
-serial testsuite driver (@pxref{Simple Tests using parallel-tests}), the
+serial test harness (@pxref{Parallel Test Harness}), the
@code{AM_TESTS_ENVIRONMENT} and @code{TESTS_ENVIRONMENT} variables
address@hidden be use to define a custom test driver; the
address@hidden be use to define a custom test runner; the
@code{LOG_COMPILER} and @code{LOG_FLAGS} (or their extension-specific
counterparts) should be used instead:
@@ -8893,28 +9136,13 @@ to HTML using @samp{RST2HTML}, which defaults to
@command{rst2html} or
set of converted log files. The log and HTML files are removed upon
@code{make mostlyclean}.
address@hidden DISABLE_HARD_ERRORS
address@hidden Exit status 99, special interpretation
address@hidden hard error
-Even in the presence of expected failures (see @code{XFAIL_TESTS}), there
-may be conditions under which a test outcome needs attention. For
-example, with test-driven development, you may write tests for features
-that you have not implemented yet, and thus mark these tests as expected
-to fail. However, you may still be interested in exceptional conditions,
-for example, tests that fail due to a segmentation violation or another
-error that is independent of the feature awaiting implementation.
-Tests can exit with an exit status of 99 to signal such a @emph{hard
-error}. Unless the variable @code{DISABLE_HARD_ERRORS} is set to a
-nonempty value, such tests will be counted as failed.
-
-By default, the test suite driver will run all tests, but there are
+By default, the test suite harness will run all tests, but there are
several ways to limit the set of tests that are run:
@itemize @bullet
@item
-You can set the @code{TESTS} variable, similarly to how you can with
-the simple test driver from the previous section. For example, you can
-use a command like this to run only a subset of the tests:
+You can set the @code{TESTS} variable. For example, you can use a
+command like this to run only a subset of the tests:
@example
env TESTS="foo.test bar.test" make -e check
@@ -8927,6 +9155,7 @@ whose testsuite takes long time to execute. Luckily,
this problem can
easily be avoided by overriding also @code{TEST_SUITE_LOG} at runtime;
for example,
address@hidden Keep in sync with parallel-tests-log-override-2.test.
@example
env TEST_SUITE_LOG=partial.log TESTS="..." make -e check
@end example
@@ -8950,14 +9179,15 @@ here too.
@item
@vindex RECHECK_LOGS
@cindex lazy test execution
-By default, the test driver removes all old per-test log files before it
-starts running tests to regenerate them. The variable
address@hidden contains the set of log files which are removed.
address@hidden defaults to @code{TEST_LOGS}, which means all tests
-need to be rechecked. By overriding this variable, you can choose which
-tests need to be reconsidered. For example, you can lazily rerun only
-those tests which are outdated, i.e., older than their prerequisite test
-files, by setting this variable to the empty value:
+By default, the test harness removes all old per-test @file{.log} and
address@hidden files before it starts running tests to regenerate them. The
+variable @code{RECHECK_LOGS} contains the set of @file{.log} (and, by
+implication, @file{.trs}) files which are removed. @code{RECHECK_LOGS}
+defaults to @code{TEST_LOGS}, which means all tests need to be rechecked.
+By overriding this variable, you can choose which tests need to be
+reconsidered. For example, you can lazily rerun only those tests which
+are outdated, i.e., older than their prerequisite test files, by setting
+this variable to the empty value:
@example
env RECHECK_LOGS= make -e check
@@ -8969,15 +9199,15 @@ env RECHECK_LOGS= make -e check
You can ensure that all tests are rerun which have failed or passed
unexpectedly, by running @code{make recheck} in the test directory.
This convenience target will set @code{RECHECK_LOGS} appropriately
-before invoking the main test driver. The @code{recheck-html} target
-does the same as @code{recheck} but again converts the resulting log
-file in HTML format, like the @code{check-html} target.
+before invoking the main test harness. The @code{recheck-html} target
+does the same as @code{recheck} but again converts the resulting
address@hidden file in HTML format, like the @code{check-html} target.
@end itemize
In order to guarantee an ordering between tests even with @code{make
address@hidden, dependencies between the corresponding log files may be
-specified through usual @command{make} dependencies. For example, the
-following snippet lets the test named @file{foo-execute.test} depend
address@hidden, dependencies between the corresponding @file{.log} files
+may be specified through usual @command{make} dependencies. For example,
+the following snippet lets the test named @file{foo-execute.test} depend
upon completion of the test @file{foo-compile.test}:
@example
@@ -8997,6 +9227,7 @@ parallel @command{make address@hidden, so be sure they
are prepared for
concurrent execution.
@cindex Unit tests
address@hidden Keep in sync with 'parallel-tests-extra-programs.test'.
The combination of lazy test execution and correct dependencies between
tests and their sources may be exploited for efficient unit testing
during development. To further speed up the edit-compile-test cycle, it
@@ -9036,20 +9267,17 @@ or work around the issue with inference rules to
generate the tests.
@node Overview of Custom Test Drivers Support
@subsection Overview of Custom Test Drivers Support
-Starting form Automake version 1.12, the @option{parallel-tests} harness
-allows the package authors to use third-party custom test drivers, in case
-the
address@hidden FIXME: this should become "default ones" once we have TAP and
Subunit
-default one is inadequate for their purposes.
address@hidden FIXME: add this once we have TAP and Subunit
address@hidden "or do not support their testing protocol of choice."
-
-A custom test driver is expected to properly run the test programs
-passed to it, including the command-line arguments passed to it,
-to analyze their execution and outcome, to create the @file{.log}
-files associated to these test runs, and to display the test results
-on the console. It is responsibility of the author of the test driver
-to ensure that it implements all the above steps meaningfully and
+Starting from Automake version 1.12, the parallel test harness allows
+the package authors to use third-party custom test drivers, in case the
+default ones are inadequate for their purposes, or do not support their
+testing protocol of choice.
+
+A custom test driver is expected to properly run the test programs passed
+to it (including the command-line arguments passed to those programs, if
+any), to analyze their execution and outcome, to create the @file{.log}
+and @file{.trs} files associated to these test runs, and to display the test
+results on the console. It is responsibility of the author of the test
+driver to ensure that it implements all the above steps meaningfully and
correctly; Automake isn't and can't be of any help here. On the other
hand, the Automake-provided code for testsuite summary generation offers
support for test drivers allowing several test results per test script,
@@ -9059,24 +9287,28 @@ generation and test results recording}).
The exact details of how test scripts' results are to be determined and
analyzed is left to the individual drivers. Some drivers might only
consider the test script exit status (this is done for example by the
-default test driver used by the @option{parallel-tests} harness, described
+default test driver used by the parallel test harness, described
in the previous section). Other drivers might implement more complex and
advanced test protocols, which might require them to parse and interpreter
the output emitted by the test script they're running (examples of such
protocols are TAP and SubUnit).
It's very important to note that, even when using custom test drivers,
-most of the @option{parallel-tests} infrastructure described in the
-previous section remains in place; this includes:
+most of the infrastructure described in the previous section about the
+the parallel harness remains in place; this includes:
@itemize
@item
-concurrency through the use of @command{make}'s option @option{-j};
+list of test scripts defined in @code{TESTS}, and overridable at
+runtime through the redefinition of @code{TESTS} or @code{TEST_LOGS};
@item
-per-test @file{.log} files, and generation of a summary @file{.log}
-file from them;
+concurrency through the use of @command{make}'s option @option{-j};
@item
address@hidden target and lazy reruns of tests;
+per-test @file{.log} and @file{.trs} files, and generation of a summary
address@hidden file from them;
address@hidden
address@hidden target, @code{RECHECK_LOGS} variable, and lazy reruns
+of tests;
@item
inter-test dependencies;
@item
@@ -9086,8 +9318,9 @@ support for @code{check_*} variables
(@code{check_PROGRAMS},
use of @code{VERBOSE} environment variable to get verbose output on
testsuite failures;
@item
-definition and honoring of @code{TESTS_ENVIRONMENT} and
address@hidden variables;
+definition and honoring of @code{TESTS_ENVIRONMENT},
address@hidden and @code{AM_TESTS_FD_REDIRECT}
+variables;
@item
definition of generic and extension-specific @code{LOG_COMPILER} and
@code{LOG_FLAGS} variables.
@@ -9145,9 +9378,9 @@ for new features or to satisfy additional portability
requirements.
The main characteristic of these APIs is that they are designed to share
as much infrastructure, semantics, and implementation details as possible
-with the @option{parallel-tests} harness and its default driver. So
-everything said in the previous section should apply here too, unless
-explicitly stated otherwise.
+with the parallel test harness and its default driver. So everything
+said in the previous section should apply here too, unless explicitly
+stated otherwise.
@menu
* Command-line arguments for test drivers::
@@ -9175,10 +9408,14 @@ The name of the test, with VPATH prefix (if any)
removed. This can have a
suffix and a directory component (as in e.g., @file{sub/foo.test}), and is
mostly meant to be used in console reports about testsuite advancements and
results (@pxref{Testsuite progress output}).
address@hidden address@hidden
-The log file the test driver must create. If it has a directory component
-(as in e.g., @file{sub/foo.log}), the Automake harness will ensure that
-such directory exist @emph{before} the test driver is called.
address@hidden address@hidden@var{PATH}.log}
+The @file{.log} file the test driver must create. If it has a directory
+component (as in e.g., @file{sub/foo.log}), the test harness will ensure
+that such directory exist @emph{before} the test driver is called.
address@hidden address@hidden@var{PATH}.trs}
+The @file{.trs} file the test driver must create. If it has a directory
+component (as in e.g., @file{sub/foo.trs}), the test harness will ensure
+that such directory exist @emph{before} the test driver is called.
@item address@hidden|address@hidden
Whether the console output should be colorized or not (@pxref{Simple
tests and color-tests}, to learn when this option gets activated and
@@ -9208,82 +9445,91 @@ consistency and a more pleasant user experience.
@node Log files generation and test results recording
@subsubsection Log files generation and test results recording
-The test driver must correctly generate the file specified by the
address@hidden option (of course even when the tested program
-fails or crashes). This file is quite free-form, but still it has
-to conform to the following conventions, in order to work with the
address@hidden harness and take advantage of its features.
+The test driver must correctly generate the files specified by the
address@hidden and @option{--trs-file} option (even when the tested
+program fails or crashes).
+
+The @file{.log} file should ideally contain all the output produced by the
+tested program, plus optionally other information that might facilitate
+debugging or analysis of bug reports. Apart from that, its format is
+basically free; the only limitation being that it must parse validly as
+reStructuredText if the @file{.log} -> @file{.html} conversion is to be
+supported.
+
+The @file{.trs} file is used to register some metadata through the use
+of custom reStructuredText fields. This metadata is expected to be
+employed in various ways by the parallel test harness; for example, to
+count the test results when printing the testsuite summary, or to decide
+which tests to re-run upon @command{make reheck}. Unrecognized metadata
+in a @file{.trs} file is currently ignored by the harness, but this might
+change in the future. The list of currently recognized metadata follows.
address@hidden @bullet
address@hidden
address@hidden Global test script result
address@hidden Test result, global
-The @emph{global test result} for the whole test script must be placed
-in the first line of the @file{.log} file.
address@hidden The following semantics is checked by tests
'test-driver-recheck.test'
address@hidden and 'test-driver-global-log.test'. Please keep them in sync
whenever
address@hidden the following is changed.
-If this line does not begin with either @code{PASS:}, @code{XFAIL:} or
address@hidden:}, the test script will be considered failed, and it will be
-re-run by @code{make recheck}. Also, if this line does not begin with
-either @code{PASS:} or @code{XFAIL:}, the content of the @file{.log} file
-will be copied into the global @file{test-suite.log} (in order to help in
-debugging and bug-report analysis).
address@hidden @code
address@hidden
address@hidden Single test case result
address@hidden Test result, single
address@hidden @code{:test-result:} reStructuredText field
address@hidden reStructuredText field, @code{:test-result:}
-One of the main features of the new testsuite harness is the ability to
-support test protocols that allow a single test script to run more
-test cases, @emph{each with its distinct result}. In order for the
-testsuite summary to be correct in this case, the test driver must
-register @emph{each} such result in the generated @file{.log} file, using
-the @code{:test-result:} reStructuredText field. Otherwise, only the
-global test result will be considered (as it is the case of the default
address@hidden driver).
address@hidden :test-result:
address@hidden Register test result
address@hidden Register test case result
address@hidden Test result, registering
address@hidden Test case result, registering
address@hidden @code{:test-result:}
address@hidden reStructuredText field, @code{:test-result:}
+The test driver must use this field to register the results of @emph{each}
+test case run by a test script file. Several @code{:test-result:} fields
+can be present in the same @file{.trs} file; this is done in order to
+support test protocols that allow a single test script to run more test
+cases.
@c Keep this in sync with lib/am/check-am:$(TEST_SUITE_LOG).
The only recognized test results are currently @code{PASS}, @code{XFAIL},
address@hidden, @code{FAIL} and @code{XPASS}. These results, when declared
-with @code{:test-result:}, can be optionally followed by text holding
-the name and/or a brief description of the corresponding test; the
address@hidden harness will ignore such extra text when
address@hidden, @code{FAIL}, @code{XPASS} and @code{ERROR}. These results,
+when declared with @code{:test-result:}, can be optionally followed by
+text holding the name and/or a brief description of the corresponding
+test; the @option{parallel-tests} harness will ignore such extra text when
generating @file{test-suite.log} and preparing the testsuite summary.
-For example, if a @file{.log} file contains the following lines:
address@hidden Keep in sync with 'test-metadata-recheck.test'.
address@hidden @code{:recheck:}
address@hidden :recheck:
address@hidden reStructuredText field, @code{:recheck:}
+If this field is present and defined to @code{no}, then the corresponding
+test script will @emph{not} be run upon a @command{make recheck}. What
+happens when two or more @code{:recheck:} fields are present in the same
address@hidden file is undefined behaviour.
+
address@hidden Keep in sync with 'test-metadata-global-log.test'.
address@hidden @code{:copy-in-global-log:}
address@hidden :copy-in-global-log:
address@hidden reStructuredText field, @code{:copy-in-global-log:}
+If this field is present and defined to @code{no}, then the content
+of the @file{.log} file will @emph{not} be copied into the global
address@hidden We allow to forsake such copying because, while
+it can be useful in debugging and analysis of bug report, it can also be
+just a waste of space in normal situations, e.g., when a test script is
+successful. What happens when two or more @code{:copy-in-global-log:}
+fields are present in the same @file{.trs} file is undefined behaviour.
+
address@hidden table
+
address@hidden
+Let's see a small example. Assume a @file{.trs} file contains the
+following lines:
@example
:test-result: PASS server starts
+:global-log-copy: no
:test-result: PASS HTTP/1.1 request
:test-result: FAIL HTTP/1.0 request
+:recheck: yes
:test-result: SKIP HTTPS request (TLS library wasn't available)
:test-result: PASS server stops
@end example
@noindent
-it means that the corresponding test script contributes with five test
-results to the testsuite summary (three of these tests being successful,
-one failed, and one skipped).
-
address@hidden FIXME: currently, the @code{:test-result:} field is recognized
anywhere
address@hidden in the @file{.log} file; this is possibly prone to generating
spurious
address@hidden results, in case of verbose tests. We should allow a special
address@hidden @code{:test-result:} that stops any further parsing (maybe a line
address@hidden @code{:test-result:END} will do?).
-
address@hidden itemize
-
-Finally, note that the declaration of the global test result in the first
-line, apart from being needed for backward-compatibility with the default
address@hidden driver, can be useful also for test protocols
-that allow more test results per test script: using it, a custom test
-driver is allowed to decide what the global outcome of the test script
-is in case of conflicting test results within the script. For example,
-if a test script reports 8 successful test cases and 2 skipped test
-cases, some drivers might report that globally as a SKIP, while others
-as a PASS.
+Then the corresponding test script will be re-run by @command{make check},
+will contribute with @emph{five} test results to the testsuite summary
+(three of these tests being successful, one failed, and one skipped), and
+the content of the corresponding @file{.log} file will @emph{not} be
+copied in the global log file @file{test-suite.log}.
@node Testsuite progress output
@subsubsection Testsuite progress output
@@ -9319,11 +9565,227 @@ implementations' output might be a good idea too.
If HTML testsuite output (with @code{check-html}) is to be supported,
the generated @file{.log} files must contain syntactically valid
-reStructuredText (among the other things, this means that every
address@hidden:test-result:} line must be followed by a blank line). If this
-is not the case, the HTML generation will not work, although all the
-other functionalities of the Automake testsuite harness should remain
-untouched, and continue to work correctly.
+reStructuredText. If this is not the case, the HTML generation will not
+work, although all the other functionalities of the Automake testsuite
+harness should remain untouched, and continue to work correctly.
+
address@hidden Using the TAP test protocol
address@hidden Using the TAP test protocol
+
address@hidden
+* Introduction to TAP::
+* Use TAP with the Automake test harness::
+* Incompatibilities with other TAP parsers and drivers::
+* Links and external resources::
address@hidden menu
+
address@hidden Introduction to TAP
address@hidden Introduction to TAP
+
+TAP, the Test Anything Protocol, is a simple text-based interface between
+testing modules or programs and a test harness. The tests (also called
+``TAP producers'' in this context) write test results in a simple format
+on standard output; a test harness (also called ``TAP consumer'') will
+parse and interpret these results, and properly present them to the user,
+and/or register them for later analysis. The exact details of how this
+is accomplished can vary among different test harnesses. The Automake
+parallel harness will present the results on the console in the usual
+fashion (@pxref{Testsuite progress on console}), and will use the
address@hidden files (@pxref{Basics of test metadata}) to store the test
+results and related metadata. Apart from that, it will try to remain
+as much compatible as possible with pre-existing and widespread utilities,
+such as the @uref{http://search.cpan.org/~andya/Test-Harness/bin/prove,
address@hidden utility}, at least for the simpler usages.
+
+TAP started its life as part of the test harness for Perl, but today
+it has been (mostly) standardized, and has various independent
+implementations in different languages; among them, C, C++, Perl,
+Python, PHP, and Java. For a semi-official specification of the
+TAP protocol, please refer to the documentation of
address@hidden://search.cpan.org/~petdance/Test-Harness/lib/Test/Harness/TAP.pod,
+ @samp{Test::Harness::TAP}}.
+
+The most relevant real-world usages of TAP are obviously in the testsuites
+of @command{perl} and of many perl modules. Still, also other important
+third-party packages, such as @uref{http://git-scm.com/, @command{git}},
+use TAP in their testsuite.
+
address@hidden Use TAP with the Automake test harness
address@hidden Use TAP with the Automake test harness
+
+Currently, the TAP driver that comes with Automake requires a perl
+interpreter to work, and requires various by-hand steps on the
+developer's part (this should be fixed in future Automake versions).
+You'll have grab the @file{tap-driver} script from the Automake
+distribution by hand, copy it in your source tree, add code to
address@hidden to search a perl interpreter and to define the
address@hidden(PERL)} variable accordingly, and use the Automake support
+for third-party test drivers to instruct the harness to use the
address@hidden to run your TAP-producing tests. See the example
+below for clarification.
+
+Apart from the options common to all the Automake test drivers
+(@pxref{Command-line arguments for test drivers}), the @file{tap-driver}
+supports the following options, whose names are chosen for enhanced
+compatibility with the @command{prove} utility.
+
address@hidden @option
address@hidden --ignore-exit
+Causes the test driver to ignore the exit status of the test scripts;
+by default, the driver will report an error if the script exit with a
+non-zero status.
address@hidden --comments
+Instruct the test driver to display TAP diagnostic (i.e., lines beginning
+with the @samp{#} character) in the testsuite progress output too; by
+default, TAP diagnostic is only copied in the @file{.log} file.
address@hidden --no-comments
+Revert the effects of @option{--comments}.
address@hidden --merge
+Instruct the test driver to merge the test scripts' standard error into
+their standard output. This is necessary if you want to ensure that
+diagnostics from the test scripts are displayed in the correct order
+relative to test results; this can be of great help in debugging
+(especially if your test scripts are shell scripts run with shell
+tracing active). As a downside, this option might cause the test
+harness to get confused if anything that appears on standard error
+looks like a test result.
address@hidden --no-merge
+Revert the effects of @option{--merge}.
address@hidden table
+
address@hidden
+Here is an example of how the TAP driver can be set up and used.
+
address@hidden Keep in sync with tap-doc2.test.
address@hidden
+% @kbd{cat configure.ac}
+AC_INIT([GNU Try Tap], [1.0], [bug-automake@@gnu.org])
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE([foreign parallel-tests -Wall -Werror])
+AC_CONFIG_FILES([Makefile])
+AC_REQUIRE_AUX_FILE([tap-driver])
+AC_PATH_PROG([PERL], [perl])
+test -n "$PERL" || AC_MSG_ERROR([perl not found])
+$PERL -MTAP::Parser -e 1 || AC_MSG_ERROR([TAP::Parser not found])
+AC_OUTPUT
+
+% @kbd{cat Makefile.am}
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/build-aux/tap-driver
+TESTS = foo.test bar.test baz.test
+EXTRA_DIST = $(TESTS)
+
+% @kbd{cat foo.test}
+#!/bin/sh
+echo 1..4 # Number of tests to be executed.
+echo 'ok 1 - Swallows fly'
+echo 'not ok 2 - Caterpillars fly # TODO metamorphosis in progress'
+echo 'ok 3 - Pigs fly # SKIP not enough acid'
+echo '# I just love word plays ...'
+echo 'ok 4 - Flies fly too :-)'
+
+% @kbd{cat bar.test}
+#!/bin/sh
+echo 1..3
+echo 'not ok 1 - Bummer, this test has failed.'
+echo 'ok 2 - This passed though.'
+echo 'Bail out! Ennui kicking in, sorry...'
+echo 'ok 3 - This will not be seen.'
+
+% @kbd{cat baz.test}
+#!/bin/sh
+echo 1..1
+echo ok 1
+# Exit with error, even if all the test case has been successful.
+exit 7
+
+% @kbd{cp @var{PREFIX}/share/address@hidden/tap-driver .}
+% @kbd{autoreconf -vi && ./configure && make check}
+...
+PASS: foo.test 1 - Swallows fly
+XFAIL: foo.test 2 - Caterpillars fly # TODO metamorphosis in progress
+SKIP: foo.test 3 - Pigs fly # SKIP not enough acid
+PASS: foo.test 4 - Flies fly too :-)
+FAIL: bar.test 1 - Bummer, this test has failed.
+PASS: bar.test 2 - This passed though.
+ERROR: bar.test - Bail out! Ennui kicking in, sorry...
+PASS: baz.test 1
+ERROR: baz.test - exited with status 7
+...
+Please report to bug-automake@@gnu.org
+...
+% @kbd{echo exit status: $?}
+exit status: 1
+
address@hidden Keep the "skewed" indentation below, it produces pretty PDF
output.
+% @kbd{env TEST_LOG_DRIVER_FLAGS='--comments --ignore-exit' \
+ TESTS='foo.test baz.test' make -e check}
+...
+PASS: foo.test 1 - Swallows fly
+XFAIL: foo.test 2 - Caterpillars fly # TODO metamorphosis in progress
+SKIP: foo.test 3 - Pigs fly # SKIP not enough acid
+# foo.test: I just love word plays...
+PASS: foo.test 4 - Flies fly too :-)
+PASS: baz.test 1
+...
+% @kbd{echo exit status: $?}
+exit status: 0
address@hidden example
+
address@hidden Incompatibilities with other TAP parsers and drivers
address@hidden Incompatibilities with other TAP parsers and drivers
+
+For implementation or historical reasons, the TAP driver and harness as
+implemented by Automake have some minors incompatibilities with the
+mainstream versions, which you should be aware of.
+
address@hidden @bullet
address@hidden
+A @code{Bail out!} directive doesn't stop the whole testsuite, but only
+the test script it occurs into. This doesn't follows TAP specifications,
+but on the other hand it maximizes compatibility (and code sharing) with
+the ``hard error'' concept of the default @option{parallel-tests} driver.
address@hidden
address@hidden: there's surely something else ...
address@hidden itemize
+
address@hidden Links and external resources
address@hidden Links and external resources
+
address@hidden
+Here are some links to more extensive official or third-party
+documentation and resources:
address@hidden @bullet
address@hidden
address@hidden://search.cpan.org/~petdance/Test-Harness/lib/Test/Harness/TAP.pod,
+ @samp{Test::Harness::TAP}},
+the (mostly) official documentation about the TAP format and protocol.
address@hidden
address@hidden://search.cpan.org/~andya/Test-Harness/bin/prove,
+ @command{prove}},
+the most famous command-line TAP test driver, included in the distribution
+of @command{perl} and
address@hidden://search.cpan.org/~andya/Test-Harness/lib/Test/Harness.pm,
+ @samp{Test::Harness}}.
address@hidden
+The @uref{http://testanything.org/wiki/index.php/Main_Page,TAP wiki}.
address@hidden
+A ``gentle introduction'' to testing for perl coders:
address@hidden://search.cpan.org/dist/Test-Simple/lib/Test/Tutorial.pod,
+ @samp{Test::Tutorial}}.
address@hidden
address@hidden://search.cpan.org/~mschwern/Test-Simple/lib/Test/Simple.pm,
+ @samp{Test::Simple}}
+and
address@hidden://search.cpan.org/~mschwern/Test-Simple/lib/Test/More.pm,
+ @samp{Test::More}},
+the standard perl testing libraries, which are based on TAP.
address@hidden
address@hidden://www.eyrie.org/~eagle/software/c-tap-harness/,C TAP Harness},
+a C-based project implementing both a TAP producer and a TAP consumer.
address@hidden
address@hidden://www.tap4j.org/,tap4j},
+a Java-based project implementing both a TAP producer and a TAP consumer.
address@hidden itemize
@node DejaGnu Tests
@section DejaGnu Tests
@@ -9368,8 +9830,6 @@ Especially, @file{site.exp} should not be distributed.
For more information regarding DejaGnu test suites, see @ref{Top, , ,
dejagnu, The DejaGnu Manual}.
-In either case, the testing is done via @samp{make check}.
-
@node Install Tests
@section Install Tests
@@ -9580,10 +10040,9 @@ in the first few lines of the @file{NEWS} file.
@item @option{color-tests}
@cindex Option, @option{color-tests}
@opindex color-tests
-Cause output of the simple and parallel test suites (see @ref{Simple Tests}
-and @ref{Simple Tests using parallel-tests}) and of properly-written custom
-test drivers (@pxref{Custom Test Drivers}) to be colorized on capable
-terminals.
+Cause output of the serial and parallel test harnesses (see @ref{Simple
+Tests}) and of properly-written custom test drivers (@pxref{Custom Test
+Drivers}) to be colorized on capable terminals.
@item @option{dejagnu}
@cindex Option, @option{dejagnu}
@@ -9711,8 +10170,8 @@ this directory.
@item @option{parallel-tests}
@cindex Option, @option{parallel-tests}
@opindex parallel-tests
-Enable test suite driver for @code{TESTS} that can run tests in parallel
-(@pxref{Simple Tests using parallel-tests}, for more information).
+Enable test suite harness for @code{TESTS} that can run tests in parallel
+(@pxref{Parallel Test Harness}, for more information).
@item @option{readme-alpha}
@cindex Option, @option{readme-alpha}
@@ -9991,6 +10450,7 @@ For instance, the following definition prevents Automake
from misinterpreting
the @samp{.idlC.cpp:} rule as an attempt to transform @file{.idlC} files into
@file{.cpp} files.
address@hidden Keep in sync with suffix7.test.
@example
SUFFIXES = .idl C.cpp
.idlC.cpp:
@@ -10459,6 +10919,7 @@ make silent rules enabled by default in his own package
can do so by
adding a @samp{yes} argument to the @code{AM_SILENT_RULES} call in
@file{configure.ac}. We advise against this approach, though.
address@hidden Keep in sync with silent-configsite.test
Users who prefer to have silent rules enabled by default can edit their
@file{config.site} file to make the variable @code{enable_silent_rules}
default to @samp{yes}. This should still allow disabling silent rules
@@ -10819,6 +11280,7 @@ Checks, autoconf, The Autoconf Manual}) and use
@samp{$(LN_S)} in
For instance, here is how you could install a versioned copy of a
program using @samp{$(LN_S)}:
address@hidden Keep in sync with insthook.test
@example
install-exec-hook:
cd $(DESTDIR)$(bindir) && \
@@ -10833,7 +11295,9 @@ destination directory in order to create relative links.
When writing @code{install-exec-hook} or @code{install-data-hook},
please bear in mind that the exec/data distinction is based on the
installation directory, not on the primary used (@pxref{The Two Parts of
-Install}). So a @code{foo_SCRIPTS} will be installed by
+Install}).
address@hidden Keep in sync with primary-prefix-couples-documented-valid.test.
+So a @code{foo_SCRIPTS} will be installed by
@code{install-data}, and a @code{barexec_SCRIPTS} will be installed by
@code{install-exec}. You should define your hooks consequently.
diff --git a/lib/Automake/tests/.gitignore b/lib/Automake/tests/.gitignore
index ffbdfaf..bc2e76d 100644
--- a/lib/Automake/tests/.gitignore
+++ b/lib/Automake/tests/.gitignore
@@ -1,2 +1,3 @@
*.log
*.log-t
+*.trs
diff --git a/lib/Automake/tests/Makefile.in b/lib/Automake/tests/Makefile.in
index e1867fb..78a7e4a 100644
--- a/lib/Automake/tests/Makefile.in
+++ b/lib/Automake/tests/Makefile.in
@@ -71,12 +71,14 @@ am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY))
am__v_at_0 = @
SOURCES =
DIST_SOURCES =
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-am__color_tests=no; \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
@@ -86,6 +88,8 @@ test "X$(AM_COLOR_TESTS)" != Xno \
grn='[0;32m'; \
lgn='[1;32m'; \
blu='[1;34m'; \
+ mgn='[0;35m'; \
+ brg='[1m'; \
std='[m'; \
}
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
@@ -111,31 +115,12 @@ am__base_list = \
sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
# Restructured Text title.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
am__sh_e_setup = case $$- in *e*) set +e;; esac
-# Default flags passed to all log compiler wrappers.
-am__test_driver_flags = \
- --test-name "$$f" \
- --log-file '$@' \
+# Default flags passed to test drivers.
+am__common_driver_flags = \
--color-tests "$$am__color_tests" \
--enable-hard-errors "$$am__enable_hard_errors" \
--expect-failure "$$am__expect_failure"
@@ -169,6 +154,14 @@ case " $(XFAIL_TESTS) " in \
am__expect_failure=no;; \
esac; \
$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT)
+# The names of the tests, with any registered extension removed. Or
+# equivalently, the names of the test logs, with the `.log' extension
+# renoved. This honours runtime overriding of TESTS and TEST_LOGS.
+am__TEST_BASES = $(TEST_LOGS:.log=)
+# This can be used instead of $(MAKE) in recipes requiring a recursive call
+# to make, but which are not intended to be executed by "make -n". See the
+# GNU make manual for more details.
+am__stealth_MAKE = $(MAKE)
RECHECK_LOGS = $(TEST_LOGS)
AM_RECURSIVE_TARGETS = check check-html recheck recheck-html
TEST_SUITE_LOG = test-suite.log
@@ -302,7 +295,7 @@ EXTRA_DIST = $(TESTS)
all: all-am
.SUFFIXES:
-.SUFFIXES: .html .log .pl
+.SUFFIXES: .html .log .pl .trs
$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
@@ -342,91 +335,123 @@ CTAGS:
cscope cscopelist:
+# Recover from deleted `.trs' file; this should ensure that
+# "rm -f foo.log; make foo.trs" re-run `foo.test', and re-create
+# both `foo.log' and `foo.trs'.
+.log.trs:
+ rm -f $< $@ && $(am__stealth_MAKE) $(AM_MAKEFLAGS) $<
+
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
- rst_magic=":test-result:"; \
- list='$(TEST_LOGS)'; \
- list2=`for f in $$list; do test ! -r $$f || echo $$f; done`; \
- results1=`for f in $$list; do test -r $$f || echo FAIL; done`; \
- results2=''; \
- if test -n "$$list2"; then \
- results2=`sed -n "s/^$$rst_magic[ ]*//p" $$list2`; \
+ @$(am__sh_e_setup); $(am__tty_colors); \
+ bases='$(am__TEST_BASES)'; \
+ ws='[ ]'; \
+ redo_bases=`for b in $$bases; do \
+ test -f $$b.trs && test -r $$b.trs \
+ && test -f $$b.log && test -r $$b.log \
+ || echo $$b; \
+ done`; \
+ if test -n "$$redo_bases"; then \
+ redo_logs=`for i in $$redo_bases; do echo $$i.log; done`; \
+ redo_results=`for i in $$redo_bases; do echo $$i.trs; done`; \
+ rm -f $$redo_logs && rm -f $$redo_results \
+ && $(am__stealth_MAKE) $(AM_MAKEFLAGS) $$redo_logs; \
fi; \
- results=`echo "$$results1" && echo "$$results2"` \
- all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[
]*//'`; \
- fail=`echo "$$results" | grep -c '^FAIL'`; \
- pass=`echo "$$results" | grep -c '^PASS'`; \
- skip=`echo "$$results" | grep -c '^SKIP'`; \
- xfail=`echo "$$results" | grep -c '^XFAIL'`; \
- xpass=`echo "$$results" | grep -c '^XPASS'`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+ st=0; \
+ for b in $$redo_bases; do \
+ for e in trs log; do \
+ if test ! -f $$b.$$e || test ! -r $$b.$$e; then \
+ echo "fatal: making $@: failed to create $$b.$$e" >&2; \
+ st=1; \
+ fi; \
+ done; \
+ done; \
+ test $$st -eq 0 || exit 1; \
+ results=`for b in $$bases; do echo $$b.trs; done`; \
+ test -n "$$results" || results=/dev/null; \
+ all=` grep "^$$ws*:test-result:" $$results | wc -l`; \
+ pass=` grep "^$$ws*:test-result:$$ws*PASS" $$results | wc -l`; \
+ fail=` grep "^$$ws*:test-result:$$ws*FAIL" $$results | wc -l`; \
+ skip=` grep "^$$ws*:test-result:$$ws*SKIP" $$results | wc -l`; \
+ xfail=`grep "^$$ws*:test-result:$$ws*XFAIL" $$results | wc -l`; \
+ xpass=`grep "^$$ws*:test-result:$$ws*XPASS" $$results | wc -l`; \
+ error=`grep "^$$ws*:test-result:$$ws*ERROR" $$results | wc -l`; \
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+ br='==================='; br=$$br$$br$$br$$br; \
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
- for f in $$list; do \
- test -r $$f && read line < $$f || line=; \
- case $$line in \
- PASS:*|XFAIL:*);; \
- *) echo; cat $$f;; \
- esac; \
+ for b in $$bases; do \
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null; then :; \
+ elif test ! -r $$b.log; then \
+ echo "ERROR: cannot read $$b.log" >&2; \
+ echo && echo "WARNING: could not read $$b.log!"; \
+ else \
+ echo && cat $$b.log; \
+ fi; \
done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
- $(am__tty_colors); \
- if $$exit; then \
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
+ fi; \
+ $$success
# Run all the tests.
check-TESTS:
@list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
+ @list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list
@test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
@list='$(TEST_LOGS)'; \
list=`for f in $$list; do \
@@ -459,20 +484,21 @@ check-html:
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML) || exit 4; \
exit $$rv
recheck recheck-html:
- @target=`echo $@ | sed 's,^re,,'`; \
- list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test -f $$f || continue; \
- if test -r $$f && read line < $$f; then \
- case $$line in PASS:*|XFAIL:*|SKIP:*);; *) echo $$f;; esac; \
- else echo $$f; fi; \
+ @ws='[ ]'; \
+ target=`echo $@ | sed 's,^re,,'`; \
+ bases='$(am__TEST_BASES)'; \
+ list=`for b in $$bases; do \
+ test ! -f $$b.trs && test ! -f $$b.log && continue; \
+ grep "^$$ws*:recheck:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null 2>&1 || echo $$b.log; \
done | tr '\012\015' ' '`; \
list=`echo "$$list" | sed 's/ *$$//'`; \
$(MAKE) $(AM_MAKEFLAGS) $$target AM_MAKEFLAGS='$(AM_MAKEFLAGS)
TEST_LOGS="'"$$list"'"'
.pl.log:
- @p='$<'; $(am__check_pre) \
- $(PL_LOG_DRIVER) $(am__test_driver_flags) $(AM_PL_LOG_DRIVER_FLAGS)
$(PL_LOG_DRIVER_FLAGS) -- \
- $(PL_LOG_COMPILE) "$$tst"
+ @p='$<'; $(am__check_pre) $(PL_LOG_DRIVER) --test-name "$$f" \
+ --log-file '$*.log' --trs-file '$*.trs' \
+ $(am__common_driver_flags) $(AM_PL_LOG_DRIVER_FLAGS)
$(PL_LOG_DRIVER_FLAGS) -- $(PL_LOG_COMPILE) "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
@@ -533,6 +559,7 @@ mostlyclean-generic:
-test -z "$(TEST_LOGS_TMP)" || rm -f $(TEST_LOGS_TMP)
-test -z "$(TEST_SUITE_HTML)" || rm -f $(TEST_SUITE_HTML)
-test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
+ -test -z "$(am__TEST_BASES:=.trs)" || rm -f $(am__TEST_BASES:=.trs)
clean-generic:
diff --git a/lib/INSTALL b/lib/INSTALL
index 81fd332..a1e89e1 100644
--- a/lib/INSTALL
+++ b/lib/INSTALL
@@ -1,8 +1,8 @@
Installation Instructions
*************************
-Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
-2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
+Copyright (C) 1994-1996, 1999-2002, 2004-2011 Free Software Foundation,
+Inc.
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright
@@ -226,6 +226,11 @@ order to use an ANSI C compiler:
and if that doesn't work, install pre-built binaries of GCC for HP-UX.
+ HP-UX `make' updates targets which have the same time stamps as
+their prerequisites, which makes it generally unusable when shipped
+generated files such as `configure' are involved. Use GNU `make'
+instead.
+
On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot
parse its `<wchar.h>' header file. The option `-nodtk' can be used as
a workaround. If GNU CC is not installed, it is therefore recommended
diff --git a/lib/Makefile.am b/lib/Makefile.am
index a26d663..a9467f1 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -29,7 +29,7 @@ dist_pkgvdata_DATA = COPYING INSTALL texinfo.tex ansi2knr.c
ansi2knr.1 \
scriptdir = $(pkgvdatadir)
dist_script_DATA = config.guess config.sub install-sh mdate-sh missing \
mkinstalldirs elisp-comp ylwrap acinstall depcomp compile py-compile \
- symlink-tree ar-lib test-driver
+ symlink-tree ar-lib test-driver tap-driver
EXTRA_DIST = gnupload
diff --git a/lib/Makefile.in b/lib/Makefile.in
index 5f8aa91..b600402 100644
--- a/lib/Makefile.in
+++ b/lib/Makefile.in
@@ -242,7 +242,7 @@ dist_pkgvdata_DATA = COPYING INSTALL texinfo.tex ansi2knr.c
ansi2knr.1 \
scriptdir = $(pkgvdatadir)
dist_script_DATA = config.guess config.sub install-sh mdate-sh missing \
mkinstalldirs elisp-comp ylwrap acinstall depcomp compile py-compile \
- symlink-tree ar-lib test-driver
+ symlink-tree ar-lib test-driver tap-driver
EXTRA_DIST = gnupload
all: all-recursive
diff --git a/lib/am/check.am b/lib/am/check.am
index b02d6c5..c406e9f 100644
--- a/lib/am/check.am
+++ b/lib/am/check.am
@@ -15,13 +15,15 @@
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
if %?COLOR%
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-am__color_tests=no; \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
@@ -31,10 +33,12 @@ test "X$(AM_COLOR_TESTS)" != Xno \
grn='[0;32m'; \
lgn='[1;32m'; \
blu='[1;34m'; \
+ mgn='[0;35m'; \
+ brg='[1m'; \
std='[m'; \
}
else !%?COLOR%
-am__tty_colors = red= grn= lgn= blu= std= am__color_tests=no
+am__tty_colors = $(am__tty_colors_dummy)
endif !%?COLOR%
.PHONY: check-TESTS
@@ -66,7 +70,7 @@ include inst-vars.am
## appended.
##
## In addition to the magic "exit 77 means SKIP" feature (which was
-## imported from automake), there is a magic "exit 99 means FAIL" feature
+## imported from automake), there is a magic "exit 99 means ERROR" feature
## which is useful if you need to issue a hard error no matter whether the
## test is XFAIL or not. You can disable this feature by setting the
## variable DISABLE_HARD_ERRORS to a nonempty value.
@@ -74,33 +78,13 @@ include inst-vars.am
# Restructured Text title.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
-
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
am__sh_e_setup = case $$- in *e*) set +e;; esac
-# Default flags passed to all log compiler wrappers.
-am__test_driver_flags = \
- --test-name "$$f" \
- --log-file '$@' \
+# Default flags passed to test drivers.
+am__common_driver_flags = \
--color-tests "$$am__color_tests" \
--enable-hard-errors "$$am__enable_hard_errors" \
--expect-failure "$$am__expect_failure"
@@ -138,100 +122,162 @@ case " $(XFAIL_TESTS) " in
\
esac; \
$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT)
+# The names of the tests, with any registered extension removed. Or
+# equivalently, the names of the test logs, with the `.log' extension
+# renoved. This honours runtime overriding of TESTS and TEST_LOGS.
+am__TEST_BASES = $(TEST_LOGS:.log=)
+
+# This can be used instead of $(MAKE) in recipes requiring a recursive call
+# to make, but which are not intended to be executed by "make -n". See the
+# GNU make manual for more details.
+am__stealth_MAKE = $(MAKE)
+
+# Recover from deleted `.trs' file; this should ensure that
+# "rm -f foo.log; make foo.trs" re-run `foo.test', and re-create
+# both `foo.log' and `foo.trs'.
+.log.trs:
+ rm -f $< $@ && $(am__stealth_MAKE) $(AM_MAKEFLAGS) $<
+
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
-## The custom reStructuredText field used to register the outcome of
-## a test. This is for supporting test protocols, such as TAP and
-## SubUnit, where a single test script can run multiple tests, each
-## with its own outcome.
- rst_magic=":test-result:"; \
-## All test logs.
- list='$(TEST_LOGS)'; \
-## Readable test logs.
- list2=`for f in $$list; do test ! -r $$f || echo $$f; done`; \
-## Each unreadable test log counts as a failed test.
- results1=`for f in $$list; do test -r $$f || echo FAIL; done`; \
-## Extract the outcome of all the testcases from the test logs.
- results2=''; \
- if test -n "$$list2"; then \
- results2=`sed -n "s/^$$rst_magic[ ]*//p" $$list2`; \
+ @$(am__sh_e_setup); $(am__tty_colors); \
+ bases='$(am__TEST_BASES)'; \
+ ws='[ ]'; \
+## We need to ensures that all the required `.trs' and `.log' files will
+## be present and readable. The direct dependencies of $(TEST_SUITE_LOG)
+## only ensure that all the `.log' files exists; they don't ensure that
+## the `.log' files are readable, and worse, they don't ensure that the
+## `.trs' files even exist.
+ redo_bases=`for b in $$bases; do \
+ test -f $$b.trs && test -r $$b.trs \
+ && test -f $$b.log && test -r $$b.log \
+ || echo $$b; \
+ done`; \
+ if test -n "$$redo_bases"; then \
+## Uh-oh, either some `.log' files were unreadable, or some `.trs' files
+## were missing (or unreadable). We need to re-run the corresponding
+## tests in order to re-create them.
+ redo_logs=`for i in $$redo_bases; do echo $$i.log; done`; \
+ redo_results=`for i in $$redo_bases; do echo $$i.trs; done`; \
+ rm -f $$redo_logs && rm -f $$redo_results \
+ && $(am__stealth_MAKE) $(AM_MAKEFLAGS) $$redo_logs; \
fi; \
-## Prepare the test suite summary.
- results=`echo "$$results1" && echo "$$results2"` \
- all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[
]*//'`; \
- fail=`echo "$$results" | grep -c '^FAIL'`; \
- pass=`echo "$$results" | grep -c '^PASS'`; \
- skip=`echo "$$results" | grep -c '^SKIP'`; \
- xfail=`echo "$$results" | grep -c '^XFAIL'`; \
- xpass=`echo "$$results" | grep -c '^XPASS'`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+## Sanity check: each unreadable or non-existent test result file should
+## has been properly remade at this point, as should the corresponding log
+## file.
+ st=0; \
+ for b in $$redo_bases; do \
+ for e in trs log; do \
+ if test ! -f $$b.$$e || test ! -r $$b.$$e; then \
+ echo "fatal: making $@: failed to create $$b.$$e" >&2; \
+ st=1; \
+ fi; \
+ done; \
+ done; \
+ test $$st -eq 0 || exit 1; \
+## List of test result files.
+ results=`for b in $$bases; do echo $$b.trs; done`; \
+ test -n "$$results" || results=/dev/null; \
+## Prepare data for the test suite summary. These do not take into account
+## unreadable test results, but they'll be appropriately updated later if
+## needed.
+ all=` grep "^$$ws*:test-result:" $$results | wc -l`; \
+ pass=` grep "^$$ws*:test-result:$$ws*PASS" $$results | wc -l`; \
+ fail=` grep "^$$ws*:test-result:$$ws*FAIL" $$results | wc -l`; \
+ skip=` grep "^$$ws*:test-result:$$ws*SKIP" $$results | wc -l`; \
+ xfail=`grep "^$$ws*:test-result:$$ws*XFAIL" $$results | wc -l`; \
+ xpass=`grep "^$$ws*:test-result:$$ws*XPASS" $$results | wc -l`; \
+ error=`grep "^$$ws*:test-result:$$ws*ERROR" $$results | wc -l`; \
+## Whether the testsuite was successful or not.
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+## Make $br a line of exactly 76 `=' characters, that will be used to
+## enclose the testsuite summary report when displayed on the console.
+ br='==================='; br=$$br$$br$$br$$br; \
+## When writing the test summary to the console, we want to color a line
+## reporting the count of some result *only* if at least one test
+## experienced such a result. This function is handy in this regard.
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+## A shell function that creates the testsuite summary. We need it
+## because we have to create *two* summaries, one for test-suite.log,
+## and a possibly-colorized one for console output.
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
## Write "global" testsuite log.
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
-## Here we assume that the test driver writes a proper "summarizing
-## result" for the test script on the first line.
- for f in $$list; do \
- test -r $$f && read line < $$f || line=; \
- case $$line in \
- PASS:*|XFAIL:*);; \
- *) echo; cat $$f;; \
- esac; \
+ for b in $$bases; do \
+## FIXME: one fork per test -- this is horrendously inefficient!
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null; then :; \
+## If we cannot read the .log file of a test ...
+ elif test ! -r $$b.log; then \
+## ... print an error message to the console ...
+ echo "ERROR: cannot read $$b.log" >&2; \
+## ... and a warning in the `test-suite.log' file.
+ echo && echo "WARNING: could not read $$b.log!"; \
+ else \
+ echo && cat $$b.log; \
+ fi; \
done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
-## Emit the test summary on the console, and exit.
- $(am__tty_colors); \
- if $$exit; then \
+## Emit the test summary on the console.
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+## Multi line coloring is problematic with "less -R", so we really need
+## to color each line individually.
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+## This is expected to go to the console, so it might have to be colorized.
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
+ fi; \
+## Be sure to exit with the proper exit status.
+ $$success
RECHECK_LOGS = $(TEST_LOGS)
@@ -239,6 +285,7 @@ RECHECK_LOGS = $(TEST_LOGS)
check-TESTS:
## Expand $(RECHECK_LOGS) only once, to avoid exceeding line length limits.
@list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
+ @list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list
## We always have to remove TEST_SUITE_LOG, to ensure its rule is run
## in any case even in lazy mode: otherwise, if no test needs rerunning,
## or a prior run plus reruns all happen within the same timestamp
@@ -298,20 +345,18 @@ AM_RECURSIVE_TARGETS += check-html
## Rechecking failures. ##
## -------------------- ##
-## Rerun all FAILed or XPASSed tests.
+## Rerun all tests that experienced an error or an unexpected failure.
recheck recheck-html:
- @target=`echo $@ | sed 's,^re,,'`; \
- list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test -f $$f || continue; \
- if test -r $$f && read line < $$f; then \
-## Here we assume that the test driver writes a proper summary for the
-## test script on the first line. See the comments in the rules of
-## $(TEST_SUITE_LOG) above for why we consider this acceptable and even
-## advisable.
- case $$line in PASS:*|XFAIL:*|SKIP:*);; *) echo $$f;; esac; \
-## A test whose log is unreadable is to be considered failed.
- else echo $$f; fi; \
+ @ws='[ ]'; \
+ target=`echo $@ | sed 's,^re,,'`; \
+ bases='$(am__TEST_BASES)'; \
+ list=`for b in $$bases; do \
+## Skip tests that haven't been run, but recover gracefully from deleted
+## `.trs' files.
+ test ! -f $$b.trs && test ! -f $$b.log && continue; \
+## FIXME: one fork per test -- this is horrendously inefficient!
+ grep "^$$ws*:recheck:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null 2>&1 || echo $$b.log; \
done | tr '\012\015' ' '`; \
## This apparently useless munging helps to avoid a nasty bug (a
## segmentation fault!) on Solaris XPG4 make.
@@ -339,7 +384,7 @@ check-TESTS: $(TESTS)
## why we also try `dir='
elif test -f $$tst; then dir=; \
else dir="$(srcdir)/"; fi; \
- if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
+ if $(TESTS_ENVIRONMENT) $${dir}$$tst $(AM_TESTS_FD_REDIRECT); then \
## Success
all=`expr $$all + 1`; \
case " $(XFAIL_TESTS) " in \
diff --git a/lib/am/check2.am b/lib/am/check2.am
index b7ed0ee..ad0a4aa 100644
--- a/lib/am/check2.am
+++ b/lib/am/check2.am
@@ -14,12 +14,13 @@
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
-## From a test file to a log file.
+## From a test file to a .log and .trs file.
?GENERIC?%EXT%.log:
?!GENERIC?%OBJ%: %SOURCE%
- @p='%SOURCE%'; $(am__check_pre) \
- %DRIVER% $(am__test_driver_flags) %DRIVER_FLAGS% -- \
- %COMPILE% "$$tst"
+ @p='%SOURCE%'; $(am__check_pre) %DRIVER% --test-name "$$f" \
+ --log-file '%BASE%.log' --trs-file '%BASE%.trs' \
+ $(am__common_driver_flags) %DRIVER_FLAGS% -- %COMPILE% "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
## If no programs are built in this package, then this rule is removed
## at automake time. Otherwise, %am__EXEEXT% expands to a configure time
@@ -27,7 +28,8 @@
## conflict with the previous one.
if %am__EXEEXT%
?GENERIC?%EXT%$(EXEEXT).log:
- @p='%SOURCE%'; $(am__check_pre) \
- %DRIVER% $(am__test_driver_flags) %DRIVER_FLAGS% -- \
- %COMPILE% "$$tst"
+ @p='%SOURCE%'; $(am__check_pre) %DRIVER% --test-name "$$f" \
+ --log-file '%BASE%.log' --trs-file '%BASE%.trs' \
+ $(am__common_driver_flags) %DRIVER_FLAGS% -- %COMPILE% "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
endif %am__EXEEXT%
diff --git a/lib/am/configure.am b/lib/am/configure.am
index d00846a..594ec67 100644
--- a/lib/am/configure.am
+++ b/lib/am/configure.am
@@ -17,9 +17,10 @@
## This dummy rule is called from subdirectories whenever one of the
-## top-level Makefile's dependencies must be updated. It does not
-## need to depend on %MAKEFILE% because GNU make will always make sure
-## %MAKEFILE% is updated before considering the am--refresh target.
+## top-level Makefile's dependencies must be updated. It does depend
+## on %MAKEFILE% for the benefit of non-GNU make implementations (GNU
+## make will always make sure %MAKEFILE% is updated before considering
+## the am--refresh target anyway).
if %?TOPDIR_P%
.PHONY: am--refresh
am--refresh: %MAKEFILE%
diff --git a/lib/config-ml.in b/lib/config-ml.in
index 2dd0d51..1198346 100644
--- a/lib/config-ml.in
+++ b/lib/config-ml.in
@@ -2,7 +2,7 @@
# wanting multilib support.
#
# Copyright (C) 1995, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
-# 2005, 2006, 2007, 2008, 2010 Free Software Foundation, Inc.
+# 2005, 2006, 2007, 2008, 2010, 2011 Free Software Foundation, Inc.
#
# This file is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -224,19 +224,6 @@ done
# $host here, not $target.
case "${host}" in
-arc-*-elf*)
- if [ x$enable_biendian != xyes ]
- then
- old_multidirs=${multidirs}
- multidirs=""
- for x in ${old_multidirs}; do
- case "${x}" in
- *be*) : ;;
- *) multidirs="${multidirs} ${x}" ;;
- esac
- done
- fi
- ;;
arm-*-*)
if [ x"$enable_fpu" = xno ]
then
diff --git a/lib/config.guess b/lib/config.guess
index 4c8f032..b02565c 100755
--- a/lib/config.guess
+++ b/lib/config.guess
@@ -1,10 +1,10 @@
#! /bin/sh
# Attempt to guess a canonical system name.
# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
-# Free Software Foundation, Inc.
+# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011 Free Software Foundation, Inc.
-timestamp='2010-09-24'
+timestamp='2011-06-03'
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
@@ -57,7 +57,7 @@ GNU config.guess ($timestamp)
Originally written by Per Bothner.
Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free
+2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free
Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
@@ -92,7 +92,7 @@ if test $# != 0; then
exit 1
fi
-trap 'exit 1' HUP INT TERM
+trap 'exit 1' 1 2 15
# CC_FOR_BUILD -- compiler used by this script. Note that the use of a
# compiler to aid in system detection is discouraged as it requires
@@ -106,7 +106,7 @@ trap 'exit 1' HUP INT TERM
set_cc_for_build='
trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) &&
exit \$exitcode" 0 ;
-trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" HUP INT
PIPE TERM ;
+trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15
;
: ${TMPDIR=/tmp} ;
{ tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n
"$tmp" && test -d "$tmp" ; } ||
{ test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp)
; } ||
@@ -181,7 +181,7 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
fi
;;
*)
- os=netbsd
+ os=netbsd
;;
esac
# The OS release
@@ -224,7 +224,7 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'`
;;
*5.*)
- UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
+ UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'`
;;
esac
# According to Compaq, /usr/sbin/psrinfo has been available on
@@ -270,7 +270,10 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
# A Xn.n version is an unreleased experimental baselevel.
# 1.2 uses "1.2" for uname -r.
echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e
's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'`
- exit ;;
+ # Reset EXIT trap before exiting to avoid spurious non-zero exit code.
+ exitcode=$?
+ trap '' 0
+ exit $exitcode ;;
Alpha\ *:Windows_NT*:*)
# How do we know it's Interix rather than the generic POSIX subsystem?
# Should we change UNAME_MACHINE based on the output of uname instead
@@ -296,7 +299,7 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
echo s390-ibm-zvmoe
exit ;;
*:OS400:*:*)
- echo powerpc-ibm-os400
+ echo powerpc-ibm-os400
exit ;;
arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
echo arm-acorn-riscix${UNAME_RELEASE}
@@ -395,23 +398,23 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
# MiNT. But MiNT is downward compatible to TOS, so this should
# be no problem.
atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*)
- echo m68k-atari-mint${UNAME_RELEASE}
+ echo m68k-atari-mint${UNAME_RELEASE}
exit ;;
atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*)
echo m68k-atari-mint${UNAME_RELEASE}
- exit ;;
+ exit ;;
*falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*)
- echo m68k-atari-mint${UNAME_RELEASE}
+ echo m68k-atari-mint${UNAME_RELEASE}
exit ;;
milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*)
- echo m68k-milan-mint${UNAME_RELEASE}
- exit ;;
+ echo m68k-milan-mint${UNAME_RELEASE}
+ exit ;;
hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*)
- echo m68k-hades-mint${UNAME_RELEASE}
- exit ;;
+ echo m68k-hades-mint${UNAME_RELEASE}
+ exit ;;
*:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*)
- echo m68k-unknown-mint${UNAME_RELEASE}
- exit ;;
+ echo m68k-unknown-mint${UNAME_RELEASE}
+ exit ;;
m68k:machten:*:*)
echo m68k-apple-machten${UNAME_RELEASE}
exit ;;
@@ -481,8 +484,8 @@ EOF
echo m88k-motorola-sysv3
exit ;;
AViiON:dgux:*:*)
- # DG/UX returns AViiON for all architectures
- UNAME_PROCESSOR=`/usr/bin/uname -p`
+ # DG/UX returns AViiON for all architectures
+ UNAME_PROCESSOR=`/usr/bin/uname -p`
if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ]
then
if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \
@@ -495,7 +498,7 @@ EOF
else
echo i586-dg-dgux${UNAME_RELEASE}
fi
- exit ;;
+ exit ;;
M88*:DolphinOS:*:*) # DolphinOS (SVR3)
echo m88k-dolphin-sysv3
exit ;;
@@ -595,52 +598,52 @@ EOF
9000/[678][0-9][0-9])
if [ -x /usr/bin/getconf ]; then
sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null`
- sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS
2>/dev/null`
- case "${sc_cpu_version}" in
- 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
- 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
- 532) # CPU_PA_RISC2_0
- case "${sc_kernel_bits}" in
- 32) HP_ARCH="hppa2.0n" ;;
- 64) HP_ARCH="hppa2.0w" ;;
+ sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null`
+ case "${sc_cpu_version}" in
+ 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0
+ 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1
+ 532) # CPU_PA_RISC2_0
+ case "${sc_kernel_bits}" in
+ 32) HP_ARCH="hppa2.0n" ;;
+ 64) HP_ARCH="hppa2.0w" ;;
'') HP_ARCH="hppa2.0" ;; # HP-UX 10.20
- esac ;;
- esac
+ esac ;;
+ esac
fi
if [ "${HP_ARCH}" = "" ]; then
eval $set_cc_for_build
- sed 's/^ //' << EOF >$dummy.c
+ sed 's/^ //' << EOF >$dummy.c
- #define _HPUX_SOURCE
- #include <stdlib.h>
- #include <unistd.h>
+ #define _HPUX_SOURCE
+ #include <stdlib.h>
+ #include <unistd.h>
- int main ()
- {
- #if defined(_SC_KERNEL_BITS)
- long bits = sysconf(_SC_KERNEL_BITS);
- #endif
- long cpu = sysconf (_SC_CPU_VERSION);
+ int main ()
+ {
+ #if defined(_SC_KERNEL_BITS)
+ long bits = sysconf(_SC_KERNEL_BITS);
+ #endif
+ long cpu = sysconf (_SC_CPU_VERSION);
- switch (cpu)
- {
- case CPU_PA_RISC1_0: puts ("hppa1.0"); break;
- case CPU_PA_RISC1_1: puts ("hppa1.1"); break;
- case CPU_PA_RISC2_0:
- #if defined(_SC_KERNEL_BITS)
- switch (bits)
- {
- case 64: puts ("hppa2.0w"); break;
- case 32: puts ("hppa2.0n"); break;
- default: puts ("hppa2.0"); break;
- } break;
- #else /* !defined(_SC_KERNEL_BITS) */
- puts ("hppa2.0"); break;
- #endif
- default: puts ("hppa1.0"); break;
- }
- exit (0);
- }
+ switch (cpu)
+ {
+ case CPU_PA_RISC1_0: puts ("hppa1.0"); break;
+ case CPU_PA_RISC1_1: puts ("hppa1.1"); break;
+ case CPU_PA_RISC2_0:
+ #if defined(_SC_KERNEL_BITS)
+ switch (bits)
+ {
+ case 64: puts ("hppa2.0w"); break;
+ case 32: puts ("hppa2.0n"); break;
+ default: puts ("hppa2.0"); break;
+ } break;
+ #else /* !defined(_SC_KERNEL_BITS) */
+ puts ("hppa2.0"); break;
+ #endif
+ default: puts ("hppa1.0"); break;
+ }
+ exit (0);
+ }
EOF
(CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) &&
HP_ARCH=`$dummy`
test -z "$HP_ARCH" && HP_ARCH=hppa
@@ -731,22 +734,22 @@ EOF
exit ;;
C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
echo c1-convex-bsd
- exit ;;
+ exit ;;
C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
if getsysinfo -f scalar_acc
then echo c32-convex-bsd
else echo c2-convex-bsd
fi
- exit ;;
+ exit ;;
C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
echo c34-convex-bsd
- exit ;;
+ exit ;;
C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
echo c38-convex-bsd
- exit ;;
+ exit ;;
C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
echo c4-convex-bsd
- exit ;;
+ exit ;;
CRAY*Y-MP:*:*:*)
echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/'
exit ;;
@@ -770,14 +773,14 @@ EOF
exit ;;
F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*)
FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'abcdefghijklmnopqrstuvwxyz'`
- FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
- FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
- echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
- exit ;;
+ FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
+ FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'`
+ echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+ exit ;;
5000:UNIX_System_V:4.*:*)
- FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
- FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
- echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
+ FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'`
+ FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'`
+ echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}"
exit ;;
i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*)
echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE}
@@ -805,14 +808,14 @@ EOF
echo ${UNAME_MACHINE}-pc-mingw32
exit ;;
i*:windows32*:*)
- # uname -m includes "-pc" on this system.
- echo ${UNAME_MACHINE}-mingw32
+ # uname -m includes "-pc" on this system.
+ echo ${UNAME_MACHINE}-mingw32
exit ;;
i*:PW*:*)
echo ${UNAME_MACHINE}-pc-pw32
exit ;;
*:Interix*:*)
- case ${UNAME_MACHINE} in
+ case ${UNAME_MACHINE} in
x86)
echo i586-pc-interix${UNAME_RELEASE}
exit ;;
@@ -867,7 +870,7 @@ EOF
EV6) UNAME_MACHINE=alphaev6 ;;
EV67) UNAME_MACHINE=alphaev67 ;;
EV68*) UNAME_MACHINE=alphaev68 ;;
- esac
+ esac
objdump --private-headers /bin/sh | grep -q ld.so.1
if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
@@ -879,7 +882,13 @@ EOF
then
echo ${UNAME_MACHINE}-unknown-linux-gnu
else
- echo ${UNAME_MACHINE}-unknown-linux-gnueabi
+ if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
+ | grep -q __ARM_PCS_VFP
+ then
+ echo ${UNAME_MACHINE}-unknown-linux-gnueabi
+ else
+ echo ${UNAME_MACHINE}-unknown-linux-gnueabihf
+ fi
fi
exit ;;
avr32*:Linux:*:*)
@@ -892,7 +901,7 @@ EOF
echo crisv32-axis-linux-gnu
exit ;;
frv:Linux:*:*)
- echo frv-unknown-linux-gnu
+ echo frv-unknown-linux-gnu
exit ;;
i*86:Linux:*:*)
LIBC=gnu
@@ -960,7 +969,7 @@ EOF
echo ${UNAME_MACHINE}-ibm-linux
exit ;;
sh64*:Linux:*:*)
- echo ${UNAME_MACHINE}-unknown-linux-gnu
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
exit ;;
sh*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
@@ -969,7 +978,7 @@ EOF
echo ${UNAME_MACHINE}-unknown-linux-gnu
exit ;;
tile*:Linux:*:*)
- echo ${UNAME_MACHINE}-tilera-linux-gnu
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
exit ;;
vax:Linux:*:*)
echo ${UNAME_MACHINE}-dec-linux-gnu
@@ -978,7 +987,7 @@ EOF
echo x86_64-unknown-linux-gnu
exit ;;
xtensa*:Linux:*:*)
- echo ${UNAME_MACHINE}-unknown-linux-gnu
+ echo ${UNAME_MACHINE}-unknown-linux-gnu
exit ;;
i*86:DYNIX/ptx:4*:*)
# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.
@@ -987,11 +996,11 @@ EOF
echo i386-sequent-sysv4
exit ;;
i*86:UNIX_SV:4.2MP:2.*)
- # Unixware is an offshoot of SVR4, but it has its own version
- # number series starting with 2...
- # I am not positive that other SVR4 systems won't match this,
+ # Unixware is an offshoot of SVR4, but it has its own version
+ # number series starting with 2...
+ # I am not positive that other SVR4 systems won't match this,
# I just have to hope. -- rms.
- # Use sysv4.2uw... so that sysv4* matches it.
+ # Use sysv4.2uw... so that sysv4* matches it.
echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION}
exit ;;
i*86:OS/2:*:*)
@@ -1023,7 +1032,7 @@ EOF
fi
exit ;;
i*86:*:5:[678]*)
- # UnixWare 7.x, OpenUNIX and OpenServer 6.
+ # UnixWare 7.x, OpenUNIX and OpenServer 6.
case `/bin/uname -X | grep "^Machine"` in
*486*) UNAME_MACHINE=i486 ;;
*Pentium) UNAME_MACHINE=i586 ;;
@@ -1051,13 +1060,13 @@ EOF
exit ;;
pc:*:*:*)
# Left here for compatibility:
- # uname -m prints for DJGPP always 'pc', but it prints nothing about
- # the processor, so we play safe by assuming i586.
+ # uname -m prints for DJGPP always 'pc', but it prints nothing about
+ # the processor, so we play safe by assuming i586.
# Note: whatever this is, it MUST be the same as what config.sub
# prints for the "djgpp" host, or else GDB configury will decide that
# this is a cross-build.
echo i586-pc-msdosdjgpp
- exit ;;
+ exit ;;
Intel:Mach:3*:*)
echo i386-pc-mach3
exit ;;
@@ -1092,8 +1101,8 @@ EOF
/bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \
&& { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;;
3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
- /bin/uname -p 2>/dev/null | grep 86 >/dev/null \
- && { echo i486-ncr-sysv4; exit; } ;;
+ /bin/uname -p 2>/dev/null | grep 86 >/dev/null \
+ && { echo i486-ncr-sysv4; exit; } ;;
NCR*:*:4.2:* | MPRAS*:*:4.2:*)
OS_REL='.3'
test -r /etc/.relid \
@@ -1136,10 +1145,10 @@ EOF
echo ns32k-sni-sysv
fi
exit ;;
- PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort
- # says <address@hidden>
- echo i586-unisys-sysv4
- exit ;;
+ PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort
+ # says <address@hidden>
+ echo i586-unisys-sysv4
+ exit ;;
*:UNIX_System_V:4*:FTX*)
# From Gerald Hewes <address@hidden>.
# How about differentiating between stratus architectures? -djm
@@ -1165,11 +1174,11 @@ EOF
exit ;;
R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*)
if [ -d /usr/nec ]; then
- echo mips-nec-sysv${UNAME_RELEASE}
+ echo mips-nec-sysv${UNAME_RELEASE}
else
- echo mips-unknown-sysv${UNAME_RELEASE}
+ echo mips-unknown-sysv${UNAME_RELEASE}
fi
- exit ;;
+ exit ;;
BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only.
echo powerpc-be-beos
exit ;;
@@ -1282,13 +1291,13 @@ EOF
echo pdp10-unknown-its
exit ;;
SEI:*:*:SEIUX)
- echo mips-sei-seiux${UNAME_RELEASE}
+ echo mips-sei-seiux${UNAME_RELEASE}
exit ;;
*:DragonFly:*:*)
echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e
's/[-(].*//'`
exit ;;
*:*VMS:*:*)
- UNAME_MACHINE=`(uname -p) 2>/dev/null`
+ UNAME_MACHINE=`(uname -p) 2>/dev/null`
case "${UNAME_MACHINE}" in
A*) echo alpha-dec-vms ; exit ;;
I*) echo ia64-dec-vms ; exit ;;
@@ -1328,11 +1337,11 @@ main ()
#include <sys/param.h>
printf ("m68k-sony-newsos%s\n",
#ifdef NEWSOS4
- "4"
+ "4"
#else
- ""
+ ""
#endif
- ); exit (0);
+ ); exit (0);
#endif
#endif
diff --git a/lib/config.sub b/lib/config.sub
index 3ec94f1..f9fcdc8 100755
--- a/lib/config.sub
+++ b/lib/config.sub
@@ -1,10 +1,10 @@
#! /bin/sh
# Configuration validation subroutine script.
# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
-# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
-# Free Software Foundation, Inc.
+# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
+# 2011 Free Software Foundation, Inc.
-timestamp='2010-12-11'
+timestamp='2011-06-03'
# This file is (in principle) common to ALL GNU software.
# The presence of a machine in this file suggests that SOME GNU software
@@ -76,7 +76,7 @@ version="\
GNU config.sub ($timestamp)
Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free
+2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free
Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
@@ -158,8 +158,8 @@ case $os in
os=
basic_machine=$1
;;
- -bluegene*)
- os=-cnk
+ -bluegene*)
+ os=-cnk
;;
-sim | -cisco | -oki | -wec | -winbond)
os=
@@ -175,10 +175,10 @@ case $os in
os=-chorusos
basic_machine=$1
;;
- -chorusrdb)
- os=-chorusrdb
+ -chorusrdb)
+ os=-chorusrdb
basic_machine=$1
- ;;
+ ;;
-hiux*)
os=-hiuxwe2
;;
@@ -286,9 +286,10 @@ case $basic_machine in
| nds32 | nds32le | nds32be \
| nios | nios2 \
| ns16k | ns32k \
+ | open8 \
| or32 \
| pdp10 | pdp11 | pj | pjl \
- | powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \
+ | powerpc | powerpc64 | powerpc64le | powerpcle \
| pyramid \
| rx \
| score \
@@ -296,12 +297,12 @@ case $basic_machine in
| sh64 | sh64le \
| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet |
sparclite \
| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
- | spu | strongarm \
- | tahoe | thumb | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
+ | spu \
+ | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
| ubicom32 \
- | v850 | v850e \
+ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
| we32k \
- | x86 | xc16x | xscale | xscalee[bl] | xstormy16 | xtensa \
+ | x86 | xc16x | xstormy16 | xtensa \
| z8k | z80)
basic_machine=$basic_machine-unknown
;;
@@ -325,6 +326,18 @@ case $basic_machine in
basic_machine=mt-unknown
;;
+ strongarm | thumb | xscale)
+ basic_machine=arm-unknown
+ ;;
+
+ xscaleeb)
+ basic_machine=armeb-unknown
+ ;;
+
+ xscaleel)
+ basic_machine=armel-unknown
+ ;;
+
# We use `pc' rather than `unknown'
# because (1) that's what they normally are, and
# (2) the word "unknown" tends to confuse beginning users.
@@ -382,24 +395,26 @@ case $basic_machine in
| nds32-* | nds32le-* | nds32be-* \
| nios-* | nios2-* \
| none-* | np1-* | ns16k-* | ns32k-* \
+ | open8-* \
| orion-* \
| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
- | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \
+ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
| pyramid-* \
| romp-* | rs6000-* | rx-* \
| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-*
| sheb-* | shbe-* \
| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* |
sparclet-* \
| sparclite-* \
- | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | strongarm-* | sv1-*
| sx?-* \
- | tahoe-* | thumb-* \
+ | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \
+ | tahoe-* \
| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
- | tile-* | tilegx-* \
+ | tile*-* \
| tron-* \
| ubicom32-* \
- | v850-* | v850e-* | vax-* \
+ | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
+ | vax-* \
| we32k-* \
- | x86-* | x86_64-* | xc16x-* | xps100-* | xscale-* | xscalee[bl]-* \
+ | x86-* | x86_64-* | xc16x-* | xps100-* \
| xstormy16-* | xtensa*-* \
| ymp-* \
| z8k-* | z80-*)
@@ -424,7 +439,7 @@ case $basic_machine in
basic_machine=a29k-amd
os=-udi
;;
- abacus)
+ abacus)
basic_machine=abacus-unknown
;;
adobe68k)
@@ -507,7 +522,7 @@ case $basic_machine in
basic_machine=c90-cray
os=-unicos
;;
- cegcc)
+ cegcc)
basic_machine=arm-unknown
os=-cegcc
;;
@@ -755,7 +770,7 @@ case $basic_machine in
basic_machine=ns32k-utek
os=-sysv
;;
- microblaze)
+ microblaze)
basic_machine=microblaze-xilinx
;;
mingw32)
@@ -862,10 +877,10 @@ case $basic_machine in
np1)
basic_machine=np1-gould
;;
- neo-tandem)
+ neo-tandem)
basic_machine=neo-tandem
;;
- nse-tandem)
+ nse-tandem)
basic_machine=nse-tandem
;;
nsr-tandem)
@@ -950,9 +965,10 @@ case $basic_machine in
;;
power) basic_machine=power-ibm
;;
- ppc) basic_machine=powerpc-unknown
+ ppc | ppcbe) basic_machine=powerpc-unknown
;;
- ppc-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ppc-* | ppcbe-*)
+ basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'`
;;
ppcle | powerpclittle | ppc-le | powerpc-little)
basic_machine=powerpcle-unknown
@@ -1046,6 +1062,9 @@ case $basic_machine in
basic_machine=i860-stratus
os=-sysv4
;;
+ strongarm-* | thumb-*)
+ basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'`
+ ;;
sun2)
basic_machine=m68000-sun
;;
@@ -1102,13 +1121,8 @@ case $basic_machine in
basic_machine=t90-cray
os=-unicos
;;
- # This must be matched before tile*.
- tilegx*)
- basic_machine=tilegx-unknown
- os=-linux-gnu
- ;;
tile*)
- basic_machine=tile-unknown
+ basic_machine=$basic_machine-unknown
os=-linux-gnu
;;
tx39)
@@ -1178,6 +1192,9 @@ case $basic_machine in
xps | xps100)
basic_machine=xps100-honeywell
;;
+ xscale-* | xscalee[bl]-*)
+ basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'`
+ ;;
ymp)
basic_machine=ymp-cray
os=-unicos
@@ -1275,11 +1292,11 @@ esac
if [ x"$os" != x"" ]
then
case $os in
- # First match some system type aliases
- # that might get confused with valid system types.
+ # First match some system type aliases
+ # that might get confused with valid system types.
# -solaris* is a basic system type, with this one exception.
- -auroraux)
- os=-auroraux
+ -auroraux)
+ os=-auroraux
;;
-solaris1 | -solaris1.*)
os=`echo $os | sed -e 's|solaris1|sunos4|'`
@@ -1364,7 +1381,7 @@ case $os in
-opened*)
os=-openedition
;;
- -os400*)
+ -os400*)
os=-os400
;;
-wince*)
@@ -1413,7 +1430,7 @@ case $os in
-sinix*)
os=-sysv4
;;
- -tpf*)
+ -tpf*)
os=-tpf
;;
-triton*)
@@ -1458,8 +1475,8 @@ case $os in
-dicos*)
os=-dicos
;;
- -nacl*)
- ;;
+ -nacl*)
+ ;;
-none)
;;
*)
@@ -1482,10 +1499,10 @@ else
# system, and we'll never get to this point.
case $basic_machine in
- score-*)
+ score-*)
os=-elf
;;
- spu-*)
+ spu-*)
os=-elf
;;
*-acorn)
@@ -1497,8 +1514,8 @@ case $basic_machine in
arm*-semi)
os=-aout
;;
- c4x-* | tic4x-*)
- os=-coff
+ c4x-* | tic4x-*)
+ os=-coff
;;
tic54x-*)
os=-coff
@@ -1534,7 +1551,7 @@ case $basic_machine in
m68*-cisco)
os=-aout
;;
- mep-*)
+ mep-*)
os=-elf
;;
mips*-cisco)
@@ -1561,7 +1578,7 @@ case $basic_machine in
*-ibm)
os=-aix
;;
- *-knuth)
+ *-knuth)
os=-mmixware
;;
*-wec)
diff --git a/lib/tap-driver b/lib/tap-driver
new file mode 100755
index 0000000..3facdfd
--- /dev/null
+++ b/lib/tap-driver
@@ -0,0 +1,426 @@
+#! /usr/bin/env perl
+# Temporary/experimental TAP test driver for Automake.
+# TODO: should be rewritten portably (e.g., in awk or shell).
+
+# ---------------------------------- #
+# Imports, static data, and setup. #
+# ---------------------------------- #
+
+use warnings FATAL => 'all';
+use strict;
+use Getopt::Long ();
+use TAP::Parser;
+
+my $ME = "tap-driver";
+
+my $USAGE = <<'END';
+Usage:
+ tap-driver --test-name=NAME --log-file=PATH --trs-file=PATH
+ [--expect-failure={yes|no}] [--color-tests={yes|no}]
+ [--enable-hard-errors={yes|no}] [--merge|--no-merge]
+ [--ignore-exit] [--comments|--no-comments] [--] TEST-COMMAND
+The `--test-name' and `--log-file' options are mandatory.
+END
+
+my $HELP = "$ME: TAP-aware test driver for Automake testsuite harness." .
+ "\n" . $USAGE;
+
+my $VERSION = '(experimental version)';
+
+# Keep this in sync with `lib/am/check.am:$(am__tty_colors)'.
+my %COLOR = (
+ red => "\e[0;31m",
+ grn => "\e[0;32m",
+ lgn => "\e[1;32m",
+ blu => "\e[1;34m",
+ mgn => "\e[0;35m",
+ brg => "\e[1m",
+ std => "\e[m",
+);
+
+# ------------------- #
+# Global variables. #
+# ------------------- #
+
+my $lineno = 0; # Number of input lines seen so far.
+my $testno = 0; # Number of test results seen so far.
+my $plan_seen = 0; # Whether the TAP plan has been seen or not.
+my $parser; # TAP parser object (will be initialized later).
+
+# ----------------- #
+# Option parsing. #
+# ----------------- #
+
+my %cfg = (
+ "color-tests" => 0,
+ "expect-failure" => 0,
+ "enable-hard-errors" => 1,
+ "merge" => 0,
+ "comments" => 0,
+ "ignore-exit" => 0,
+);
+
+my $test_script_name = undef;
+my $log_file = undef;
+my $trs_file = undef;
+
+Getopt::Long::GetOptions (
+ 'help' => sub { print $HELP; exit 0; },
+ 'version' => sub { print "$ME $VERSION"; exit 0; },
+ 'test-name=s' => \$test_script_name,
+ 'log-file=s' => \$log_file,
+ 'trs-file=s' => \$trs_file,
+ 'color-tests=s' => \&bool_opt,
+ 'expect-failure=s' => \&bool_opt,
+ 'enable-hard-errors=s' => \&bool_opt,
+ 'comments' => sub { $cfg{"comments"} = 1; },
+ 'no-comments' => sub { $cfg{"comments"} = 0; },
+ 'merge' => sub { $cfg{"merge"} = 1; },
+ 'no-merge' => sub { $cfg{"merge"} = 0; },
+ 'ignore-exit' => sub { $cfg{"ignore-exit"} = 1; },
+ ) or exit 1;
+
+# -------------- #
+# Subroutines. #
+# -------------- #
+
+sub bool_opt ($$)
+{
+ my ($opt, $val) = @_;
+ if ($val =~ /^(?:y|yes)\z/i)
+ {
+ $cfg{$opt} = 1;
+ }
+ elsif ($val =~ /^(?:n|no)\z/i)
+ {
+ $cfg{$opt} = 0;
+ }
+ else
+ {
+ die "invalid argument '$val' for option '$opt'\n";
+ }
+}
+
+TAP_PEEKING :
+{
+ my @tap_lines = ();
+
+ sub get_tap_line ()
+ {
+ @tap_lines > 0 ? pop @tap_lines : $parser->next;
+ }
+
+ sub unget_tap_line ($)
+ {
+ push @tap_lines, @_;
+ }
+
+ sub peek_tap_line ()
+ {
+ my $res = get_tap_line;
+ unget_tap_line ($res);
+ return $res;
+ }
+
+}
+
+TEST_RESULTS :
+{
+ my (@test_results, %test_results);
+
+ sub add_test_result ($)
+ {
+ my $res = shift;
+ push @test_results, $res;
+ $test_results{$res} = 1;
+ }
+
+ sub get_test_results ()
+ {
+ return @test_results;
+ }
+
+ # Whether the test script should be re-run by "make recheck".
+ sub must_recheck ()
+ {
+ return grep { !/^(?:XFAIL|PASS|SKIP)$/ } (keys %test_results);
+ }
+
+ # Whether the content of the log file associated to this test should
+ # be copied into the "global" test-suite.log.
+ sub copy_in_global_log ()
+ {
+ return grep { not $_ eq "PASS" } (keys %test_results);
+ }
+
+ # FIXME: this can certainly be improved ...
+ sub get_global_test_result ()
+ {
+ my @results = keys %test_results;
+ return "ERROR" if exists $test_results{"ERROR"};
+ return "SKIP" if @results == 1 && $results[0] eq "SKIP";
+ return "FAIL" if exists $test_results{"FAIL"};
+ return "FAIL" if exists $test_results{"XPASS"};
+ return "PASS";
+ }
+
+}
+
+sub write_test_results ()
+{
+ open RES, ">", $trs_file or die "opening $trs_file: $!\n";
+ print RES ":recheck: " .
+ (must_recheck ? "yes" : "no") . "\n";
+ print RES ":copy-in-global-log: " .
+ (copy_in_global_log ? "yes" : "no") . "\n";
+ foreach my $result (get_test_results)
+ {
+ print RES ":test-result: $result\n";
+ }
+ close RES or die "closing $trs_file: $!\n";
+}
+
+sub start (@)
+{
+ # Redirect stderr and stdout to a temporary log file. Save the
+ # original stdout stream, since we need it to print testsuite
+ # progress output.
+ open OLDOUT, ">&STDOUT" or die "duplicating stdout: $!\n";
+ open TMP, ">$log_file-t" or die "opening $log_file-t: $!\n";
+ open STDOUT, ">&TMP" or die "redirecting stdout: $!\n";
+ open STDERR, ">&TMP" or die "redirecting stderr: $!\n";
+ $parser = TAP::Parser->new ({ exec => address@hidden, merge => $cfg{merge}
});
+ $parser->ignore_exit(1) if $cfg{"ignore-exit"};
+}
+
+sub finish ()
+{
+ write_test_results;
+
+ open LOG, ">", $log_file or die "opening $log_file: $!\n";
+ my $global_result = get_global_test_result;
+ my $global_result_line = "$global_result: $test_script_name";
+ print LOG "$global_result_line\n";
+ print LOG "=" x length ($global_result_line) . "\n";
+ print LOG "\n";
+
+ close TMP or die "closing $log_file-t: $!\n";
+ # FIXME: remove this hack!
+ my $test_output = `cat $log_file-t && rm -f $log_file-t`;
+ print LOG $test_output;
+ close LOG or die "closing $log_file: $!\n";
+
+ exit 0;
+}
+
+sub stringify_test_result ($)
+{
+ my $result = shift;
+ my $PASS = $cfg{"expect-failure"} ? "XPASS": "PASS";
+ my $FAIL = $cfg{"expect-failure"} ? "XFAIL": "FAIL";
+ if ($result->is_unplanned || $result->number != $testno)
+ {
+ return "ERROR";
+ }
+ elsif (!$result->directive)
+ {
+ return $result->is_ok ? $PASS: $FAIL;
+ }
+ elsif ($result->has_todo)
+ {
+ return $result->is_actual_ok ? "XPASS" : "XFAIL";
+ }
+ elsif ($result->has_skip)
+ {
+ return $result->is_ok ? "SKIP" : $FAIL;
+ }
+ die "INTERNAL ERROR"; # NOTREACHED
+}
+
+sub colored ($$)
+{
+ my ($color_name, $text) = @_;
+ return $COLOR{$color_name} . $text . $COLOR{'std'};
+}
+
+sub decorate_result ($)
+{
+ my $result = shift;
+ return $result unless $cfg{"color-tests"};
+ my %color_for_result =
+ (
+ "ERROR" => 'mgn',
+ "PASS" => 'grn',
+ "XPASS" => 'red',
+ "FAIL" => 'red',
+ "XFAIL" => 'lgn',
+ "SKIP" => 'blu',
+ );
+ if (my $color = $color_for_result{$result})
+ {
+ return colored ($color, $result);
+ }
+ else
+ {
+ return $result; # Don't colorize unknown stuff.
+ }
+}
+
+sub report ($;$)
+{
+ my ($msg, $result, $explanation) = (undef, @_);
+ if ($result =~ /^(?:X?(?:PASS|FAIL)|SKIP|ERROR)/)
+ {
+ $msg = ": $test_script_name";
+ add_test_result $result;
+ }
+ elsif ($result eq "#")
+ {
+ $msg = " $test_script_name:";
+ }
+ else
+ {
+ die "INTERNAL ERROR"; # NOTREACHED
+ }
+ $msg .= " $explanation" if defined $explanation;
+ $msg .= "\n";
+ # Output on console might be colorized.
+ print OLDOUT decorate_result ($result) . $msg;
+ # Log the result in the log file too, to help debugging (this is
+ # especially true when said result is a TAP error or "Bail out!").
+ print $result . $msg;
+}
+
+sub testuite_error ($)
+{
+ report "ERROR", "- $_[0]";
+}
+
+sub handle_tap_test ($)
+{
+ $testno++;
+ my $test = shift;
+
+ my $test_result = stringify_test_result $test;
+ my $string = $test->number;
+
+ if (my $description = $test->description)
+ {
+ $string .= " $description";
+ }
+ if ($test->is_unplanned)
+ {
+ $string .= " # UNPLANNED";
+ }
+ elsif ($test->number != $testno)
+ {
+ $string .= " # OUT-OF-ORDER (expecting $testno)";
+ }
+ elsif (my $directive = $test->directive)
+ {
+ $string .= " # $directive";
+ if (my $explanation = $test->explanation)
+ {
+ $string .= " $explanation";
+ }
+ }
+ report $test_result, $string;
+}
+
+sub handle_tap_plan ($)
+{
+ my $plan = shift;
+ testuite_error "multiple test plans" if $plan_seen;
+ $plan_seen = 1;
+ # TAP plan must be either in the first or in the last line.
+ if ($lineno > 1 && peek_tap_line)
+ {
+ testuite_error "test plan in middle of output";
+ return;
+ }
+ # Nothing more to do, unless the plan contains a SKIP directive.
+ return
+ if not defined $plan->directive && length ($plan->directive) > 0;
+ my $explanation = $plan->explanation ?
+ "- " . $plan->explanation : undef;
+ report "SKIP", $explanation;
+ finish;
+}
+
+sub handle_tap_bailout ($)
+{
+ my ($bailout, $msg) = ($_[0], "Bail out!");
+ $msg .= " " . $bailout->explanation if $bailout->explanation;
+ testuite_error $msg;
+ finish;
+}
+
+sub handle_tap_comment ($)
+{
+ return unless $cfg{comments};
+ my $comment = $_[0]->comment;
+ report "#", "$comment" if length $comment;
+}
+
+sub main (@)
+{
+ start @_;
+
+ while (defined (my $cur = get_tap_line))
+ {
+ # Verbatim copy any input line into the log file.
+ print $cur->raw . "\n";
+ $lineno++;
+ if ($cur->is_plan)
+ {
+ handle_tap_plan ($cur);
+ }
+ elsif ($cur->is_test)
+ {
+ handle_tap_test ($cur);
+ }
+ elsif ($cur->is_comment)
+ {
+ handle_tap_comment ($cur);
+ }
+ elsif ($cur->is_bailout)
+ {
+ handle_tap_bailout ($cur);
+ }
+ }
+ if (!$plan_seen)
+ {
+ testuite_error "missing test plan";
+ }
+ elsif ($parser->tests_planned != $parser->tests_run)
+ {
+ my ($planned, $run) = ($parser->tests_planned, $parser->tests_run);
+ my $bad_amount = $run > $planned ? "many" : "few";
+ testuite_error (sprintf "too %s tests run (expected %d, got %d)",
+ $bad_amount, $planned, $run);
+ }
+ # TODO: we should probably use $parser->wait here, to catch signals too
+ if ($parser->exit != 0)
+ {
+ my $msg = sprintf "exited with status %d", $parser->exit;
+ if ($cfg{"ignore-exit"})
+ {
+ # Log the exit status of the script anyway, even if it is not
+ # considered to be an error, to help debugging.
+ print "INFO: $test_script_name - $msg\n";
+ }
+ else
+ {
+ testuite_error $msg;;
+ }
+ }
+ finish;
+}
+
+# ----------- #
+# Main code. #
+# ----------- #
+
+main @ARGV;
+
+# vim: ft=perl ts=4 sw=4 et
diff --git a/lib/test-driver b/lib/test-driver
index 31640c3..472ba49 100755
--- a/lib/test-driver
+++ b/lib/test-driver
@@ -1,7 +1,7 @@
#! /bin/sh
# test-driver - basic driver script for the `parallel-tests' mode.
-scriptversion=2011-06-21.19; # UTC
+scriptversion=2011-08-01.21; # UTC
# Copyright (C) 2011 Free Software Foundation, Inc.
#
@@ -48,7 +48,7 @@ print_usage ()
{
cat <<END
Usage:
- test-driver [--help|--version] --test-name=NAME --log-file=PATH
+ test-driver --test-name=NAME --log-file=PATH --trs-file=PATH
[--expect-failure={yes|no}] [--color-tests={yes|no}]
[--enable-hard-errors={yes|no}] [--] TEST-SCRIPT
The \`--test-name' and \`--log-file' options are mandatory.
@@ -59,9 +59,10 @@ END
rst_section () { sed 'p;s/./=/g;p;g'; }
# TODO: better error handling in option parsing (in particular, ensure
-# TODO: $logfile and $test_name are defined).
+# TODO: $logfile, $trsfile and $test_name are defined).
test_name= # Used for reporting.
-logfile= # Where to save the result and output of the test script.
+logfile= # Where to save the output of the test script.
+trsfile= # Where to save the result(s) the test script.
expect_failure=no
color_tests=no
enable_hard_errors=yes
@@ -71,6 +72,7 @@ while test $# -gt 0; do
--version) echo "test-driver $scriptversion"; exit $?;;
--test-name) test_name=$2; shift;;
--log-file) logfile=$2; shift;;
+ --trs-file) trsfile=$2; shift;;
--color-tests) color_tests=$2; shift;;
--expect-failure) expect_failure=$2; shift;;
--enable-hard-errors) enable_hard_errors=$2; shift;;
@@ -85,9 +87,10 @@ if test $color_tests = yes; then
grn='[0;32m' # Green.
lgn='[1;32m' # Light green.
blu='[1;34m' # Blue.
+ mgn='[0;35m' # Magenta.
std='[m' # No color.
else
- red= grn= lgn= blu= std=
+ red= grn= lgn= blu= mgn= std=
fi
tmpfile=$logfile-t
@@ -106,25 +109,25 @@ if test $enable_hard_errors = no && test $estatus -eq 99;
then
fi
case $estatus:$expect_failure in
- 0:yes) col=$red; res=XPASS;;
- 0:*) col=$grn; res=PASS ;;
- 77:*) col=$blu; res=SKIP ;;
- 99:*) col=$red; res=FAIL ;;
- *:yes) col=$lgn; res=XFAIL;;
- *:*) col=$red; res=FAIL ;;
+ 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;;
+ 0:*) col=$grn res=PASS recheck=no gcopy=no;;
+ 77:*) col=$blu res=SKIP recheck=no gcopy=yes;;
+ 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;;
+ *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;;
+ *:*) col=$red res=FAIL recheck=yes gcopy=yes;;
esac
# Report outcome to console.
echo "${col}${res}${std}: $test_name"
+# Register the test result, and other relevant metadata.
+echo ":test-result: $res (exit status: $estatus)" > $trsfile
+echo ":recheck: $recheck" >> $trsfile
+echo ":copy-in-global-log: $gcopy" >> $trsfile
+
# Now write log file.
{
echo "$res: $test_name (exit: $estatus)" | rst_section
- echo ":test-result: $res (exit status: $estatus)"
- # Use a reStructuredText transition to better separate the test
- # outcome report from its registered output.
- echo
- echo ------------
echo
cat $tmpfile
} > $logfile
diff --git a/lib/texinfo.tex b/lib/texinfo.tex
index 1130b8f..fd22fd6 100644
--- a/lib/texinfo.tex
+++ b/lib/texinfo.tex
@@ -3,11 +3,11 @@
% Load plain if necessary, i.e., if running under initex.
\expandafter\ifx\csname fmtname\endcsname\relax\input plain\fi
%
-\def\texinfoversion{2010-12-23.17}
+\def\texinfoversion{2011-05-23.16}
%
% Copyright 1985, 1986, 1988, 1990, 1991, 1992, 1993, 1994, 1995,
% 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
-% 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
+% 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc.
%
% This texinfo.tex file is free software: you can redistribute it and/or
% modify it under the terms of the GNU General Public License as
@@ -118,8 +118,8 @@
\ifx\putwordChapter\undefined \gdef\putwordChapter{Chapter}\fi
\ifx\putwordfile\undefined \gdef\putwordfile{file}\fi
\ifx\putwordin\undefined \gdef\putwordin{in}\fi
-\ifx\putwordIndexIsEmpty\undefined \gdef\putwordIndexIsEmpty{(Index is
empty)}\fi
-\ifx\putwordIndexNonexistent\undefined \gdef\putwordIndexNonexistent{(Index is
nonexistent)}\fi
+\ifx\putwordIndexIsEmpty\undefined \gdef\putwordIndexIsEmpty{(Index is
empty)}\fi
+\ifx\putwordIndexNonexistent\undefined \gdef\putwordIndexNonexistent{(Index
is nonexistent)}\fi
\ifx\putwordInfo\undefined \gdef\putwordInfo{Info}\fi
\ifx\putwordInstanceVariableof\undefined
\gdef\putwordInstanceVariableof{Instance Variable of}\fi
\ifx\putwordMethodon\undefined \gdef\putwordMethodon{Method on}\fi
@@ -158,15 +158,18 @@
\def\spaceisspace{\catcode`\ =\spacecat}
% sometimes characters are active, so we need control sequences.
+\chardef\ampChar = `\&
\chardef\colonChar = `\:
\chardef\commaChar = `\,
\chardef\dashChar = `\-
\chardef\dotChar = `\.
\chardef\exclamChar= `\!
+\chardef\hashChar = `\#
\chardef\lquoteChar= `\`
\chardef\questChar = `\?
\chardef\rquoteChar= `\'
\chardef\semiChar = `\;
+\chardef\slashChar = `\/
\chardef\underChar = `\_
% Ignore a token.
@@ -215,7 +218,7 @@
\tracingmacros2
\tracingrestores1
\showboxbreadth\maxdimen \showboxdepth\maxdimen
- \ifx\eTeXversion\undefined\else % etex gives us more logging
+ \ifx\eTeXversion\thisisundefined\else % etex gives us more logging
\tracingscantokens1
\tracingifs1
\tracinggroups1
@@ -545,7 +548,7 @@
}
\def\inenvironment#1{%
\ifx#1\empty
- out of any environment%
+ outside of any environment%
\else
in environment \expandafter\string#1%
\fi
@@ -557,7 +560,7 @@
\parseargdef\end{%
\if 1\csname iscond.#1\endcsname
\else
- % The general wording of \badenverr may not be ideal, but... --kasal,
06nov03
+ % The general wording of \badenverr may not be ideal.
\expandafter\checkenv\csname#1\endcsname
\csname E#1\endcsname
\endgroup
@@ -608,7 +611,7 @@
\else\ifx\temp\offword \plainnonfrenchspacing
\else
\errhelp = \EMsimple
- \errmessage{Unknown @frenchspacing option `\temp', must be on/off}%
+ \errmessage{Unknown @frenchspacing option `\temp', must be on|off}%
\fi\fi
}
@@ -690,15 +693,6 @@ where each line of input produces a line of output.}
\newdimen\mil \mil=0.001in
-% Old definition--didn't work.
-%\parseargdef\need{\par %
-%% This method tries to make TeX break the page naturally
-%% if the depth of the box does not fit.
-%{\baselineskip=0pt%
-%\vtop to #1\mil{\vfil}\kern -#1\mil\nobreak
-%\prevdepth=-1000pt
-%}}
-
\parseargdef\need{%
% Ensure vertical mode, so we don't make a big box in the middle of a
% paragraph.
@@ -849,6 +843,7 @@ where each line of input produces a line of output.}
\makevalueexpandable % we want to expand any @value in FILE.
\turnoffactive % and allow special characters in the expansion
\indexnofonts % Allow `@@' and other weird things in file names.
+ \wlog{texinfo.tex: doing @include of #1^^J}%
\edef\temp{\noexpand\input #1 }%
%
% This trickery is to read FILE outside of a group, in case it makes
@@ -1078,9 +1073,8 @@ where each line of input produces a line of output.}
\newif\ifpdfmakepagedest
% when pdftex is run in dvi mode, \pdfoutput is defined (so \pdfoutput=1
-% can be set). So we test for \relax and 0 as well as \undefined,
-% borrowed from ifpdf.sty.
-\ifx\pdfoutput\undefined
+% can be set). So we test for \relax and 0 as well as being undefined.
+\ifx\pdfoutput\thisisundefined
\else
\ifx\pdfoutput\relax
\else
@@ -1512,7 +1506,7 @@ output) for that.)}
% if we are producing pdf, and we have \pdffontattr, then define cmaps.
% (\pdffontattr was introduced many years ago, but people still run
% older pdftex's; it's easy to conditionalize, so we do.)
-\ifpdf \ifx\pdffontattr\undefined \else
+\ifpdf \ifx\pdffontattr\thisisundefined \else
\begingroup
\catcode`\^^M=\active \def^^M{^^J}% Output line endings as the ^^J char.
\catcode`\%=12 \immediate\pdfobj stream {%!PS-Adobe-3.0 Resource-CMap
@@ -1779,7 +1773,7 @@ end
% Use cm as the default font prefix.
% To specify the font prefix, you must define \fontprefix
% before you read in texinfo.tex.
-\ifx\fontprefix\undefined
+\ifx\fontprefix\thisisundefined
\def\fontprefix{cm}
\fi
% Support font families that don't use the same naming scheme as CM.
@@ -2070,7 +2064,7 @@ end
%
\parseargdef\fonttextsize{%
\def\textsizearg{#1}%
- \wlog{doing @fonttextsize \textsizearg}%
+ %\wlog{doing @fonttextsize \textsizearg}%
%
% Set \globaldefs so that documents can use this inside @tex, since
% makeinfo 4.8 does not support it, but we need it nonetheless.
@@ -2252,12 +2246,14 @@ end
% Markup style setup for left and right quotes.
\defmarkupstylesetup\markupsetuplq{%
- \expandafter\let\expandafter \temp \csname
markupsetuplq\currentmarkupstyle\endcsname
+ \expandafter\let\expandafter \temp
+ \csname markupsetuplq\currentmarkupstyle\endcsname
\ifx\temp\relax \markupsetuplqdefault \else \temp \fi
}
\defmarkupstylesetup\markupsetuprq{%
- \expandafter\let\expandafter \temp \csname
markupsetuprq\currentmarkupstyle\endcsname
+ \expandafter\let\expandafter \temp
+ \csname markupsetuprq\currentmarkupstyle\endcsname
\ifx\temp\relax \markupsetuprqdefault \else \temp \fi
}
@@ -2291,12 +2287,11 @@ end
\let\markupsetuplqkbd \markupsetnoligaturesquoteleft
-% Allow an option to not replace quotes with a regular directed right
-% quote/apostrophe (char 0x27), but instead use the undirected quote
-% from cmtt (char 0x0d). The undirected quote is ugly, so don't make it
-% the default, but it works for pasting with more pdf viewers (at least
-% evince), the lilypond developers report. xpdf does work with the
-% regular 0x27.
+% Allow an option to not use regular directed right quote/apostrophe
+% (char 0x27), but instead the undirected quote from cmtt (char 0x0d).
+% The undirected quote is ugly, so don't make it the default, but it
+% works for pasting with more pdf viewers (at least evince), the
+% lilypond developers report. xpdf does work with the regular 0x27.
%
\def\codequoteright{%
\expandafter\ifx\csname SETtxicodequoteundirected\endcsname\relax
@@ -2320,6 +2315,36 @@ end
\else \char'22 \fi
}
+% Commands to set the quote options.
+%
+\parseargdef\codequoteundirected{%
+ \def\temp{#1}%
+ \ifx\temp\onword
+ \expandafter\let\csname SETtxicodequoteundirected\endcsname
+ = t%
+ \else\ifx\temp\offword
+ \expandafter\let\csname SETtxicodequoteundirected\endcsname
+ = \relax
+ \else
+ \errhelp = \EMsimple
+ \errmessage{Unknown @codequoteundirected value `\temp', must be on|off}%
+ \fi\fi
+}
+%
+\parseargdef\codequotebacktick{%
+ \def\temp{#1}%
+ \ifx\temp\onword
+ \expandafter\let\csname SETtxicodequotebacktick\endcsname
+ = t%
+ \else\ifx\temp\offword
+ \expandafter\let\csname SETtxicodequotebacktick\endcsname
+ = \relax
+ \else
+ \errhelp = \EMsimple
+ \errmessage{Unknown @codequotebacktick value `\temp', must be on|off}%
+ \fi\fi
+}
+
% [Knuth] pp. 380,381,391, disable Spanish ligatures ?` and !` of \tt font.
\def\noligaturesquoteleft{\relax\lq}
@@ -2487,6 +2512,8 @@ end
}
}
+\def\codex #1{\tclose{#1}\endgroup}
+
\def\realdash{-}
\def\codedash{-\discretionary{}{}{}}
\def\codeunder{%
@@ -2500,7 +2527,6 @@ end
\discretionary{}{}{}}%
{\_}%
}
-\def\codex #1{\tclose{#1}\endgroup}
% An additional complication: the above will allow breaks after, e.g.,
% each of the four underscores in __typeof__. This is undesirable in
@@ -2520,63 +2546,18 @@ end
\allowcodebreaksfalse
\else
\errhelp = \EMsimple
- \errmessage{Unknown @allowcodebreaks option `\txiarg'}%
+ \errmessage{Unknown @allowcodebreaks option `\txiarg', must be true|false}%
\fi\fi
}
-% @kbd is like @code, except that if the argument is just one @key command,
-% then @kbd has no effect.
-\def\kbd#1{{\setupmarkupstyle{kbd}\def\look{#1}\expandafter\kbdfoo\look??\par}}
-
-% @kbdinputstyle -- arg is `distinct' (@kbd uses slanted tty font always),
-% `example' (@kbd uses ttsl only inside of @example and friends),
-% or `code' (@kbd uses normal tty font always).
-\parseargdef\kbdinputstyle{%
- \def\txiarg{#1}%
- \ifx\txiarg\worddistinct
- \gdef\kbdexamplefont{\ttsl}\gdef\kbdfont{\ttsl}%
- \else\ifx\txiarg\wordexample
- \gdef\kbdexamplefont{\ttsl}\gdef\kbdfont{\tt}%
- \else\ifx\txiarg\wordcode
- \gdef\kbdexamplefont{\tt}\gdef\kbdfont{\tt}%
- \else
- \errhelp = \EMsimple
- \errmessage{Unknown @kbdinputstyle option `\txiarg'}%
- \fi\fi\fi
-}
-\def\worddistinct{distinct}
-\def\wordexample{example}
-\def\wordcode{code}
-
-% Default is `distinct'.
-\kbdinputstyle distinct
-
-\def\xkey{\key}
-\def\kbdfoo#1#2#3\par{\def\one{#1}\def\three{#3}\def\threex{??}%
-\ifx\one\xkey\ifx\threex\three \key{#2}%
-\else{\tclose{\kbdfont\setupmarkupstyle{kbd}\look}}\fi
-\else{\tclose{\kbdfont\setupmarkupstyle{kbd}\look}}\fi}
-
-% For @indicateurl, @env, @command quotes seem unnecessary, so use \code.
-\let\indicateurl=\code
-\let\env=\code
-\let\command=\code
-
-% @clicksequence{File @click{} Open ...}
-\def\clicksequence#1{\begingroup #1\endgroup}
-
-% @clickstyle @arrow (by default)
-\parseargdef\clickstyle{\def\click{#1}}
-\def\click{\arrow}
-
% @uref (abbreviation for `urlref') takes an optional (comma-separated)
% second argument specifying the text to display and an optional third
% arg as text to display instead of (rather than in addition to) the url
-% itself. First (mandatory) arg is the url. Perhaps eventually put in
-% a hypertex \special here.
-%
-\def\uref#1{\douref #1,,,\finish}
-\def\douref#1,#2,#3,#4\finish{\begingroup
+% itself. First (mandatory) arg is the url.
+% (This \urefnobreak definition isn't used now, leaving it for a while
+% for comparison.)
+\def\urefnobreak#1{\dourefnobreak #1,,,\finish}
+\def\dourefnobreak#1,#2,#3,#4\finish{\begingroup
\unsepspaces
\pdfurl{#1}%
\setbox0 = \hbox{\ignorespaces #3}%
@@ -2597,6 +2578,103 @@ end
\endlink
\endgroup}
+% This \urefbreak definition is the active one.
+\def\urefbreak{\begingroup \urefcatcodes \dourefbreak}
+\let\uref=\urefbreak
+\def\dourefbreak#1{\urefbreakfinish #1,,,\finish}
+\def\urefbreakfinish#1,#2,#3,#4\finish{% doesn't work in @example
+ \unsepspaces
+ \pdfurl{#1}%
+ \setbox0 = \hbox{\ignorespaces #3}%
+ \ifdim\wd0 > 0pt
+ \unhbox0 % third arg given, show only that
+ \else
+ \setbox0 = \hbox{\ignorespaces #2}%
+ \ifdim\wd0 > 0pt
+ \ifpdf
+ \unhbox0 % PDF: 2nd arg given, show only it
+ \else
+ \unhbox0\ (\urefcode{#1})% DVI: 2nd arg given, show both it and url
+ \fi
+ \else
+ \urefcode{#1}% only url given, so show it
+ \fi
+ \fi
+ \endlink
+\endgroup}
+
+% Allow line breaks around only a few characters (only).
+\def\urefcatcodes{%
+ \catcode\ampChar=\active \catcode\dotChar=\active
+ \catcode\hashChar=\active \catcode\questChar=\active
+ \catcode\slashChar=\active
+}
+{
+ \urefcatcodes
+ %
+ \global\def\urefcode{\begingroup
+ \setupmarkupstyle{code}%
+ \urefcatcodes
+ \let&\urefcodeamp
+ \let.\urefcodedot
+ \let#\urefcodehash
+ \let?\urefcodequest
+ \let/\urefcodeslash
+ \codex
+ }
+ %
+ % By default, they are just regular characters.
+ \global\def&{\normalamp}
+ \global\def.{\normaldot}
+ \global\def#{\normalhash}
+ \global\def?{\normalquest}
+ \global\def/{\normalslash}
+}
+
+% we put a little stretch before and after the breakable chars, to help
+% line breaking of long url's. The unequal skips make look better in
+% cmtt at least, especially for dots.
+\def\urefprestretch{\urefprebreak \hskip0pt plus.13em }
+\def\urefpoststretch{\urefpostbreak \hskip0pt plus.1em }
+%
+\def\urefcodeamp{\urefprestretch \&\urefpoststretch}
+\def\urefcodedot{\urefprestretch .\urefpoststretch}
+\def\urefcodehash{\urefprestretch \#\urefpoststretch}
+\def\urefcodequest{\urefprestretch ?\urefpoststretch}
+\def\urefcodeslash{\futurelet\next\urefcodeslashfinish}
+{
+ \catcode`\/=\active
+ \global\def\urefcodeslashfinish{%
+ \urefprestretch \slashChar
+ % Allow line break only after the final / in a sequence of
+ % slashes, to avoid line break between the slashes in http://.
+ \ifx\next/\else \urefpoststretch \fi
+ }
+}
+
+% One more complication: by default we'll break after the special
+% characters, but some people like to break before the special chars, so
+% allow that. Also allow no breaking at all, for manual control.
+%
+\parseargdef\urefbreakstyle{%
+ \def\txiarg{#1}%
+ \ifx\txiarg\wordnone
+ \def\urefprebreak{\nobreak}\def\urefpostbreak{\nobreak}
+ \else\ifx\txiarg\wordbefore
+ \def\urefprebreak{\allowbreak}\def\urefpostbreak{\nobreak}
+ \else\ifx\txiarg\wordafter
+ \def\urefprebreak{\nobreak}\def\urefpostbreak{\allowbreak}
+ \else
+ \errhelp = \EMsimple
+ \errmessage{Unknown @urefbreakstyle setting `\txiarg'}%
+ \fi\fi\fi
+}
+\def\wordafter{after}
+\def\wordbefore{before}
+\def\wordnone{none}
+
+\urefbreakstyle after
+
% @url synonym for @uref, since that's how everyone uses it.
%
\let\url=\uref
@@ -2618,6 +2696,51 @@ end
\let\email=\uref
\fi
+% @kbd is like @code, except that if the argument is just one @key command,
+% then @kbd has no effect.
+\def\kbd#1{{\setupmarkupstyle{kbd}\def\look{#1}\expandafter\kbdfoo\look??\par}}
+
+% @kbdinputstyle -- arg is `distinct' (@kbd uses slanted tty font always),
+% `example' (@kbd uses ttsl only inside of @example and friends),
+% or `code' (@kbd uses normal tty font always).
+\parseargdef\kbdinputstyle{%
+ \def\txiarg{#1}%
+ \ifx\txiarg\worddistinct
+ \gdef\kbdexamplefont{\ttsl}\gdef\kbdfont{\ttsl}%
+ \else\ifx\txiarg\wordexample
+ \gdef\kbdexamplefont{\ttsl}\gdef\kbdfont{\tt}%
+ \else\ifx\txiarg\wordcode
+ \gdef\kbdexamplefont{\tt}\gdef\kbdfont{\tt}%
+ \else
+ \errhelp = \EMsimple
+ \errmessage{Unknown @kbdinputstyle setting `\txiarg'}%
+ \fi\fi\fi
+}
+\def\worddistinct{distinct}
+\def\wordexample{example}
+\def\wordcode{code}
+
+% Default is `distinct'.
+\kbdinputstyle distinct
+
+\def\xkey{\key}
+\def\kbdfoo#1#2#3\par{\def\one{#1}\def\three{#3}\def\threex{??}%
+\ifx\one\xkey\ifx\threex\three \key{#2}%
+\else{\tclose{\kbdfont\setupmarkupstyle{kbd}\look}}\fi
+\else{\tclose{\kbdfont\setupmarkupstyle{kbd}\look}}\fi}
+
+% For @indicateurl, @env, @command quotes seem unnecessary, so use \code.
+\let\indicateurl=\code
+\let\env=\code
+\let\command=\code
+
+% @clicksequence{File @click{} Open ...}
+\def\clicksequence#1{\begingroup #1\endgroup}
+
+% @clickstyle @arrow (by default)
+\parseargdef\clickstyle{\def\click{#1}}
+\def\click{\arrow}
+
% Typeset a dimension, e.g., `in' or `pt'. The only reason for the
% argument is to make the input look right: @dmn{pt} instead of @dmn{}pt.
%
@@ -3005,7 +3128,7 @@ end
% Textures 1.7.7 (preloaded format=plain 93.10.14) (68K) 16 APR 2004 02:38
% so we'll define it if necessary.
%
-\ifx\Orb\undefined
+\ifx\Orb\thisisundefined
\def\Orb{\mathhexbox20D}
\fi
@@ -3033,8 +3156,9 @@ end
\newif\ifsetshortcontentsaftertitlepage
\let\setshortcontentsaftertitlepage = \setshortcontentsaftertitlepagetrue
-\parseargdef\shorttitlepage{\begingroup\hbox{}\vskip 1.5in \chaprm
\centerline{#1}%
- \endgroup\page\hbox{}\page}
+\parseargdef\shorttitlepage{%
+ \begingroup \hbox{}\vskip 1.5in \chaprm \centerline{#1}%
+ \endgroup\page\hbox{}\page}
\envdef\titlepage{%
% Open one extra group, as we want to close it in the middle of \Etitlepage.
@@ -3094,7 +3218,7 @@ end
\finishedtitlepagetrue
}
-%%% Macros to be used within @titlepage:
+% Macros to be used within @titlepage:
\let\subtitlerm=\tenrm
\def\subtitlefont{\subtitlerm \normalbaselineskip = 13pt \normalbaselines}
@@ -3127,7 +3251,7 @@ end
}
-%%% Set up page headings and footings.
+% Set up page headings and footings.
\let\thispage=\folio
@@ -3279,7 +3403,7 @@ end
% This produces Day Month Year style of output.
% Only define if not already defined, in case a txi-??.tex file has set
% up a different format (e.g., txi-cs.tex does this).
-\ifx\today\undefined
+\ifx\today\thisisundefined
\def\today{%
\number\day\space
\ifcase\month
@@ -3826,9 +3950,9 @@ end
\setbox0=\vbox{X}\global\multitablelinespace=\the\baselineskip
\global\advance\multitablelinespace by-\ht0
\fi
-%% Test to see if parskip is larger than space between lines of
-%% table. If not, do nothing.
-%% If so, set to same dimension as multitablelinespace.
+% Test to see if parskip is larger than space between lines of
+% table. If not, do nothing.
+% If so, set to same dimension as multitablelinespace.
\ifdim\multitableparskip>\multitablelinespace
\global\multitableparskip=\multitablelinespace
\global\advance\multitableparskip-7pt %% to keep parskip somewhat smaller
@@ -5242,7 +5366,8 @@ end
\global\let\subsubsection = \appendixsubsubsec
}
-\outer\parseargdef\unnumbered{\unnmhead0{#1}} % normally unnmhead0 calls
unnumberedzzz
+% normally unnmhead0 calls unnumberedzzz:
+\outer\parseargdef\unnumbered{\unnmhead0{#1}}
\def\unnumberedzzz#1{%
\global\secno=0 \global\subsecno=0 \global\subsubsecno=0
\global\advance\unnumberedno by 1
@@ -5286,40 +5411,47 @@ end
\let\top\unnumbered
% Sections.
+%
\outer\parseargdef\numberedsec{\numhead1{#1}} % normally calls seczzz
\def\seczzz#1{%
\global\subsecno=0 \global\subsubsecno=0 \global\advance\secno by 1
\sectionheading{#1}{sec}{Ynumbered}{\the\chapno.\the\secno}%
}
-\outer\parseargdef\appendixsection{\apphead1{#1}} % normally calls
appendixsectionzzz
+% normally calls appendixsectionzzz:
+\outer\parseargdef\appendixsection{\apphead1{#1}}
\def\appendixsectionzzz#1{%
\global\subsecno=0 \global\subsubsecno=0 \global\advance\secno by 1
\sectionheading{#1}{sec}{Yappendix}{\appendixletter.\the\secno}%
}
\let\appendixsec\appendixsection
-\outer\parseargdef\unnumberedsec{\unnmhead1{#1}} % normally calls
unnumberedseczzz
+% normally calls unnumberedseczzz:
+\outer\parseargdef\unnumberedsec{\unnmhead1{#1}}
\def\unnumberedseczzz#1{%
\global\subsecno=0 \global\subsubsecno=0 \global\advance\secno by 1
\sectionheading{#1}{sec}{Ynothing}{\the\unnumberedno.\the\secno}%
}
% Subsections.
-\outer\parseargdef\numberedsubsec{\numhead2{#1}} % normally calls
numberedsubseczzz
+%
+% normally calls numberedsubseczzz:
+\outer\parseargdef\numberedsubsec{\numhead2{#1}}
\def\numberedsubseczzz#1{%
\global\subsubsecno=0 \global\advance\subsecno by 1
\sectionheading{#1}{subsec}{Ynumbered}{\the\chapno.\the\secno.\the\subsecno}%
}
-\outer\parseargdef\appendixsubsec{\apphead2{#1}} % normally calls
appendixsubseczzz
+% normally calls appendixsubseczzz:
+\outer\parseargdef\appendixsubsec{\apphead2{#1}}
\def\appendixsubseczzz#1{%
\global\subsubsecno=0 \global\advance\subsecno by 1
\sectionheading{#1}{subsec}{Yappendix}%
{\appendixletter.\the\secno.\the\subsecno}%
}
-\outer\parseargdef\unnumberedsubsec{\unnmhead2{#1}} %normally calls
unnumberedsubseczzz
+% normally calls unnumberedsubseczzz:
+\outer\parseargdef\unnumberedsubsec{\unnmhead2{#1}}
\def\unnumberedsubseczzz#1{%
\global\subsubsecno=0 \global\advance\subsecno by 1
\sectionheading{#1}{subsec}{Ynothing}%
@@ -5327,21 +5459,25 @@ end
}
% Subsubsections.
-\outer\parseargdef\numberedsubsubsec{\numhead3{#1}} % normally
numberedsubsubseczzz
+%
+% normally numberedsubsubseczzz:
+\outer\parseargdef\numberedsubsubsec{\numhead3{#1}}
\def\numberedsubsubseczzz#1{%
\global\advance\subsubsecno by 1
\sectionheading{#1}{subsubsec}{Ynumbered}%
{\the\chapno.\the\secno.\the\subsecno.\the\subsubsecno}%
}
-\outer\parseargdef\appendixsubsubsec{\apphead3{#1}} % normally
appendixsubsubseczzz
+% normally appendixsubsubseczzz:
+\outer\parseargdef\appendixsubsubsec{\apphead3{#1}}
\def\appendixsubsubseczzz#1{%
\global\advance\subsubsecno by 1
\sectionheading{#1}{subsubsec}{Yappendix}%
{\appendixletter.\the\secno.\the\subsecno.\the\subsubsecno}%
}
-\outer\parseargdef\unnumberedsubsubsec{\unnmhead3{#1}} %normally
unnumberedsubsubseczzz
+% normally unnumberedsubsubseczzz:
+\outer\parseargdef\unnumberedsubsubsec{\unnmhead3{#1}}
\def\unnumberedsubsubseczzz#1{%
\global\advance\subsubsecno by 1
\sectionheading{#1}{subsubsec}{Ynothing}%
@@ -5391,14 +5527,13 @@ end
% (including whitespace, linebreaking, etc. around it),
% given all the information in convenient, parsed form.
-%%% Args are the skip and penalty (usually negative)
+% Args are the skip and penalty (usually negative)
\def\dobreak#1#2{\par\ifdim\lastskip<#1\removelastskip\penalty#2\vskip#1\fi}
-%%% Define plain chapter starts, and page on/off switching for it
% Parameter controlling skip before chapter headings (if needed)
-
\newskip\chapheadingskip
+% Define plain chapter starts, and page on/off switching for it.
\def\chapbreak{\dobreak \chapheadingskip {-4000}}
\def\chappager{\par\vfill\supereject}
% Because \domark is called before \chapoddpage, the filler page will
@@ -5601,6 +5736,8 @@ end
%
\def\sectionheading#1#2#3#4{%
{%
+ \checkenv{}% should not be in an environment.
+ %
% Switch to the right set of fonts.
\csname #2fonts\endcsname \rmisbold
%
@@ -6346,7 +6483,7 @@ end
%
\def\Equotation{%
\par
- \ifx\quotationauthor\undefined\else
+ \ifx\quotationauthor\thisisundefined\else
% indent a bit.
\leftline{\kern 2\leftskip \sl ---\quotationauthor}%
\fi
@@ -6505,6 +6642,7 @@ end
\makevalueexpandable
\setupverbatim
\indexnofonts % Allow `@@' and other weird things in file names.
+ \wlog{texinfo.tex: doing @verbatiminclude of #1^^J}%
\input #1
\afterenvbreak
}%
@@ -6554,7 +6692,7 @@ end
% commands also insert a nobreak penalty, and we don't want to allow
% a break between a section heading and a defun.
%
- % As a minor refinement, we avoid "club" headers by signalling
+ % As a further refinement, we avoid "club" headers by signalling
% with penalty of 10003 after the very first @deffn in the
% sequence (see above), and penalty of 10002 after any following
% @def command.
@@ -6621,13 +6759,36 @@ end
\def\domakedefun#1#2#3{%
\envdef#1{%
\startdefun
+ \doingtypefnfalse % distinguish typed functions from all else
\parseargusing\activeparens{\printdefunline#3}%
}%
\def#2{\dodefunx#1}%
\def#3%
}
-%%% Untyped functions:
+\newif\ifdoingtypefn % doing typed function?
+\newif\ifrettypeownline % typeset return type on its own line?
+
+% @deftypefnnewline on|off says whether the return type of typed functions
+% are printed on their own line. This affects @deftypefn, @deftypefun,
+% @deftypeop, and @deftypemethod.
+%
+\parseargdef\deftypefnnewline{%
+ \def\temp{#1}%
+ \ifx\temp\onword
+ \expandafter\let\csname SETtxideftypefnnl\endcsname
+ = \empty
+ \else\ifx\temp\offword
+ \expandafter\let\csname SETtxideftypefnnl\endcsname
+ = \relax
+ \else
+ \errhelp = \EMsimple
+ \errmessage{Unknown @txideftypefnnl value `\temp',
+ must be on|off}%
+ \fi\fi
+}
+
+% Untyped functions:
% @deffn category name args
\makedefun{deffn}{\deffngeneral{}}
@@ -6646,7 +6807,7 @@ end
\defname{#2}{}{#3}\magicamp\defunargs{#4\unskip}%
}
-%%% Typed functions:
+% Typed functions:
% @deftypefn category type name args
\makedefun{deftypefn}{\deftypefngeneral{}}
@@ -6661,10 +6822,11 @@ end
%
\def\deftypefngeneral#1#2 #3 #4 #5\endheader{%
\dosubind{fn}{\code{#4}}{#1}%
+ \doingtypefntrue
\defname{#2}{#3}{#4}\defunargs{#5\unskip}%
}
-%%% Typed variables:
+% Typed variables:
% @deftypevr category type var args
\makedefun{deftypevr}{\deftypecvgeneral{}}
@@ -6682,7 +6844,7 @@ end
\defname{#2}{#3}{#4}\defunargs{#5\unskip}%
}
-%%% Untyped variables:
+% Untyped variables:
% @defvr category var args
\makedefun{defvr}#1 {\deftypevrheader{#1} {} }
@@ -6693,7 +6855,8 @@ end
% \defcvof {category of}class var args
\def\defcvof#1#2 {\deftypecvof{#1}#2 {} }
-%%% Type:
+% Types:
+
% @deftp category name args
\makedefun{deftp}#1 #2 #3\endheader{%
\doind{tp}{\code{#2}}%
@@ -6721,25 +6884,49 @@ end
% We are followed by (but not passed) the arguments, if any.
%
\def\defname#1#2#3{%
+ \par
% Get the values of \leftskip and \rightskip as they were outside the @def...
\advance\leftskip by -\defbodyindent
%
- % How we'll format the type name. Putting it in brackets helps
+ % Determine if we are typesetting the return type of a typed function
+ % on a line by itself.
+ \rettypeownlinefalse
+ \ifdoingtypefn % doing a typed function specifically?
+ % then check user option for putting return type on its own line:
+ \expandafter\ifx\csname SETtxideftypefnnl\endcsname\relax \else
+ \rettypeownlinetrue
+ \fi
+ \fi
+ %
+ % How we'll format the category name. Putting it in brackets helps
% distinguish it from the body text that may end up on the next line
% just below it.
\def\temp{#1}%
\setbox0=\hbox{\kern\deflastargmargin \ifx\temp\empty\else [\rm\temp]\fi}
%
- % Figure out line sizes for the paragraph shape.
+ % Figure out line sizes for the paragraph shape. We'll always have at
+ % least two.
+ \tempnum = 2
+ %
% The first line needs space for \box0; but if \rightskip is nonzero,
% we need only space for the part of \box0 which exceeds it:
\dimen0=\hsize \advance\dimen0 by -\wd0 \advance\dimen0 by \rightskip
+ %
+ % If doing a return type on its own line, we'll have another line.
+ \ifrettypeownline
+ \advance\tempnum by 1
+ \def\maybeshapeline{0in \hsize}%
+ \else
+ \def\maybeshapeline{}%
+ \fi
+ %
% The continuations:
\dimen2=\hsize \advance\dimen2 by -\defargsindent
- % (plain.tex says that \dimen1 should be used only as global.)
- \parshape 2 0in \dimen0 \defargsindent \dimen2
%
- % Put the type name to the right margin.
+ % The final paragraph shape:
+ \parshape \tempnum 0in \dimen0 \maybeshapeline \defargsindent \dimen2
+ %
+ % Put the category name at the right margin.
\noindent
\hbox to 0pt{%
\hfil\box0 \kern-\hsize
@@ -6761,8 +6948,16 @@ end
% . this still does not fix the ?` and !` ligatures, but so far no
% one has made identifiers using them :).
\df \tt
- \def\temp{#2}% return value type
- \ifx\temp\empty\else \tclose{\temp} \fi
+ \def\temp{#2}% text of the return type
+ \ifx\temp\empty\else
+ \tclose{\temp}% typeset the return type
+ \ifrettypeownline
+ % put return type on its own line; prohibit line break following:
+ \hfil\vadjust{\nobreak}\break
+ \else
+ \space % type on same line, so just followed by a space
+ \fi
+ \fi % no return type
#3% output function name
}%
{\rm\enskip}% hskip 0.5 em of \tenrm
@@ -6880,7 +7075,7 @@ end
% To do this right we need a feature of e-TeX, \scantokens,
% which we arrange to emulate with a temporary file in ordinary TeX.
-\ifx\eTeXversion\undefined
+\ifx\eTeXversion\thisisundefined
\newwrite\macscribble
\def\scantokens#1{%
\toks0={#1}%
@@ -6905,12 +7100,14 @@ end
% ... and for \example:
\spaceisspace
%
- % The \empty here causes a following catcode 5 newline to be eaten
- % as part of reading whitespace after a control sequence. It does
- % not eat a catcode 13 newline. There's no good way to handle the
- % two cases. See the Macro Details node in the manual for the
- % workaround we currently have to recommend for macros and
+ % The \empty here causes a following catcode 5 newline to be eaten as
+ % part of reading whitespace after a control sequence. It does not
+ % eat a catcode 13 newline. There's no good way to handle the two
+ % cases (untried: maybe e-TeX's \everyeof could help, though plain TeX
+ % would then have different behavior). See the Macro Details node in
+ % the manual for the workaround we recommend for macros and
% line-oriented commands.
+ %
\scantokens{#1\empty}%
\endgroup}
@@ -7235,7 +7432,8 @@ end
% @inforef is relatively simple.
\def\inforef #1{\inforefzzz #1,,,,**}
-\def\inforefzzz #1,#2,#3,#4**{\putwordSee{} \putwordInfo{} \putwordfile{}
\file{\ignorespaces #3{}},
+\def\inforefzzz #1,#2,#3,#4**{%
+ \putwordSee{} \putwordInfo{} \putwordfile{} \file{\ignorespaces #3{}},
node \samp{\ignorespaces#1{}}}
% @node's only job in TeX is to define \lastnode, which is used in
@@ -7296,11 +7494,32 @@ end
\toks0 = \expandafter{\lastsection}%
\immediate \writexrdef{title}{\the\toks0 }%
\immediate \writexrdef{snt}{\csname #2\endcsname}% \Ynumbered etc.
- \safewhatsit{\writexrdef{pg}{\folio}}% will be written later, during
\shipout
+ \safewhatsit{\writexrdef{pg}{\folio}}% will be written later, at \shipout
}%
\fi
}
+% @xrefautosectiontitle on|off says whether @section(ing) names are used
+% automatically in xrefs, if the third arg is not explicitly specified.
+% This was provided as a "secret" @set xref-automatic-section-title
+% variable, now it's official.
+%
+\parseargdef\xrefautomaticsectiontitle{%
+ \def\temp{#1}%
+ \ifx\temp\onword
+ \expandafter\let\csname SETxref-automatic-section-title\endcsname
+ = \empty
+ \else\ifx\temp\offword
+ \expandafter\let\csname SETxref-automatic-section-title\endcsname
+ = \relax
+ \else
+ \errhelp = \EMsimple
+ \errmessage{Unknown @xrefautomaticsectiontitle value `\temp',
+ must be on|off}%
+ \fi\fi
+}
+
+
% @xref, @pxref, and @ref generate cross-references. For \xrefX, #1 is
% the node name, #2 the name of the Info cross-reference, #3 the printed
% node name, #4 the name of the Info file, #5 the name of the printed
@@ -7317,7 +7536,7 @@ end
\setbox0=\hbox{\printedrefname\unskip}%
\ifdim \wd0 = 0pt
% No printed node name was explicitly given.
- \expandafter\ifx\csname SETxref-automatic-section-title\endcsname\relax
+ \expandafter\ifx\csname SETxref-automatic-section-title\endcsname \relax
% Use the node name inside the square brackets.
\def\printedrefname{\ignorespaces #1}%
\else
@@ -7787,7 +8006,7 @@ end
it from ftp://tug.org/tex/epsf.tex.}
%
\def\image#1{%
- \ifx\epsfbox\undefined
+ \ifx\epsfbox\thisiundefined
\ifwarnednoepsf \else
\errhelp = \noepsfhelp
\errmessage{epsf.tex not found, images will be ignored}%
@@ -8252,7 +8471,7 @@ directory should work if nowhere else does.}
%
% Latin1 (ISO-8859-1) character definitions.
\def\latonechardefs{%
- \gdef^^a0{~}
+ \gdef^^a0{\tie}
\gdef^^a1{\exclamdown}
\gdef^^a2{\missingcharmsg{CENT SIGN}}
\gdef^^a3{{\pounds}}
@@ -8282,7 +8501,7 @@ directory should work if nowhere else does.}
\gdef^^b9{$^1$}
\gdef^^ba{\ordm}
%
- \gdef^^bb{\guilletright}
+ \gdef^^bb{\guillemetright}
\gdef^^bc{$1\over4$}
\gdef^^bd{$1\over2$}
\gdef^^be{$3\over4$}
@@ -8374,7 +8593,7 @@ directory should work if nowhere else does.}
% Latin2 (ISO-8859-2) character definitions.
\def\lattwochardefs{%
- \gdef^^a0{~}
+ \gdef^^a0{\tie}
\gdef^^a1{\ogonek{A}}
\gdef^^a2{\u{}}
\gdef^^a3{\L}
@@ -8547,7 +8766,7 @@ directory should work if nowhere else does.}
\gdef\DeclareUnicodeCharacter#1#2{%
\countUTFz = "#1\relax
- \wlog{\space\space defining Unicode char U+#1 (decimal \the\countUTFz)}%
+ %\wlog{\space\space defining Unicode char U+#1 (decimal \the\countUTFz)}%
\begingroup
\parseXMLCharref
\def\UTFviiiTwoOctets##1##2{%
@@ -9223,28 +9442,21 @@ directory should work if nowhere else does.}
\message{and turning on texinfo input format.}
+\def^^L{\par} % remove \outer, so ^L can appear in an @comment
+
% DEL is a comment character, in case @c does not suffice.
\catcode`\^^? = 14
% Define macros to output various characters with catcode for normal text.
-\catcode`\"=\other
-\catcode`\~=\other
-\catcode`\^=\other
-\catcode`\_=\other
-\catcode`\|=\other
-\catcode`\<=\other
-\catcode`\>=\other
-\catcode`\+=\other
-\catcode`\$=\other
-\def\normaldoublequote{"}
-\def\normaltilde{~}
-\def\normalcaret{^}
-\def\normalunderscore{_}
-\def\normalverticalbar{|}
-\def\normalless{<}
-\def\normalgreater{>}
-\def\normalplus{+}
-\def\normaldollar{$}%$ font-lock fix
+\catcode`\"=\other \def\normaldoublequote{"}
+\catcode`\$=\other \def\normaldollar{$}%$ font-lock fix
+\catcode`\+=\other \def\normalplus{+}
+\catcode`\<=\other \def\normalless{<}
+\catcode`\>=\other \def\normalgreater{>}
+\catcode`\^=\other \def\normalcaret{^}
+\catcode`\_=\other \def\normalunderscore{_}
+\catcode`\|=\other \def\normalverticalbar{|}
+\catcode`\~=\other \def\normaltilde{~}
% This macro is used to make a character print one way in \tt
% (where it can probably be output as-is), and another way in other fonts,
@@ -9337,16 +9549,16 @@ directory should work if nowhere else does.}
% the literal character `\'.
%
@address@hidden
- @address@hidden
@let"address@hidden
- @address@hidden
+ @address@hidden %$ font-lock fix
+ @address@hidden
+ @let<address@hidden
+ @let>address@hidden
+ @address@hidden
@address@hidden
@address@hidden
@let|address@hidden
- @let<address@hidden
- @let>address@hidden
- @address@hidden
- @address@hidden %$ font-lock fix
+ @address@hidden
@markupsetuplqdefault
@markupsetuprqdefault
@unsepspaces
@@ -9378,10 +9590,16 @@ directory should work if nowhere else does.}
% Say @foo, not \foo, in error messages.
@escapechar = `@@
+% These (along with & and #) are made active for url-breaking, so need
+% active definitions as the normal characters.
address@hidden@normaldot{.}
address@hidden@normalquest{?}
address@hidden@normalslash{/}
+
% These look ok in all fonts, so just make them not special.
address@hidden@& = @other
address@hidden@# = @other
address@hidden@% = @other
address@hidden@& = @other @address@hidden&}
address@hidden@# = @other @address@hidden
address@hidden@% = @other @address@hidden
@c Finally, make ` and ' active, so that txicodequoteundirected and
@c txicodequotebacktick work right in, e.g., @address@hidden'}}. If we
diff --git a/tests/.gitignore b/tests/.gitignore
index 026e261..0d5a032 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -5,6 +5,7 @@ instspc-tests.am
parallel-tests.am
*.dir
*.log
+*.trs
*.log-t
*-p.test
instspc-*-build.test
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 0b83b0c..b9b75ff 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -16,6 +16,10 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+TEST_EXTENSIONS = .test .tap
+TAP_LOG_DRIVER = $(PERL) $(top_srcdir)/lib/tap-driver
+TAP_LOG_DRIVER_FLAGS = --merge
+
MAINTAINERCLEANFILES =
EXTRA_DIST = ChangeLog-old
@@ -27,6 +31,9 @@ gcj6.test \
override-conditional-2.test \
pr8365-remake-timing.test \
yacc-dist-nobuild-subdir.test \
+tap-plan-corner2.test \
+tap-message-0.test \
+tap-signal.test \
txinfo5.test
@@ -103,6 +110,7 @@ XFAIL_TESTS += $(instspc_xfail_tests)
AM_TESTS_ENVIRONMENT = \
test x"$$me" = x || unset me; \
test x"$$required" = x || unset required; \
+ test x"$$use_tap" = x || unset use_tap; \
test x"$$parallel_tests" = x || unset parallel_tests; \
test x"$$original_AUTOMAKE" = x || unset original_AUTOMAKE; \
test x"$$original_ACLOCAL" = x || unset original_ACLOCAL;
@@ -130,12 +138,7 @@ acloca20.test \
acloca21.test \
acloca22.test \
acloca23.test \
-acoutnoq.test \
-acoutpt.test \
-acoutpt2.test \
-acoutqnl.test \
-acoutbs.test \
-acoutbs2.test \
+ac-output-old.tap \
acsilent.test \
acsubst.test \
acsubst2.test \
@@ -145,6 +148,9 @@ alloca.test \
alloca2.test \
alpha.test \
alpha2.test \
+amhello-cflags.test \
+amhello-cross-compile.test \
+amhello-binpkg.test \
amassign.test \
ammissing.test \
amopt.test \
@@ -234,8 +240,8 @@ check10.test \
check11.test \
check12.test \
check-exported-srcdir.test \
+check-fd-redirect.test \
check-tests-in-builddir.test \
-check-tests_environment.test \
check-no-test-driver.test \
checkall.test \
clean.test \
@@ -719,8 +725,10 @@ parallel-tests7.test \
parallel-tests8.test \
parallel-tests9.test \
parallel-tests10.test \
-parallel-tests-am_tests_environment.test \
-parallel-tests-unreadable-log.test \
+parallel-tests-dry-run.test \
+parallel-tests-fd-redirect.test \
+parallel-tests-extra-programs.test \
+parallel-tests-unreadable.test \
parallel-tests-subdir.test \
parallel-tests-interrupt.test \
parallel-tests-reset-term.test \
@@ -728,8 +736,22 @@ parallel-tests-harderror.test \
parallel-tests-log-override-1.test \
parallel-tests-log-override-2.test \
parallel-tests-log-override-recheck.test \
+parallel-tests-cmdline-override.test \
+parallel-tests-log-compiler-example.test \
parallel-tests-empty-testlogs.test \
-parallel-test-driver-install.test \
+parallel-tests-driver-install.test \
+parallel-tests-no-color-in-log.test \
+parallel-tests-no-spurious-summary.test \
+parallel-tests-exit-statuses.test \
+parallel-tests-console-output.test \
+parallel-tests-once.test \
+tests-environment.test \
+am-tests-environment.test \
+tests-environment-backcompat.test \
+testsuite-summary-color.test \
+testsuite-summary-count.test \
+testsuite-summary-count-many.test \
+testsuite-summary-reference-log.test \
test-driver-custom-no-extra-driver.test \
test-driver-custom.test \
test-driver-custom-xfail-tests.test \
@@ -739,10 +761,18 @@ test-driver-custom-multitest-recheck2.test \
test-driver-custom-html.test \
test-driver-custom-no-html.test \
test-driver-create-log-dir.test \
-test-driver-global-log.test \
-test-driver-recheck.test \
test-driver-strip-vpath.test \
+test-driver-trs-suffix-registered.test \
test-driver-fail.test \
+test-log.test \
+test-metadata-global-log.test \
+test-metadata-recheck.test \
+test-metadata-results.test \
+test-missing.test \
+test-missing2.test \
+test-trs-basic.test \
+test-trs-recover.test \
+test-trs-recover2.test \
parse.test \
percent.test \
percent2.test \
@@ -841,6 +871,7 @@ remake-subdir-from-subdir.test \
remake-subdir-gnu.test \
remake-subdir.test \
remake-subdir2.test \
+remake-subdir-long-time.test \
remake-gnulib-add-acsubst.test \
remake-gnulib-add-header.test \
remake-gnulib-remove-header.test \
@@ -865,6 +896,7 @@ self-check-is_newest.test \
self-check-me.test \
self-check-report.test \
self-check-sanity.test \
+self-check-tap.test \
self-check-unindent.test \
sanity.test \
scripts.test \
@@ -972,6 +1004,8 @@ suffix13.test \
symlink.test \
symlink2.test \
syntax.test \
+$(tap_with_common_setup_tests) \
+$(tap_other_tests) \
tags.test \
tags2.test \
tagsub.test \
@@ -1085,14 +1119,98 @@ $(parallel_tests)
EXTRA_DIST += $(TESTS)
+## Files containing auxiliary functions used by our test cases.
+EXTRA_DIST += tap-functions.sh plain-functions.sh
+
test-driver-custom-multitest.log: trivial-test-driver
test-driver-custom-multitest-recheck.log: trivial-test-driver
test-driver-custom-multitest-recheck2.log: trivial-test-driver
test-driver-custom-html.log: trivial-test-driver
EXTRA_DIST += trivial-test-driver
+testsuite-summary-color.log testsuite-summary-count.log: \
+ testsuite-summary-checks.sh extract-testsuite-summary
+EXTRA_DIST += testsuite-summary-checks.sh
+EXTRA_DIST += extract-testsuite-summary
+
+testsuite-summary-count-many.log: trivial-test-driver
+testsuite-summary-count-many.log: extract-testsuite-summary
+
+# List of tests on TAP support that use the files pre-computed by
+# `tap-common-setup.test', and sources the `tap-setup.sh' helper
+# script.
+tap_with_common_setup_tests = \
+tap-autonumber.test \
+tap-bailout.test \
+tap-color.test \
+tap-deps.test \
+tap-diagnostic.test \
+tap-empty-diagnostic.test \
+tap-empty.test \
+tap-escape-directive.test \
+tap-exit.test \
+tap-signal.test \
+tap-fancy.test \
+tap-fancy2.test \
+tap-global-log.test \
+tap-global-result.test \
+tap-html.test \
+tap-log.test \
+tap-merge-stdout-stderr.test \
+tap-no-merge-stdout-stderr.test \
+tap-message-0.test \
+tap-no-disable-hard-error.test \
+tap-no-spurious-summary.test \
+tap-no-spurious.test \
+tap-not-ok-skip.test \
+tap-numeric-description.test \
+tap-out-of-order.test \
+tap-passthrough.test \
+tap-passthrough-exit.test \
+tap-plan.test \
+tap-plan-corner.test \
+tap-plan-corner2.test \
+tap-plan-errors.test \
+tap-realtime.test \
+tap-recheck-logs.test \
+tap-skip-whole-whitespace.test \
+tap-skip-whole.test \
+tap-todo-skip-together.test \
+tap-todo-skip-whitespace.test \
+tap-todo-skip.test \
+tap-unplanned.test \
+tap-whitespace-normalization.test \
+tap-with-and-without-number.test \
+tap-xfail-tests.test
+
+# Their log files.
+tap_with_common_setup_logs = $(tap_with_common_setup_tests:.test=.log)
+
+# Their dependencies.
+$(tap_with_common_setup_logs): tap-common-setup.log tap-setup.sh
+EXTRA_DIST += tap-setup.sh
+
+# Other tests on TAP support.
+tap_other_tests = \
+tap-common-setup.test \
+tap-bad-prog.tap \
+tap-basic.test \
+tap-doc.test \
+tap-doc2.test \
+tap-more.test \
+tap-more2.test \
+tap-recheck.test \
+tap-summary.test \
+tap-summary-color.test
+
+tap-summary.log tap-summary-color.log: tap-summary-aux.sh
+EXTRA_DIST += tap-summary-aux.sh
+
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
+# FIXME: this should be made more granular once we have a cleaner
+# subdivision of the tests.
+$(TEST_LOGS): plain-functions.sh tap-functions.sh
clean-local: clean-local-check
.PHONY: clean-local-check
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 193fc95..b030585 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -76,12 +76,14 @@ am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY))
am__v_at_0 = @
SOURCES =
DIST_SOURCES =
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-am__color_tests=no; \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
@@ -91,6 +93,8 @@ test "X$(AM_COLOR_TESTS)" != Xno \
grn='[0;32m'; \
lgn='[1;32m'; \
blu='[1;34m'; \
+ mgn='[0;35m'; \
+ brg='[1m'; \
std='[m'; \
}
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
@@ -116,31 +120,12 @@ am__base_list = \
sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
# Restructured Text title.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
am__sh_e_setup = case $$- in *e*) set +e;; esac
-# Default flags passed to all log compiler wrappers.
-am__test_driver_flags = \
- --test-name "$$f" \
- --log-file '$@' \
+# Default flags passed to test drivers.
+am__common_driver_flags = \
--color-tests "$$am__color_tests" \
--enable-hard-errors "$$am__enable_hard_errors" \
--expect-failure "$$am__expect_failure"
@@ -174,16 +159,26 @@ case " $(XFAIL_TESTS) " in
\
am__expect_failure=no;; \
esac; \
$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT)
+# The names of the tests, with any registered extension removed. Or
+# equivalently, the names of the test logs, with the `.log' extension
+# renoved. This honours runtime overriding of TESTS and TEST_LOGS.
+am__TEST_BASES = $(TEST_LOGS:.log=)
+# This can be used instead of $(MAKE) in recipes requiring a recursive call
+# to make, but which are not intended to be executed by "make -n". See the
+# GNU make manual for more details.
+am__stealth_MAKE = $(MAKE)
RECHECK_LOGS = $(TEST_LOGS)
AM_RECURSIVE_TARGETS = check check-html recheck recheck-html
TEST_SUITE_LOG = test-suite.log
TEST_SUITE_HTML = $(TEST_SUITE_LOG:.log=.html)
-TEST_EXTENSIONS = .test
am__test_logs1 = $(TESTS:=.log)
-TEST_LOGS = $(am__test_logs1:.test.log=.log)
+am__test_logs2 = $(am__test_logs1:.test.log=.log)
TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/lib/test-driver
TEST_LOG_COMPILE = $(TEST_LOG_COMPILER) $(AM_TEST_LOG_FLAGS) \
$(TEST_LOG_FLAGS)
+TEST_LOGS = $(am__test_logs2:.tap.log=.log)
+TAP_LOG_COMPILE = $(TAP_LOG_COMPILER) $(AM_TAP_LOG_FLAGS) \
+ $(TAP_LOG_FLAGS)
TEST_LOGS_TMP = $(TEST_LOGS:.log=.log-t)
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
@@ -280,21 +275,27 @@ target_alias = @target_alias@
top_build_prefix = @top_build_prefix@
top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
+TEST_EXTENSIONS = .test .tap
+TAP_LOG_DRIVER = $(PERL) $(top_srcdir)/lib/tap-driver
+TAP_LOG_DRIVER_FLAGS = --merge
MAINTAINERCLEANFILES = $(parallel_tests) $(instspc_tests)
EXTRA_DIST = ChangeLog-old gen-parallel-tests instspc-tests.sh \
- $(TESTS) trivial-test-driver
+ $(TESTS) tap-functions.sh plain-functions.sh \
+ trivial-test-driver testsuite-summary-checks.sh \
+ extract-testsuite-summary tap-setup.sh tap-summary-aux.sh
XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test \
override-conditional-2.test pr8365-remake-timing.test \
- yacc-dist-nobuild-subdir.test txinfo5.test \
+ yacc-dist-nobuild-subdir.test tap-plan-corner2.test \
+ tap-message-0.test tap-signal.test txinfo5.test \
$(instspc_xfail_tests)
parallel_tests = backcompat5-p.test check-exported-srcdir-p.test \
- check-tests-in-builddir-p.test check-tests_environment-p.test \
- check-p.test check10-p.test check11-p.test check12-p.test \
- check2-p.test check3-p.test check4-p.test check5-p.test \
- check6-p.test check7-p.test check8-p.test check9-p.test \
- color-p.test color2-p.test comment9-p.test dejagnu-p.test \
- exeext4-p.test maken3-p.test maken4-p.test \
- posixsubst-tests-p.test
+ check-fd-redirect-p.test check-tests-in-builddir-p.test \
+ check-p.test check11-p.test check12-p.test check2-p.test \
+ check3-p.test check4-p.test check5-p.test check6-p.test \
+ check7-p.test check8-p.test check9-p.test color-p.test \
+ color2-p.test comment9-p.test dejagnu-p.test exeext4-p.test \
+ maken3-p.test maken4-p.test posixsubst-tests-p.test \
+ tests-environment-p.test
instspc_tests = instspc-squote-build.test instspc-squote-install.test \
instspc-dquote-build.test instspc-dquote-install.test \
instspc-bquote-build.test instspc-bquote-install.test \
@@ -363,6 +364,7 @@ instspc_xfail_tests = instspc-squote-build.test \
AM_TESTS_ENVIRONMENT = \
test x"$$me" = x || unset me; \
test x"$$required" = x || unset required; \
+ test x"$$use_tap" = x || unset use_tap; \
test x"$$parallel_tests" = x || unset parallel_tests; \
test x"$$original_AUTOMAKE" = x || unset original_AUTOMAKE; \
test x"$$original_ACLOCAL" = x || unset original_ACLOCAL;
@@ -390,12 +392,7 @@ acloca20.test \
acloca21.test \
acloca22.test \
acloca23.test \
-acoutnoq.test \
-acoutpt.test \
-acoutpt2.test \
-acoutqnl.test \
-acoutbs.test \
-acoutbs2.test \
+ac-output-old.tap \
acsilent.test \
acsubst.test \
acsubst2.test \
@@ -405,6 +402,9 @@ alloca.test \
alloca2.test \
alpha.test \
alpha2.test \
+amhello-cflags.test \
+amhello-cross-compile.test \
+amhello-binpkg.test \
amassign.test \
ammissing.test \
amopt.test \
@@ -494,8 +494,8 @@ check10.test \
check11.test \
check12.test \
check-exported-srcdir.test \
+check-fd-redirect.test \
check-tests-in-builddir.test \
-check-tests_environment.test \
check-no-test-driver.test \
checkall.test \
clean.test \
@@ -979,8 +979,10 @@ parallel-tests7.test \
parallel-tests8.test \
parallel-tests9.test \
parallel-tests10.test \
-parallel-tests-am_tests_environment.test \
-parallel-tests-unreadable-log.test \
+parallel-tests-dry-run.test \
+parallel-tests-fd-redirect.test \
+parallel-tests-extra-programs.test \
+parallel-tests-unreadable.test \
parallel-tests-subdir.test \
parallel-tests-interrupt.test \
parallel-tests-reset-term.test \
@@ -988,8 +990,22 @@ parallel-tests-harderror.test \
parallel-tests-log-override-1.test \
parallel-tests-log-override-2.test \
parallel-tests-log-override-recheck.test \
+parallel-tests-cmdline-override.test \
+parallel-tests-log-compiler-example.test \
parallel-tests-empty-testlogs.test \
-parallel-test-driver-install.test \
+parallel-tests-driver-install.test \
+parallel-tests-no-color-in-log.test \
+parallel-tests-no-spurious-summary.test \
+parallel-tests-exit-statuses.test \
+parallel-tests-console-output.test \
+parallel-tests-once.test \
+tests-environment.test \
+am-tests-environment.test \
+tests-environment-backcompat.test \
+testsuite-summary-color.test \
+testsuite-summary-count.test \
+testsuite-summary-count-many.test \
+testsuite-summary-reference-log.test \
test-driver-custom-no-extra-driver.test \
test-driver-custom.test \
test-driver-custom-xfail-tests.test \
@@ -999,10 +1015,18 @@ test-driver-custom-multitest-recheck2.test \
test-driver-custom-html.test \
test-driver-custom-no-html.test \
test-driver-create-log-dir.test \
-test-driver-global-log.test \
-test-driver-recheck.test \
test-driver-strip-vpath.test \
+test-driver-trs-suffix-registered.test \
test-driver-fail.test \
+test-log.test \
+test-metadata-global-log.test \
+test-metadata-recheck.test \
+test-metadata-results.test \
+test-missing.test \
+test-missing2.test \
+test-trs-basic.test \
+test-trs-recover.test \
+test-trs-recover2.test \
parse.test \
percent.test \
percent2.test \
@@ -1101,6 +1125,7 @@ remake-subdir-from-subdir.test \
remake-subdir-gnu.test \
remake-subdir.test \
remake-subdir2.test \
+remake-subdir-long-time.test \
remake-gnulib-add-acsubst.test \
remake-gnulib-add-header.test \
remake-gnulib-remove-header.test \
@@ -1125,6 +1150,7 @@ self-check-is_newest.test \
self-check-me.test \
self-check-report.test \
self-check-sanity.test \
+self-check-tap.test \
self-check-unindent.test \
sanity.test \
scripts.test \
@@ -1232,6 +1258,8 @@ suffix13.test \
symlink.test \
symlink2.test \
syntax.test \
+$(tap_with_common_setup_tests) \
+$(tap_other_tests) \
tags.test \
tags2.test \
tagsub.test \
@@ -1343,10 +1371,75 @@ yflags-force-conditional.test \
yflags-var-expand.test \
$(parallel_tests)
+
+# List of tests on TAP support that use the files pre-computed by
+# `tap-common-setup.test', and sources the `tap-setup.sh' helper
+# script.
+tap_with_common_setup_tests = \
+tap-autonumber.test \
+tap-bailout.test \
+tap-color.test \
+tap-deps.test \
+tap-diagnostic.test \
+tap-empty-diagnostic.test \
+tap-empty.test \
+tap-escape-directive.test \
+tap-exit.test \
+tap-signal.test \
+tap-fancy.test \
+tap-fancy2.test \
+tap-global-log.test \
+tap-global-result.test \
+tap-html.test \
+tap-log.test \
+tap-merge-stdout-stderr.test \
+tap-no-merge-stdout-stderr.test \
+tap-message-0.test \
+tap-no-disable-hard-error.test \
+tap-no-spurious-summary.test \
+tap-no-spurious.test \
+tap-not-ok-skip.test \
+tap-numeric-description.test \
+tap-out-of-order.test \
+tap-passthrough.test \
+tap-passthrough-exit.test \
+tap-plan.test \
+tap-plan-corner.test \
+tap-plan-corner2.test \
+tap-plan-errors.test \
+tap-realtime.test \
+tap-recheck-logs.test \
+tap-skip-whole-whitespace.test \
+tap-skip-whole.test \
+tap-todo-skip-together.test \
+tap-todo-skip-whitespace.test \
+tap-todo-skip.test \
+tap-unplanned.test \
+tap-whitespace-normalization.test \
+tap-with-and-without-number.test \
+tap-xfail-tests.test
+
+
+# Their log files.
+tap_with_common_setup_logs = $(tap_with_common_setup_tests:.test=.log)
+
+# Other tests on TAP support.
+tap_other_tests = \
+tap-common-setup.test \
+tap-bad-prog.tap \
+tap-basic.test \
+tap-doc.test \
+tap-doc2.test \
+tap-more.test \
+tap-more2.test \
+tap-recheck.test \
+tap-summary.test \
+tap-summary-color.test
+
all: all-am
.SUFFIXES:
-.SUFFIXES: .html .log .test
+.SUFFIXES: .html .log .tap .test .trs
$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(srcdir)/parallel-tests.am
$(srcdir)/instspc-tests.am $(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
@@ -1392,91 +1485,123 @@ CTAGS:
cscope cscopelist:
+# Recover from deleted `.trs' file; this should ensure that
+# "rm -f foo.log; make foo.trs" re-run `foo.test', and re-create
+# both `foo.log' and `foo.trs'.
+.log.trs:
+ rm -f $< $@ && $(am__stealth_MAKE) $(AM_MAKEFLAGS) $<
+
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
- rst_magic=":test-result:"; \
- list='$(TEST_LOGS)'; \
- list2=`for f in $$list; do test ! -r $$f || echo $$f; done`; \
- results1=`for f in $$list; do test -r $$f || echo FAIL; done`; \
- results2=''; \
- if test -n "$$list2"; then \
- results2=`sed -n "s/^$$rst_magic[ ]*//p" $$list2`; \
+ @$(am__sh_e_setup); $(am__tty_colors); \
+ bases='$(am__TEST_BASES)'; \
+ ws='[ ]'; \
+ redo_bases=`for b in $$bases; do \
+ test -f $$b.trs && test -r $$b.trs \
+ && test -f $$b.log && test -r $$b.log \
+ || echo $$b; \
+ done`; \
+ if test -n "$$redo_bases"; then \
+ redo_logs=`for i in $$redo_bases; do echo $$i.log; done`; \
+ redo_results=`for i in $$redo_bases; do echo $$i.trs; done`; \
+ rm -f $$redo_logs && rm -f $$redo_results \
+ && $(am__stealth_MAKE) $(AM_MAKEFLAGS) $$redo_logs; \
fi; \
- results=`echo "$$results1" && echo "$$results2"` \
- all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[
]*//'`; \
- fail=`echo "$$results" | grep -c '^FAIL'`; \
- pass=`echo "$$results" | grep -c '^PASS'`; \
- skip=`echo "$$results" | grep -c '^SKIP'`; \
- xfail=`echo "$$results" | grep -c '^XFAIL'`; \
- xpass=`echo "$$results" | grep -c '^XPASS'`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+ st=0; \
+ for b in $$redo_bases; do \
+ for e in trs log; do \
+ if test ! -f $$b.$$e || test ! -r $$b.$$e; then \
+ echo "fatal: making $@: failed to create $$b.$$e" >&2; \
+ st=1; \
+ fi; \
+ done; \
+ done; \
+ test $$st -eq 0 || exit 1; \
+ results=`for b in $$bases; do echo $$b.trs; done`; \
+ test -n "$$results" || results=/dev/null; \
+ all=` grep "^$$ws*:test-result:" $$results | wc -l`; \
+ pass=` grep "^$$ws*:test-result:$$ws*PASS" $$results | wc -l`; \
+ fail=` grep "^$$ws*:test-result:$$ws*FAIL" $$results | wc -l`; \
+ skip=` grep "^$$ws*:test-result:$$ws*SKIP" $$results | wc -l`; \
+ xfail=`grep "^$$ws*:test-result:$$ws*XFAIL" $$results | wc -l`; \
+ xpass=`grep "^$$ws*:test-result:$$ws*XPASS" $$results | wc -l`; \
+ error=`grep "^$$ws*:test-result:$$ws*ERROR" $$results | wc -l`; \
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+ br='==================='; br=$$br$$br$$br$$br; \
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
- for f in $$list; do \
- test -r $$f && read line < $$f || line=; \
- case $$line in \
- PASS:*|XFAIL:*);; \
- *) echo; cat $$f;; \
- esac; \
+ for b in $$bases; do \
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null; then :; \
+ elif test ! -r $$b.log; then \
+ echo "ERROR: cannot read $$b.log" >&2; \
+ echo && echo "WARNING: could not read $$b.log!"; \
+ else \
+ echo && cat $$b.log; \
+ fi; \
done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
- $(am__tty_colors); \
- if $$exit; then \
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
+ fi; \
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+ $$success
# Run all the tests.
check-TESTS:
@list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
+ @list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list
@test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
@list='$(TEST_LOGS)'; \
list=`for f in $$list; do \
@@ -1509,20 +1634,26 @@ check-html:
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML) || exit 4; \
exit $$rv
recheck recheck-html:
- @target=`echo $@ | sed 's,^re,,'`; \
- list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test -f $$f || continue; \
- if test -r $$f && read line < $$f; then \
- case $$line in PASS:*|XFAIL:*|SKIP:*);; *) echo $$f;; esac; \
- else echo $$f; fi; \
+ @ws='[ ]'; \
+ target=`echo $@ | sed 's,^re,,'`; \
+ bases='$(am__TEST_BASES)'; \
+ list=`for b in $$bases; do \
+ test ! -f $$b.trs && test ! -f $$b.log && continue; \
+ grep "^$$ws*:recheck:$$ws*no$$ws*$$" $$b.trs \
+ >/dev/null 2>&1 || echo $$b.log; \
done | tr '\012\015' ' '`; \
list=`echo "$$list" | sed 's/ *$$//'`; \
$(MAKE) $(AM_MAKEFLAGS) $$target AM_MAKEFLAGS='$(AM_MAKEFLAGS)
TEST_LOGS="'"$$list"'"'
.test.log:
- @p='$<'; $(am__check_pre) \
- $(TEST_LOG_DRIVER) $(am__test_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS)
$(TEST_LOG_DRIVER_FLAGS) -- \
- $(TEST_LOG_COMPILE) "$$tst"
+ @p='$<'; $(am__check_pre) $(TEST_LOG_DRIVER) --test-name "$$f" \
+ --log-file '$*.log' --trs-file '$*.trs' \
+ $(am__common_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS)
$(TEST_LOG_DRIVER_FLAGS) -- $(TEST_LOG_COMPILE) "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
+.tap.log:
+ @p='$<'; $(am__check_pre) $(TAP_LOG_DRIVER) --test-name "$$f" \
+ --log-file '$*.log' --trs-file '$*.trs' \
+ $(am__common_driver_flags) $(AM_TAP_LOG_DRIVER_FLAGS)
$(TAP_LOG_DRIVER_FLAGS) -- $(TAP_LOG_COMPILE) "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
@@ -1583,6 +1714,7 @@ mostlyclean-generic:
-test -z "$(TEST_LOGS_TMP)" || rm -f $(TEST_LOGS_TMP)
-test -z "$(TEST_SUITE_HTML)" || rm -f $(TEST_SUITE_HTML)
-test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
+ -test -z "$(am__TEST_BASES:=.trs)" || rm -f $(am__TEST_BASES:=.trs)
clean-generic:
@@ -1676,10 +1808,9 @@ uninstall-am:
backcompat5-p.log: backcompat5.test
check-exported-srcdir-p.log: check-exported-srcdir.test
+check-fd-redirect-p.log: check-fd-redirect.test
check-tests-in-builddir-p.log: check-tests-in-builddir.test
-check-tests_environment-p.log: check-tests_environment.test
check-p.log: check.test
-check10-p.log: check10.test
check11-p.log: check11.test
check12-p.log: check12.test
check2-p.log: check2.test
@@ -1698,6 +1829,7 @@ exeext4-p.log: exeext4.test
maken3-p.log: maken3.test
maken4-p.log: maken4.test
posixsubst-tests-p.log: posixsubst-tests.test
+tests-environment-p.log: tests-environment.test
$(srcdir)/parallel-tests.am: gen-parallel-tests Makefile.am
$(AM_V_GEN)($(am__cd) $(srcdir) && $(SHELL) ./gen-parallel-tests) >$@
@@ -1759,8 +1891,22 @@ test-driver-custom-multitest-recheck.log:
trivial-test-driver
test-driver-custom-multitest-recheck2.log: trivial-test-driver
test-driver-custom-html.log: trivial-test-driver
+testsuite-summary-color.log testsuite-summary-count.log: \
+ testsuite-summary-checks.sh extract-testsuite-summary
+
+testsuite-summary-count-many.log: trivial-test-driver
+testsuite-summary-count-many.log: extract-testsuite-summary
+
+# Their dependencies.
+$(tap_with_common_setup_logs): tap-common-setup.log tap-setup.sh
+
+tap-summary.log tap-summary-color.log: tap-summary-aux.sh
+
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
+# FIXME: this should be made more granular once we have a cleaner
+# subdivision of the tests.
+$(TEST_LOGS): plain-functions.sh tap-functions.sh
clean-local: clean-local-check
.PHONY: clean-local-check
diff --git a/tests/README b/tests/README
index 3c33c87..b385915 100644
--- a/tests/README
+++ b/tests/README
@@ -113,6 +113,13 @@ Do
Cite the PR number (if any), and the original reporter (if any), so
we can find or ask for information if needed.
+ If a test checks examples or idioms given in the documentation, make
+ sure the documentation reference them appropriately in comments, as in:
+ @c Keep in sync with autodist-config-headers.test.
+ @example
+ ...
+ @end example
+
Use `required=...' for required tools. Do not explicitly require
tools which can be taken for granted because they're listed in the
GNU Coding Standards (for example, `gzip').
diff --git a/tests/ac-output-old.tap b/tests/ac-output-old.tap
new file mode 100755
index 0000000..74bbba3
--- /dev/null
+++ b/tests/ac-output-old.tap
@@ -0,0 +1,179 @@
+#!/bin/sh
+# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
+# Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test for various older bugs related to quoting, escaping and
+# line breaking in the use of AC_OUTPUT. Synthesised by a bunch
+# of older tests (referenced below).
+
+. ./defs || Exit 1
+
+rm -f configure.in depcomp # Not required.
+
+# -----------------------------------------------------------------------
+
+# Test for bug reported by François Pinard.
+# If \ is in AC_OUTPUT, automake barfs.
+# This was the old test `acoutbs.test'.
+
+# Also test for bug reported by David A. Swierczek.
+# Another bug with \ in AC_OUTPUT (this time with whitespace).
+# This was the old test `acoutbs2.test'.
+
+acoutbs_check ()
+{
+ dir=acoutbs$1
+ mkdir $dir
+ cd $dir
+ cat > configure.in
+ touch Makefile.am zot.in
+ command_ok_ "aclocal groks '\\' in AC_OUTPUT ($dir)" $ACLOCAL
+ command_ok_ "automake groks '\\' in AC_OUTPUT ($dir)" $AUTOMAKE
+ command_ok_ "autoconf groks '\\' in AC_OUTPUT ($dir)" $AUTOCONF
+ command_ok_ "can ./configure in $dir" ./configure
+ command_ok_ "zot created in $dir" test -f zot
+ ls -a > lst || bailout_ "cannot get file listing in $dir"
+ command_not_ok_ "'\\' not leaked in filenames in $dir" grep '\\' lst
+ cd ..
+}
+
+acoutbs_check 1 << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_OUTPUT(Makefile \
+zot)
+END
+
+acoutbs_check 2 << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_ARG_PROGRAM
+AC_PROG_INSTALL
+AC_PROG_MAKE_SET
+AC_OUTPUT(\
+ Makefile \
+ zot
+)
+END
+
+# -----------------------------------------------------------------------
+
+# Test for bug reported by Jerome Santini.
+# If I put this line in my configure.in:
+# AC_OUTPUT(Makefile src/Makefile tests/Makefile, echo timestamp > stamp-h)dnl
+# automake is not happy:
+# [ ... ]
+# This was the old test `acoutnoq.test'.
+
+mkdir acoutnoq
+cd acoutnoq
+
+cat > configure.in << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_OUTPUT(Makefile, [true])
+END
+
+: > Makefile.am
+
+command_ok_ "aclocal and quoted AC_OUTPUT second argument" $ACLOCAL
+command_ok_ "automake and quoted AC_OUTPUT second argument" $AUTOMAKE
+
+cd ..
+
+# -----------------------------------------------------------------------
+
+# Test for bug when AC_OUTPUT has 2 args on the same line, eg:
+# AC_OUTPUT([Makefile automake tests/Makefile],[chmod +x automake])
+# This was the old test `acoutpt.test'.
+
+mkdir acoutpt
+cd acoutpt
+
+cat > configure.in << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_OUTPUT([Makefile], [true])
+END
+
+: > Makefile.am
+
+command_ok_ "aclocal and two AC_OUTPUT arguments on same line" $ACLOCAL
+command_ok_ "automake and two AC_OUTPUT arguments on same line" $AUTOMAKE
+
+cd ..
+
+# -----------------------------------------------------------------------
+
+# Test for bug reported by Eric Magnien.
+# This was the old test `acoutpt2.test'.
+
+mkdir acoutpt2
+cd acoutpt2
+
+# Name of the current "subtest".
+cur=acoutput2
+
+cat > configure.in <<END
+AC_INIT([$cur], [1.0])
+AM_INIT_AUTOMAKE
+AC_OUTPUT([subdir/Makefile subdir/foo Makefile foo], [true])
+END
+
+mkdir subdir
+
+: > Makefile.am
+: > subdir/Makefile.am
+: > foo.in
+: > subdir/foo.in
+
+command_ok_ "aclocal and AC_OUTPUT ($cur)" $ACLOCAL
+command_ok_ "automake and AC_OUTPUT ($cur)" $AUTOMAKE
+command_ok_ "foo.in mentioned two times in Makefile.in ($cur)" \
+ test `$FGREP foo.in Makefile.in | wc -l` -eq 2
+
+# This ought to work as well.
+command_ok_ "'automake -a -f' and AC_OUTPUT ($cur)" \
+ $AUTOMAKE --add-missing --force-missing
+
+cd ..
+
+# -----------------------------------------------------------------------
+
+# Test for bug reported by François Pinard.
+# This was the old test `acoutqnl.test'.
+
+mkdir acoutqnl
+cd acoutqnl
+
+cat > configure.in << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_OUTPUT([Makefile],
+[echo zardoz has spoken])
+END
+
+: > Makefile.am
+
+command_ok_ "aclocal and two AC_OUTPUT arguments on two lines" $ACLOCAL
+command_ok_ "automake and two AC_OUTPUT arguments on two lines" $AUTOMAKE
+
+cd ..
+
+# -----------------------------------------------------------------------
+
+# And we're done.
+Exit 0
diff --git a/tests/acoutbs.test b/tests/acoutbs.test
deleted file mode 100755
index 232267d..0000000
--- a/tests/acoutbs.test
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/sh
-# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
-# Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by François Pinard.
-# If \ is in AC_OUTPUT, automake barfs.
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_OUTPUT(Makefile \
-zot)
-END
-
-: > Makefile.am
-: > zot.in
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE
-
-./configure
-
-test -f zot || Exit 1
-ls -a | grep '\\' && Exit 1
-
-:
diff --git a/tests/acoutbs2.test b/tests/acoutbs2.test
deleted file mode 100755
index cd1cd9d..0000000
--- a/tests/acoutbs2.test
+++ /dev/null
@@ -1,47 +0,0 @@
-#! /bin/sh
-# Copyright (C) 1996, 2000, 2002, 2010, 2011 Free Software Foundation,
-# Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by David A. Swierczek.
-# Another bug with \ in AC_OUTPUT (this time with whitespace).
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_ARG_PROGRAM
-AC_PROG_INSTALL
-AC_PROG_MAKE_SET
-AC_OUTPUT(\
- Makefile \
- zot
-)
-END
-
-: > Makefile.am
-: > zot.in
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE
-
-./configure
-
-test -f zot || Exit 1
-ls -a | grep '\\' && Exit 1
-
-:
diff --git a/tests/acoutnoq.test b/tests/acoutnoq.test
deleted file mode 100755
index d49f878..0000000
--- a/tests/acoutnoq.test
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/sh
-# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
-# Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by Jerome Santini.
-# If I put this line in my configure.in:
-# AC_OUTPUT(Makefile src/Makefile tests/Makefile, echo timestamp > stamp-h)dnl
-# automake is not happy:
-# [ ... ]
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_OUTPUT(Makefile, [true])
-END
-
-: > Makefile.am
-
-$ACLOCAL
-$AUTOMAKE
diff --git a/tests/acoutpt.test b/tests/acoutpt.test
deleted file mode 100755
index 12b6427..0000000
--- a/tests/acoutpt.test
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
-# Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug when AC_OUTPUT has 2 args on the same line, eg:
-# AC_OUTPUT([Makefile automake tests/Makefile],[chmod +x automake])
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_OUTPUT([Makefile], [true])
-END
-
-: > Makefile.am
-
-$ACLOCAL
-$AUTOMAKE
diff --git a/tests/acoutpt2.test b/tests/acoutpt2.test
deleted file mode 100755
index d972989..0000000
--- a/tests/acoutpt2.test
+++ /dev/null
@@ -1,44 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2001, 2002, 2010, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by Eric Magnien.
-
-. ./defs || Exit 1
-
-cat > configure.in <<END
-AC_INIT([$me], [1.0])
-AM_INIT_AUTOMAKE
-AC_OUTPUT([subdir/Makefile subdir/foo Makefile foo], [true])
-END
-
-mkdir subdir
-
-: > Makefile.am
-: > subdir/Makefile.am
-: > foo.in
-: > subdir/foo.in
-
-$ACLOCAL
-$AUTOMAKE
-
-count=`$FGREP foo.in Makefile.in | wc -l`
-test $count -eq 2
-
-# This ought to work as well.
-$ACLOCAL
-$AUTOMAKE --add-missing --force-missing
-
-:
diff --git a/tests/acoutqnl.test b/tests/acoutqnl.test
deleted file mode 100755
index 5cfac28..0000000
--- a/tests/acoutqnl.test
+++ /dev/null
@@ -1,32 +0,0 @@
-#! /bin/sh
-# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
-# Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by François Pinard.
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_OUTPUT([Makefile],
-[echo zardoz has spoken])
-END
-
-: > Makefile.am
-
-$ACLOCAL
-$AUTOMAKE
diff --git a/tests/am-tests-environment.test b/tests/am-tests-environment.test
new file mode 100755
index 0000000..436b6df
--- /dev/null
+++ b/tests/am-tests-environment.test
@@ -0,0 +1,93 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests: check AM_TESTS_ENVIRONMENT support, and its
+# interactions with TESTS_ENVIRONMENT.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in << 'END'
+AC_PROG_MKDIR_P
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .sh .test
+TESTS = foo.test bar.sh
+SH_LOG_COMPILER = sh
+AM_TESTS_ENVIRONMENT = \
+ $(MKDIR_P) quux.dir; \
+ if test -f $(srcdir)/test-env.sh; then \
+ . $(srcdir)/test-env.sh; \
+ fi; \
+ FOO=1; export FOO;
+EXTRA_DIST = $(TESTS) test-env.sh
+END
+
+cat > foo.test << 'END'
+#! /bin/sh
+ls -l && test -d quux.dir
+END
+chmod a+x foo.test
+
+cat > bar.sh << 'END'
+echo "FOO='$FOO'"
+echo "BAR='$BAR'"
+test x"$FOO" = x"$BAR"
+END
+
+debug_info ()
+{
+ cat test-suite.log
+ cat foo.log
+ cat bar.log
+}
+
+minicheck ()
+{
+ debug_info
+ test -d quux.dir
+}
+
+miniclean ()
+{
+ rmdir quux.dir
+ rm -f foo.log bar.log test-suite.log
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+TESTS_ENVIRONMENT='BAR=1' $MAKE check || { debug_info; Exit 1; }
+minicheck
+miniclean
+
+TESTS_ENVIRONMENT='BAR=2' $MAKE check && { debug_info; Exit 1; }
+minicheck
+miniclean
+
+echo 'BAR=1 && export BAR' > test-env.sh
+$MAKE check || { debug_info; Exit 1; }
+minicheck
+$MAKE distcheck
+
+:
diff --git a/tests/amhello-binpkg.test b/tests/amhello-binpkg.test
new file mode 100755
index 0000000..f11421f
--- /dev/null
+++ b/tests/amhello-binpkg.test
@@ -0,0 +1,50 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test an example from the manual about the `amhello' package:
+# using DESTDIR to build simple, no-frills binary packages.
+
+. ./defs || Exit 1
+
+set -e
+
+cp "$testsrcdir"/../doc/amhello-1.0.tar.gz . \
+ || fatal_ "cannot get amhello tarball"
+
+gzip -dc amhello-1.0.tar.gz | tar xf -
+cd amhello-1.0
+
+./configure --prefix /usr
+$MAKE
+$MAKE DESTDIR="`pwd`/inst" install
+cd inst
+find . -type f -print > ../files.lst
+tar cvf amhello-1.0-i686.tar.gz `cat ../files.lst` > tar.got 2>&1
+
+if tar --version </dev/null | grep GNU; then
+ LC_ALL=C sort tar.got > t
+ mv -f t tar.got
+ diff - tar.got <<'END'
+./usr/bin/hello
+./usr/share/doc/amhello/README
+END
+else
+ : Be laxer with other tar implementations, to avoid spurious failures.
+ $EGREP '(^| )\./usr/bin/hello( |$)' tar.got
+ $EGREP '(^| )\./usr/share/doc/amhello/README( |$)' tar.got
+fi
+
+:
diff --git a/tests/amhello-cflags.test b/tests/amhello-cflags.test
new file mode 100755
index 0000000..56f7729
--- /dev/null
+++ b/tests/amhello-cflags.test
@@ -0,0 +1,51 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test an example from the manual about the `amhello' package:
+# using VPATH builds to configure the same package with different
+# options.
+
+required=gcc
+. ./defs || Exit 1
+
+set -e
+
+cp "$testsrcdir"/../doc/amhello-1.0.tar.gz . \
+ || fatal_ "cannot get amhello tarball"
+
+gzip -dc amhello-1.0.tar.gz | tar xf -
+cd amhello-1.0
+
+mkdir debug optim
+cd debug
+../configure CFLAGS='-g -O0'
+$MAKE
+ls -l . src # For debugging.
+cd ../optim
+../configure CFLAGS='-O3 -fomit-frame-pointer'
+$MAKE
+ls -l . src # For debugging.
+cd ..
+
+# Check that we have really compiled the objects with two different
+# configurations.
+for exeext in '' .exe :; do
+ test -f optim/src/hello$exeext && break
+ test "$exeext" = : && fatal_ "cannot determine extension of executables"
+done
+cmp optim/src/hello$exeext debug/src/hello$exeext && Exit 1
+
+:
diff --git a/tests/amhello-cross-compile.test b/tests/amhello-cross-compile.test
new file mode 100755
index 0000000..6db91ba
--- /dev/null
+++ b/tests/amhello-cross-compile.test
@@ -0,0 +1,54 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test an example from the manual about the `amhello' package:
+# cross-compiling a package from Linux/Unix to MinGW.
+
+required=i586-mingw32msvc-gcc
+. ./defs || Exit 1
+
+set -e
+
+cp "$testsrcdir"/../doc/amhello-1.0.tar.gz . \
+ || fatal_ "cannot get amhello tarball"
+
+host=i586-mingw32msvc
+build=`"$testsrcdir"/../lib/config.guess` && test -n "$build" \
+ || fatal_ "cannot guess build platform"
+case $build in *mingw*) skip_ "build system is MinGW too";; esac
+
+gzip -dc amhello-1.0.tar.gz | tar xf -
+cd amhello-1.0
+
+./configure --build "$build" --host "$host" > stdout \
+ || { cat stdout ; Exit 1; }
+cat stdout
+grep '^checking for i586-mingw32msvc-strip\.\.\.' stdout
+grep '^checking for i586-mingw32msvc-gcc\.\.\.' stdout
+grep '^checking for suffix of executables\.\.\. \.exe *$' stdout
+grep '^checking for suffix of object files\.\.\. o *$' stdout
+grep '^checking whether i586-mingw32msvc-gcc accepts -g\.\.\. yes' stdout
+
+$MAKE
+
+cd src
+file hello.exe > whatis
+cat whatis
+$EGREP 'DOS|Win' whatis
+grep 'executable' whatis
+grep 'ELF' whatis && Exit 1
+
+:
diff --git a/tests/check-fd-redirect.test b/tests/check-fd-redirect.test
new file mode 100755
index 0000000..6da8704
--- /dev/null
+++ b/tests/check-fd-redirect.test
@@ -0,0 +1,110 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Simple Tests support: redirection of file descriptors with
+# AM_TESTS_FD_REDIRECT.
+# See also related test 'parallel-tests-fd-redirect.test'.
+
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = foo.test
+AM_TESTS_FD_REDIRECT = 3<three 4>four 5>>five 7<&0 8>&1 9>&2
+END
+
+echo '3333' > three
+chmod a-w three
+
+: > foo.test
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+./configure
+
+do_check ()
+{
+ cat foo.test # For debugging.
+ echo 'this line will be removed' > four
+ echo 'this line will not be removed' > five
+ st=0
+ echo 'ok ok ok' | $MAKE check >stdout 2>stderr || st=1
+ cat stdout
+ cat stderr >&2
+ cat four
+ test x"$parallel_tests" != x"yes" || cat foo.log
+ test $st -eq 0
+ grep '[ /]foo\.test: foofoofoo$' stdout
+ grep '[ /]foo\.test: barbarbar$' stderr
+ grep 'this line' four && Exit 1
+ grep '^3333$' four
+ grep '^this line will not be removed$' five
+ grep '^ok ok ok$' five
+ $EGREP '(foofoofoo|barbarbar|3333|ok ok ok|this line)' foo.log && Exit 1
+ :
+}
+
+# Try using both shell script and a perl script as the test, for
+# better coverage.
+
+cat > foo.test <<'END'
+#! /bin/sh
+set -e
+
+read FOO <&3
+test 3333 -eq "$FOO"
+echo "$FOO" >&4
+
+grep '^ok ok ok$' <&7 >&5
+
+echo " " $0: foofoofoo >&8
+echo " " $0: barbarbar >&9
+END
+
+do_check
+
+echo "#! $PERL -w" > foo.test
+cat >> foo.test <<'END'
+use warnings FATAL => 'all';
+use strict;
+
+open (FD3, "<&=3") or die "opening FD3: $!";
+open (FD4, ">&=4") or die "opening FD4: $!";
+open (FD5, ">&=5") or die "opening FD5: $!";
+open (FD7, "<&=7") or die "opening FD7: $!";
+open (FD8, ">&=8") or die "opening FD8: $!";
+open (FD9, ">&=9") or die "opening FD9: $!";
+
+chomp (my $FOO = <FD3>);
+die "$FOO != 3333" if not $FOO eq "3333";
+print FD4 "$FOO\n";
+
+chomp ($_ = <FD7>);
+die "$_ != 'ok ok ok'" if not $_ eq 'ok ok ok';
+print FD5 "$_\n";
+
+print FD8 " $0: foofoofoo\n";
+print FD9 " $0: barbarbar\n";
+END
+
+do_check
+
+:
diff --git a/tests/check10.test b/tests/check10.test
index ef79969..a13c8590 100755
--- a/tests/check10.test
+++ b/tests/check10.test
@@ -16,6 +16,8 @@
# Check singular and plural in test summaries.
+# This test only makes sense for the older serial testsuite driver.
+parallel_tests=no
. ./defs || Exit 1
cat >> configure.in << 'END'
diff --git a/tests/check11.test b/tests/check11.test
index 912b619..acd05b0 100755
--- a/tests/check11.test
+++ b/tests/check11.test
@@ -38,12 +38,23 @@ $AUTOCONF
$AUTOMAKE -a
./configure
+
env TESTS=skip $MAKE -e check >stdout
cat stdout
-grep '1.*passed' stdout && Exit 1
+if test x"$parallel_tests" = x"yes"; then
+ count_test_results total=1 pass=0 fail=0 skip=1 xfail=0 xpass=0 error=0
+else
+ grep '1.*passed' stdout && Exit 1
+ : For shells with buggy 'set -e'.
+fi
env TESTS="skip skip2" $MAKE -e check >stdout
cat stdout
-grep '2.*passed' stdout && Exit 1
+if test x"$parallel_tests" = x"yes"; then
+ count_test_results total=2 pass=0 fail=0 skip=2 xfail=0 xpass=0 error=0
+else
+ grep '2.*passed' stdout && Exit 1
+ : For shells with buggy 'set -e'.
+fi
:
diff --git a/tests/color.test b/tests/color.test
index ecd4f94..2b56a32 100755
--- a/tests/color.test
+++ b/tests/color.test
@@ -22,11 +22,14 @@
TERM=ansi
export TERM
-red='[0;31m'
-grn='[0;32m'
-lgn='[1;32m'
-blu='[1;34m'
-std='[m'
+esc=''
+# Escape `[' for grep, below.
+red="$esc\[0;31m"
+grn="$esc\[0;32m"
+lgn="$esc\[1;32m"
+blu="$esc\[1;34m"
+mgn="$esc\[0;35m"
+std="$esc\[m"
# Check that grep can parse nonprinting characters.
# BSD 'grep' works from a pipe, but not a seekable file.
@@ -43,7 +46,7 @@ END
cat >Makefile.am <<'END'
AUTOMAKE_OPTIONS = color-tests
TESTS = $(check_SCRIPTS)
-check_SCRIPTS = pass fail skip xpass xfail
+check_SCRIPTS = pass fail skip xpass xfail error
XFAIL_TESTS = xpass xfail
END
@@ -62,39 +65,93 @@ cat >skip <<END
exit 77
END
+cat >error <<END
+#! /bin/sh
+exit 99
+END
+
cp fail xfail
cp pass xpass
-chmod +x pass fail skip xpass xfail
+chmod +x pass fail skip xpass xfail error
$ACLOCAL
$AUTOCONF
$AUTOMAKE --add-missing
-./configure
-
test_color ()
{
# Not a useless use of cat; see above comments about grep.
- cat stdout | grep ": pass" | $FGREP "$grn"
- cat stdout | grep ": fail" | $FGREP "$red"
- cat stdout | grep ": xfail" | $FGREP "$lgn"
- cat stdout | grep ": xpass" | $FGREP "$red"
- cat stdout | grep ": skip" | $FGREP "$blu"
+ cat stdout | grep "^${grn}PASS${std}: .*pass"
+ cat stdout | grep "^${red}FAIL${std}: .*fail"
+ cat stdout | grep "^${blu}SKIP${std}: .*skip"
+ cat stdout | grep "^${lgn}XFAIL${std}: .*xfail"
+ cat stdout | grep "^${red}XPASS${std}: .*xpass"
+ # The old serial testsuite driver doesn't distinguish between failures
+ # and hard errors.
+ if test x"$parallel_tests" = x"yes"; then
+ cat stdout | grep "^${mgn}ERROR${std}: .*error"
+ else
+ cat stdout | grep "^${red}FAIL${std}: .*error"
+ fi
+ :
}
test_no_color ()
{
- # Not a useless use of cat; see above comments about grep.
- cat stdout | grep ": pass" | $FGREP "$grn" && Exit 1
- cat stdout | grep ": fail" | $FGREP "$red" && Exit 1
- cat stdout | grep ": xfail" | $FGREP "$lgn" && Exit 1
- cat stdout | grep ": xpass" | $FGREP "$red" && Exit 1
- cat stdout | grep ": skip" | $FGREP "$blu" && Exit 1
- :
+ # With make implementations that, like Solaris make, in case of errors
+ # print the whole failing recipe on standard output, we should content
+ # ourselves with a laxer check, to avoid false positives.
+ # Keep this in sync with lib/am/check.am:$(am__color_tests).
+ if $FGREP '= Xalways || test -t 1 ' stdout; then
+ # Extra verbose make, resort to laxer checks.
+ # Note that we also want to check that the testsuite summary is
+ # not unduly colorized.
+ (
+ set +e # In case some grepped regex below isn't matched.
+ # Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "TOTAL.*:"
+ cat stdout | grep "PASS.*:"
+ cat stdout | grep "FAIL.*:"
+ cat stdout | grep "SKIP.*:"
+ cat stdout | grep "XFAIL.*:"
+ cat stdout | grep "XPASS.*:"
+ cat stdout | grep "ERROR.*:"
+ cat stdout | grep 'test.*expected'
+ cat stdout | grep 'test.*not run'
+ cat stdout | grep '===='
+ cat stdout | grep '[Ss]ee .*test-suite\.log'
+ cat stdout | grep '[Tt]estsuite summary'
+ ) | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ else
+ cat stdout | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ fi
}
-AM_COLOR_TESTS=always $MAKE -e check >stdout && { cat stdout; Exit 1; }
-cat stdout
-test_color
+for vpath in false :; do
+
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ AM_COLOR_TESTS=always $MAKE -e check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ test_color
+
+ $MAKE -e check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ test_no_color
+
+ $MAKE distclean
+ cd $srcdir
+
+done
:
diff --git a/tests/color2.test b/tests/color2.test
index 4f8862e..21d67ac 100755
--- a/tests/color2.test
+++ b/tests/color2.test
@@ -22,11 +22,14 @@
TERM=ansi
export TERM
-red='[0;31m'
-grn='[0;32m'
-lgn='[1;32m'
-blu='[1;34m'
-std='[m'
+esc=''
+# Escape `[' for grep, below.
+red="$esc\[0;31m"
+grn="$esc\[0;32m"
+lgn="$esc\[1;32m"
+blu="$esc\[1;34m"
+mgn="$esc\[0;35m"
+std="$esc\[m"
# Check that grep can parse nonprinting characters.
# BSD 'grep' works from a pipe, but not a seekable file.
@@ -56,7 +59,7 @@ END
cat >Makefile.am <<'END'
AUTOMAKE_OPTIONS = color-tests
TESTS = $(check_SCRIPTS)
-check_SCRIPTS = pass fail skip xpass xfail
+check_SCRIPTS = pass fail skip xpass xfail error
XFAIL_TESTS = xpass xfail
END
@@ -75,34 +78,68 @@ cat >skip <<END
exit 77
END
+cat >error <<END
+#! /bin/sh
+exit 99
+END
+
cp fail xfail
cp pass xpass
-chmod +x pass fail skip xpass xfail
+chmod +x pass fail skip xpass xfail error
$ACLOCAL
-$AUTOMAKE -a
$AUTOCONF
-./configure
+$AUTOMAKE --add-missing
test_color ()
{
# Not a useless use of cat; see above comments about grep.
- cat stdout | grep ": pass" | $FGREP "$grn"
- cat stdout | grep ": fail" | $FGREP "$red"
- cat stdout | grep ": xfail" | $FGREP "$lgn"
- cat stdout | grep ": xpass" | $FGREP "$red"
- cat stdout | grep ": skip" | $FGREP "$blu"
+ cat stdout | grep "^${grn}PASS${std}: .*pass"
+ cat stdout | grep "^${red}FAIL${std}: .*fail"
+ cat stdout | grep "^${blu}SKIP${std}: .*skip"
+ cat stdout | grep "^${lgn}XFAIL${std}: .*xfail"
+ cat stdout | grep "^${red}XPASS${std}: .*xpass"
+ # The old serial testsuite driver doesn't distinguish between failures
+ # and hard errors.
+ if test x"$parallel_tests" = x"yes"; then
+ cat stdout | grep "^${mgn}ERROR${std}: .*error"
+ else
+ cat stdout | grep "^${red}FAIL${std}: .*error"
+ fi
+ :
}
test_no_color ()
{
- # Not a useless use of cat; see above comments about grep.
- cat stdout | grep ": pass" | $FGREP "$grn" && Exit 1
- cat stdout | grep ": fail" | $FGREP "$red" && Exit 1
- cat stdout | grep ": xfail" | $FGREP "$lgn" && Exit 1
- cat stdout | grep ": xpass" | $FGREP "$red" && Exit 1
- cat stdout | grep ": skip" | $FGREP "$blu" && Exit 1
- :
+ # With make implementations that, like Solaris make, in case of errors
+ # print the whole failing recipe on standard output, we should content
+ # ourselves with a laxer check, to avoid false positives.
+ # Keep this in sync with lib/am/check.am:$(am__color_tests).
+ if $FGREP '= Xalways || test -t 1 ' stdout; then
+ # Extra verbose make, resort to laxer checks.
+ # Note that we also want to check that the testsuite summary is
+ # not unduly colorized.
+ (
+ set +e # In case some grepped regex below isn't matched.
+ # Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "TOTAL.*:"
+ cat stdout | grep "PASS.*:"
+ cat stdout | grep "FAIL.*:"
+ cat stdout | grep "SKIP.*:"
+ cat stdout | grep "XFAIL.*:"
+ cat stdout | grep "XPASS.*:"
+ cat stdout | grep "ERROR.*:"
+ cat stdout | grep 'test.*expected'
+ cat stdout | grep 'test.*not run'
+ cat stdout | grep '===='
+ cat stdout | grep '[Ss]ee .*test-suite\.log'
+ cat stdout | grep '[Tt]estsuite summary'
+ ) | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ else
+ cat stdout | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ fi
}
cat >expect-make <<'END'
@@ -111,14 +148,31 @@ eval spawn $env(MAKE) -e check
expect eof
END
-MAKE=$MAKE expect -f expect-make >stdout \
- || { cat stdout; Exit 1; }
-cat stdout
-test_color
+for vpath in false :; do
+
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ MAKE=$MAKE expect -f $srcdir/expect-make >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ test_color
+
+ AM_COLOR_TESTS=no MAKE=$MAKE expect -f $srcdir/expect-make >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ test_no_color
+
+ $MAKE distclean
+ cd $srcdir
-AM_COLOR_TESTS=no MAKE=$MAKE expect -f expect-make >stdout \
- || { cat stdout; Exit 1; }
-cat stdout
-test_no_color
+done
:
diff --git a/tests/defs b/tests/defs
index fb0b535..3aadda3 100644
--- a/tests/defs
+++ b/tests/defs
@@ -117,20 +117,19 @@ Exit ()
exit $1
}
-# Print warnings (e.g., about skipped and failed tests) to this file
-# number. Override by putting, say:
-# stderr_fileno_=9; export stderr_fileno_; exec 9>&2;
-# in the definition of AM_TESTS_ENVIRONMENT.
-# This is useful when using automake's parallel tests mode, to print the
-# reason for skip/failure to console, rather than to the *.log files.
-: ${stderr_fileno_=2}
-
-# Copied from Gnulib's `tests/init.sh'.
-warn_ () { echo "$@" 1>&$stderr_fileno_; }
-fail_ () { warn_ "$me: failed test: $@"; Exit 1; }
-skip_ () { warn_ "$me: skipped test: $@"; Exit 77; }
-fatal_ () { warn_ "$me: hard error: $@"; Exit 99; }
-framework_failure_ () { warn_ "$me: set-up failure: $@"; Exit 99; }
+if test $using_tap = yes; then
+ funcs_file_=tap-functions.sh
+else
+ funcs_file_=plain-functions.sh
+fi
+
+if test -f "$testsrcdir/$funcs_file_"; then
+ . "$testsrcdir/$funcs_file_"
+else
+ echo "$me: $testsrcdir/$funcs_file_ not found, check \$testsrcdir" >&2
+ Exit 99
+fi
+unset funcs_file_
# cross_compiling
# ---------------
@@ -147,7 +146,7 @@ cross_compiling ()
# Resolve ties in favor of FILE.
is_newest ()
{
- is_newest_files=`find "$@" -newer "$1"`
+ is_newest_files=`find "$@" -prune -newer "$1"`
test -z "$is_newest_files"
}
@@ -203,6 +202,43 @@ using_gmake ()
esac
}
+# count_test_results total=N pass=N fail=N xpass=N xfail=N skip=N error=N
+# -----------------------------------------------------------------------
+# Check that a testsuite run driven by the parallel-tests harness has
+# had the specified numbers of test results (specified by kind).
+# This function assumes that the output of "make check" or "make recheck"
+# has been saved in the `stdout' file in the current directory, and its
+# log in the `test-suite.log' file.
+count_test_results ()
+{
+ # Use a subshell so that we won't pollute the script namespace.
+ (
+ # TODO: Do proper checks on the arguments?
+ total=ERR pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
+ eval "$@"
+ # For debugging.
+ $EGREP -i '(total|x?pass|x?fail|skip|error)' stdout || :
+ rc=0
+ # Avoid spurious failures with shells with "overly sensible"
+ # `errexit' shell flag, such as e.g., Solaris /bin/sh.
+ set +e
+ test `grep -c '^PASS:' stdout` -eq $pass || rc=1
+ test `grep -c '^XFAIL:' stdout` -eq $xfail || rc=1
+ test `grep -c '^SKIP:' stdout` -eq $skip || rc=1
+ test `grep -c '^FAIL:' stdout` -eq $fail || rc=1
+ test `grep -c '^XPASS:' stdout` -eq $xpass || rc=1
+ test `grep -c '^ERROR:' stdout` -eq $error || rc=1
+ grep "^# TOTAL: *$total$" stdout || rc=1
+ grep "^# PASS: *$pass$" stdout || rc=1
+ grep "^# XFAIL: *$xfail$" stdout || rc=1
+ grep "^# SKIP: *$skip$" stdout || rc=1
+ grep "^# FAIL: *$fail$" stdout || rc=1
+ grep "^# XPASS: *$xpass$" stdout || rc=1
+ grep "^# ERROR: *$error$" stdout || rc=1
+ test $rc -eq 0
+ )
+}
+
commented_sed_unindent_prog='
/^$/b # Nothing to do for empty lines.
x # Get x<indent> into pattern space.
@@ -604,8 +640,10 @@ if test "$sh_errexit_works" = yes; then
rm -rf $testSubDir
;;
esac
+ set +x
test "$signal" != 0 && echo "$me: caught signal $signal"
echo "$me: exit $exit_status"
+ test $using_tap = yes && late_plan_
exit $exit_status
' 0
for signal in 1 2 13 15; do
diff --git a/tests/defs-static.in b/tests/defs-static.in
index 6d82956..f1294c6 100644
--- a/tests/defs-static.in
+++ b/tests/defs-static.in
@@ -66,7 +66,9 @@ fi
# Check that the environment is properly sanitized.
# Having variables exported to the empty string is OK, since our code
# treats such variables as if they were unset.
-for var in me required parallel_tests original_AUTOMAKE original_ACLOCAL; do
+for var in me required use_tap parallel_tests \
+ original_AUTOMAKE original_ACLOCAL;
+do
if eval "test x\"\$$var\" != x" && env | grep "^$var=" >/dev/null; then
echo "$argv0: variable \`$var' is set in the environment:" \
"this is unsafe" >&2
@@ -75,6 +77,13 @@ for var in me required parallel_tests original_AUTOMAKE
original_ACLOCAL; do
done
unset var
+# See whether the current test script is expected to use TAP or not.
+# Use a sensible default, while allowing the scripts to override this
+# check.
+if test -z "$using_tap"; then
+ case $argv0 in *.tap) using_tap=yes;; *) using_tap=no;; esac
+fi
+
testsrcdir='@abs_srcdir@'
top_testsrcdir='@abs_top_srcdir@'
testbuilddir='@abs_builddir@'
diff --git a/tests/extract-testsuite-summary b/tests/extract-testsuite-summary
new file mode 100644
index 0000000..a3a0d87
--- /dev/null
+++ b/tests/extract-testsuite-summary
@@ -0,0 +1,15 @@
+#! /usr/bin/env perl
+# Extract the testsuite summary generated by the parallel-tests harness
+# from the output of "make check".
+
+use warnings FATAL => 'all';
+use strict;
+
+my $br = '=' x 76;
+my @sections = ('');
+while (<>)
+ {
+ push @sections, $_, '' if /$br/;
+ $sections[-1] .= $_ if !/$br/;
+ }
+print @sections[1..$#sections-1];
diff --git a/tests/interp.test b/tests/interp.test
index e4352d7..911dc65 100755
--- a/tests/interp.test
+++ b/tests/interp.test
@@ -24,14 +24,16 @@ AC_PROG_CC
END
cat > Makefile.am << 'END'
-bin_PROGRAMS = qqq
-s1 = z.c
-qqq_SOURCES = $(s1)
+bin_PROGRAMS = foo
+xs = a.c b.c
+foo_SOURCES = c.c $(xs)
END
$ACLOCAL
$AUTOMAKE
-$FGREP 'z.$(OBJEXT)' Makefile.in
+$FGREP ' a.$(OBJEXT)' Makefile.in
+$FGREP ' b.$(OBJEXT)' Makefile.in
+$FGREP ' c.$(OBJEXT)' Makefile.in
:
diff --git a/tests/parallel-tests-am_tests_environment.test
b/tests/parallel-tests-am_tests_environment.test
deleted file mode 100755
index 072e614..0000000
--- a/tests/parallel-tests-am_tests_environment.test
+++ /dev/null
@@ -1,97 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# parallel-tests: check AM_TESTS_ENVIRONMENT support, and its
-# interactions with TESTS_ENVIRONMENT.
-
-parallel_tests=yes
-. ./defs || Exit 1
-
-set -e
-
-cat >> configure.in << 'END'
-AC_PROG_MKDIR_P
-AC_OUTPUT
-END
-
-cat > Makefile.am << 'END'
-TEST_EXTENSIONS = .sh .test
-TESTS = foo.test bar.sh
-SH_LOG_COMPILER = sh
-AM_TESTS_ENVIRONMENT = $(MKDIR_P) quux.dir; exec 9>&2; fd=9; export fd;
-EXTRA_DIST = $(TESTS)
-END
-
-cat > foo.test << 'END'
-#! /bin/sh
-echo "FOO='$FOO'"
-echo "BAR='$BAR'"
-test x"$FOO" = x"$BAR"
-END
-chmod a+x foo.test
-
-cat > bar.sh << 'END'
-: ${fd=2}
-eval "echo 'diagnostic message from test bar' >&${fd}"
-END
-
-minicheck ()
-{
- cat foo.log
- cat bar.log
- test -d quux.dir
-}
-
-miniclean ()
-{
- rmdir quux.dir
- rm -f foo.log bar.log test-suite.log
-}
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE -a
-
-./configure
-
-FOO=1; export FOO
-
-TESTS_ENVIRONMENT='BAR=1' $MAKE check || { cat test-suite.log; Exit 1; }
-minicheck
-miniclean
-TESTS_ENVIRONMENT='BAR=2' $MAKE check && { cat test-suite.log; Exit 1; }
-minicheck
-miniclean
-
-BAR=$FOO; export BAR
-
-$MAKE check 2>stderr || { cat stderr >&2; Exit1; }
-cat stderr >&2
-minicheck
-grep 'diagnostic message from test bar' stderr
-grep 'diagnostic message from test bar' bar.log && Exit 1
-miniclean
-
-TESTS_ENVIRONMENT='fd=2' $MAKE check 2>stderr || { cat stderr >&2; Exit1; }
-cat stderr >&2
-minicheck
-grep 'diagnostic message from test bar' stderr && Exit 1
-grep 'diagnostic message from test bar' bar.log
-miniclean
-
-$MAKE distcheck
-
-:
diff --git a/tests/parallel-tests-cmdline-override.test
b/tests/parallel-tests-cmdline-override.test
new file mode 100755
index 0000000..fa47032
--- /dev/null
+++ b/tests/parallel-tests-cmdline-override.test
@@ -0,0 +1,84 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that we can use indirections when overriding TESTS and
+# TEST_LOGS from the command line.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .test .t
+TEST_LOG_COMPILER = cat
+T_LOG_COMPILER = cat
+TESTS = bad.test
+var1 = b.test $(var2)
+var2 = c.test
+var3 = d.d
+var4 = e
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+rm -f config.log # Do not create false positives below.
+
+cat > exp-log <<'END'
+a.log
+b.log
+c.log
+d.log
+e.log
+test-suite.log
+END
+
+cat > exp-out <<'END'
+PASS: a.t
+PASS: b.test
+PASS: c.test
+PASS: d.t
+PASS: e.test
+END
+
+do_check ()
+{
+ env "$@" $MAKE -e check >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ grep '^PASS:' stdout | LC_ALL=C sort > got-out
+ cat got-out
+ ls . | grep '\.log$' | LC_ALL=C sort > got-log
+ cat got-log
+ st=0
+ diff exp-out got-out || st=1
+ diff exp-log got-log || st=1
+ return $st
+}
+
+tests='a.t $(var1) $(var3:.d=.t) $(var4:=.test)'
+test_logs='a.log $(var1:.test=.log) $(var3:.d=.log) $(var4:=.log)'
+
+touch a.t b.test c.test d.t e.test
+
+do_check TESTS="$tests"
+do_check TEST_LOGS="$test_logs"
+
+:
diff --git a/tests/parallel-tests-console-output.test
b/tests/parallel-tests-console-output.test
new file mode 100755
index 0000000..8122bfd
--- /dev/null
+++ b/tests/parallel-tests-console-output.test
@@ -0,0 +1,104 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests: some checks on console output about testsuite
+# progress.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+XFAIL_TESTS = sub/xpass.test xfail.test error.test
+TESTS = $(XFAIL_TESTS) fail.test pass.test a/b/skip.test sub/error2.test
+pass.log: fail.log
+error.log: pass.log
+sub/xpass.log: error.log
+sub/error2.log: xfail.log
+a/b/skip.log: sub/error2.log
+END
+
+cat > exp <<'END'
+FAIL: fail.test
+PASS: pass.test
+ERROR: error.test
+XPASS: sub/xpass.test
+XFAIL: xfail.test
+ERROR: sub/error2.test
+SKIP: a/b/skip.test
+END
+
+mkdir sub a a/b
+
+cat > pass.test << 'END'
+#!/bin/sh
+exit 0
+END
+cp pass.test sub/xpass.test
+
+cat > fail.test << 'END'
+#!/bin/sh
+exit 1
+END
+
+cat > xfail.test << 'END'
+#!/bin/sh
+# The sleep should ensure expected execution order of tests
+# even when make is run in parallel mode.
+# FIXME: quotes below required by maintainer-check.
+sleep '10'
+exit 1
+END
+
+cat > error.test << 'END'
+#!/bin/sh
+exit 99
+END
+cp error.test sub/error2.test
+
+cat > a/b/skip.test << 'END'
+#!/bin/sh
+exit 77
+END
+
+chmod a+x pass.test fail.test xfail.test sub/xpass.test \
+ a/b/skip.test error.test sub/error2.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+ $srcdir/configure
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ LC_ALL=C grep '^[A-Z][A-Z]*:' stdout > got
+ cat got
+ diff $srcdir/exp got
+ cd $srcdir
+done
+
+:
diff --git a/tests/parallel-test-driver-install.test
b/tests/parallel-tests-driver-install.test
similarity index 100%
rename from tests/parallel-test-driver-install.test
rename to tests/parallel-tests-driver-install.test
diff --git a/tests/parallel-tests-dry-run.test
b/tests/parallel-tests-dry-run.test
new file mode 100755
index 0000000..888d888
--- /dev/null
+++ b/tests/parallel-tests-dry-run.test
@@ -0,0 +1,112 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check `make -n' for testsuite-related targets, when `parallel-tests'
+# is in use.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = foo.test bar.test
+TEST_LOG_COMPILER = $(SHELL)
+END
+
+$ACLOCAL
+$AUTOMAKE -a
+$AUTOCONF
+
+./configure
+
+make_n_ ()
+{
+ st=0
+ $MAKE -n "$@" >output 2>&1 || { cat output; ls -l; Exit 1; }
+ cat output
+ # Look out for possible errors from common tools used by recipes.
+ $EGREP -i ' (exist|permission|denied|no .*(such|file))' output && Exit 1
+ $EGREP '(mv|cp|rm|cat|grep|sed|awk): ' output && Exit 1
+ :
+}
+
+: > output
+
+files='foo.log bar.log foo.trs bar.trs'
+
+for target in check recheck test-suite.log; do
+ test ! -f foo.log
+ test ! -f foo.trs
+ test ! -f bar.log
+ test ! -f bar.trs
+ test ! -f test-suite.log
+done
+
+# FIXME: creative quoting below to please maintainer-check.
+echo exit '0' > foo.test
+echo exit '1' > bar.test
+
+$MAKE check && Exit 1
+
+chmod a-w .
+
+make_n_ clean
+test -f foo.log
+test -f foo.trs
+test -f foo.log
+test -f bar.trs
+
+# FIXME: creative quoting below to please maintainer-check.
+cat > foo.test <<END
+echo this is bad
+exit 1
+END
+echo exit '0' > bar.test
+
+for target in check recheck test-suite.log; do
+ make_n_ $target
+ grep '^:test-result: *FAIL' bar.trs
+ grep 'this is bad' foo.log test-suite.log && Exit 1
+done
+
+chmod a-rw $files
+if test -r foo.log; then
+ : You can still read unreadable files! Skip these checks.
+else
+ for target in check recheck test-suite.log; do
+ make_n_ $target
+ for f in $files; do
+ test -f $f && test ! -r $f || Exit 1
+ done
+ done
+fi
+chmod u+r $files
+
+chmod u+w .
+rm -f foo.log bar.trs
+chmod a-w .
+for target in check recheck test-suite.log $files; do
+ make_n_ $target
+ test ! -f foo.log
+ test -f foo.trs
+ test ! -f bar.trs
+ test -f bar.log
+done
+
+:
diff --git a/tests/parallel-tests-empty-testlogs.test
b/tests/parallel-tests-empty-testlogs.test
index 593dce3..e3d77ea 100755
--- a/tests/parallel-tests-empty-testlogs.test
+++ b/tests/parallel-tests-empty-testlogs.test
@@ -55,7 +55,7 @@ $AUTOMAKE -a
no_test_has_run ()
{
ls -1 *.log | grep -v '^test-suite\.log$' | grep . && Exit 1
- grep ' 0 tests passed' test-suite.log
+ grep '^# TOTAL: *0$' test-suite.log
:
}
diff --git a/tests/parallel-tests-exit-statuses.test
b/tests/parallel-tests-exit-statuses.test
new file mode 100755
index 0000000..531526a
--- /dev/null
+++ b/tests/parallel-tests-exit-statuses.test
@@ -0,0 +1,108 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: normal and special exit statuses
+# in the test scripts.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+# $failure_statuses should be defined to the list of all integers between
+# 1 and 255 (inclusive), excluded 77 and 99.
+# Let's use `seq' if available, it's faster than the loop.
+failure_statuses=`seq 1 255 \
+ || { i=1; while test $i -le 255; do echo $i; i=\`expr $i + 1\`; done; }`
+failure_statuses=`
+ for i in $failure_statuses; do
+ test $i -eq 77 || test $i -eq 99 || echo $i
+ done | tr "$nl" ' '`
+# For debugging.
+echo "failure_statuses: $failure_statuses"
+# Sanity check.
+test `for st in $failure_statuses; do echo $st; done | wc -l` -eq 253 \
+ || fatal_ "initializing list of exit statuses for simple failures"
+
+cat > Makefile.am <<END
+LOG_COMPILER = ./do-exit
+fail_tests = $failure_statuses
+TESTS = 0 77 99 $failure_statuses
+\$(TESTS):
+END
+
+cat > do-exit <<'END'
+#!/bin/sh
+echo "$0: $1"
+case $1 in
+ [0-9]|[0-9][0-9]|[0-9][0-9][0-9]) st=$1;;
+ */[0-9]|*/[0-9][0-9]|*/[0-9][0-9][0-9]) st=`echo x"$1" | sed 's|.*/||'`;;
+ *) st=99;;
+esac
+exit $st
+END
+chmod a+x do-exit
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+{
+ echo PASS: 0
+ echo SKIP: 77
+ echo ERROR: 99
+ for st in $failure_statuses; do
+ echo "FAIL: $st"
+ done
+} | LC_ALL=C sort > exp-fail
+
+sed 's/^FAIL:/XFAIL:/' exp-fail | LC_ALL=C sort > exp-xfail-1
+sed '/^ERROR:/d' exp-xfail-1 > exp-xfail-2
+
+sort exp-fail
+sort exp-xfail-1
+sort exp-xfail-2
+
+./configure
+
+st=1
+$MAKE check >stdout && st=0
+cat stdout
+cat test-suite.log
+test $st -gt 0 || Exit 1
+LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-fail
+diff exp-fail got-fail
+
+st=1
+XFAIL_TESTS="$failure_statuses 99" $MAKE -e check >stdout && st=0
+cat stdout
+cat test-suite.log
+test $st -gt 0 || Exit 1
+LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-xfail-1
+diff exp-xfail-1 got-xfail-1
+
+st=0
+XFAIL_TESTS="$failure_statuses" TESTS="0 77 $failure_statuses" \
+ $MAKE -e check >stdout || st=$?
+cat stdout
+cat test-suite.log
+test $st -eq 0 || Exit 1
+LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-xfail-2
+diff exp-xfail-2 got-xfail-2
+
+:
diff --git a/tests/parallel-tests-extra-programs.test
b/tests/parallel-tests-extra-programs.test
new file mode 100755
index 0000000..f309b4c
--- /dev/null
+++ b/tests/parallel-tests-extra-programs.test
@@ -0,0 +1,174 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Parallel test harness: check that $(TESTS) can lazily depend on
+# (or even be) $(EXTRA_PROGRAMS).
+
+required='cc native'
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AC_OUTPUT
+END
+
+# Will be extended later.
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .bin .test
+EXTRA_PROGRAMS =
+TESTS =
+END
+
+#
+# Now try various kinds of test dependencies ...
+#
+
+# 1. A program that is also a test, and whose source files
+# already exist.
+
+cat >> Makefile.am <<'END'
+EXTRA_PROGRAMS += foo.bin
+TESTS += foo.bin
+foo_bin_SOURCES = foo.c
+END
+
+cat > foo.c <<'END'
+#include <stdio.h>
+int main (void)
+{
+ printf ("foofoofoo\n");
+ return 0;
+}
+END
+
+# 2. A program that is also a test, and whose source files
+# are buildable by make.
+cat >> Makefile.am <<'END'
+EXTRA_PROGRAMS += bar.bin
+TESTS += bar.bin
+bar_bin_SOURCES = bar.c
+bar.c: foo.c
+ sed -e 's/foofoofoo/barbarbar/' foo.c > $@
+END
+
+# 3. A test script that already exists, whose execution depends
+# on a program whose source files already exist and which is
+# not itself a test.
+cat >> Makefile.am <<'END'
+EXTRA_PROGRAMS += y
+TESTS += baz.test
+baz.log: y$(EXEEXT)
+END
+
+cat > baz.test <<'END'
+#!/bin/sh
+$srcdir/y "$@" | sed 's/.*/&ep&ep&ep/'
+END
+chmod a+x baz.test
+
+cat > y.c <<'END'
+#include <stdio.h>
+int main (void)
+{
+ printf ("y\n");
+ return 0;
+}
+END
+
+# 4. A program that is also a test, but whose source files
+# do not exit and are not buildable by make.
+
+cat >> Makefile.am <<'END'
+EXTRA_PROGRAMS += none.bin
+TESTS += none.bin
+none_bin_SOURCES = none.c
+END
+
+#
+# Setup done, go with the tests.
+#
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+# What we check now:
+# 1. even if we cannot build the `none.bin' program, all the other
+# test programs should be built, and all the other tests should
+# be run;
+# 2. still, since we cannot create the `none.log' file, the
+# `test-suite.log' file shouldn't be created (as it depends
+# on *all* the test logs).
+
+st=0
+$MAKE check >stdout 2>stderr || st=$?
+cat stdout
+cat stderr >&2
+ls -l
+test $st -gt 0 || Exit 1
+
+# Files that should have been created, with the expected content.
+cat bar.c
+grep foofoofoo foo.log
+grep barbarbar bar.log
+grep yepyepyep baz.log
+# Files that shouldn't have been created.
+test ! -f none.log
+test ! -f test-suite.log
+# Expected testsuite progress output.
+grep '^PASS: baz\.test$' stdout
+# Don't anchor the end of the next two patterns, to allow for non-empty
+# $(EXEEXT).
+grep '^PASS: foo\.bin' stdout
+grep '^PASS: bar\.bin' stdout
+# Expected error messages from make.
+$EGREP 'none\.(bin|o|c)' stderr
+
+# What we check now:
+# 1. if we make the last EXTRA_PROGRAM buildable, the failed tests
+# pass;
+# 2. on a lazy re-run, the passed tests are not re-run, and
+# 3. their log files are not updated or touched.
+
+: > stamp
+$sleep
+
+echo 'int main (void) { return 0; }' > none.c
+
+st=0
+RECHECK_LOGS= $MAKE -e check >stdout || st=$?
+cat stdout
+ls -l
+test $st -eq 0 || Exit 1
+
+# For debugging.
+stat stamp foo.log bar.log baz.log || :
+
+# Files that shouldn't have been updated or otherwise touched.
+is_newest stamp foo.log bar.log baz.log
+# Files that should have been created now.
+test -f none.log
+test -f test-suite.log
+# Tests that shouldn't have been re-run.
+$EGREP '(foo|bar)\.bin|baz\.test$' stdout && Exit 1
+# Tests that should have been run. Again, we don't anchor the end
+# of the next pattern, to allow for non-empty $(EXEEXT).
+grep '^PASS: none\.bin' stdout
+
+:
diff --git a/tests/parallel-tests-fd-redirect.test
b/tests/parallel-tests-fd-redirect.test
new file mode 100755
index 0000000..e00581c
--- /dev/null
+++ b/tests/parallel-tests-fd-redirect.test
@@ -0,0 +1,109 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests support: redirection of file descriptors with
+# AM_TESTS_FD_REDIRECT, even when using tests without suffix, or
+# which are binary executables.
+# We use some tricks to ensure that all code paths in `lib/am/check2.am'
+# are covered, even on platforms where $(EXEEXT) would be naturally empty.
+# See also the more generic test 'check-fd-redirect.test'.
+
+required='cc native'
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AM_CONDITIONAL([real_EXEEXT], [test -n "$EXEEXT"])
+test -n "$EXEEXT" || EXEEXT=.bin
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+AM_TESTS_FD_REDIRECT = 9>&1
+TEST_EXTENSIONS = .test .sh
+TESTS = foo.sh bar $(check_PROGRAMS)
+check_PROGRAMS = baz qux.test
+qux_test_SOURCES = zardoz.c
+
+## Sanity check.
+if !real_EXEEXT
+check-local:
+ test -f baz.bin
+ test -f qux.test.bin
+endif
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+cat > foo.sh <<'END'
+#!/bin/sh
+echo " foofoofoo" >&9
+END
+chmod a+x foo.sh
+
+cat > bar <<'END'
+#!/bin/sh
+echo " barbarbar" >&9
+END
+
+chmod a+x foo.sh bar
+
+cat > baz.c <<'END'
+#include <unistd.h>
+int main (void)
+{
+ write (9, " bazbazbaz\n", 11);
+ return 0;
+}
+END
+
+cat > zardoz.c <<'END'
+#include <unistd.h>
+int main (void)
+{
+ write (9, " quxquxqux\n", 11);
+ return 0;
+}
+END
+
+./configure
+
+# Sanity checks.
+st=0
+grep '^bar\.log:.*bar' Makefile || st=1
+grep '^baz\.log:.*baz\$(EXEEXT)' Makefile || st=1
+grep '^\.test\$(EXEEXT)\.log:' Makefile || st=1
+$EGREP '^(foo|qux)\.log:' Makefile && st=1
+test $st -eq 0 || fatal_ "doesn't cover expected code paths"
+
+st=0
+$MAKE check >stdout || st=1
+cat stdout
+cat foo.log
+cat bar.log
+cat baz.log
+cat qux.log
+test $st -eq 0
+grep "^ foofoofoo$" stdout
+grep "^ barbarbar$" stdout
+grep "^ bazbazbaz$" stdout
+grep "^ quxquxqux$" stdout
+$EGREP '(foofoofoo|barbarbar|bazbazbaz|quxquxqux)' *.log && Exit 1
+
+:
diff --git a/tests/parallel-tests-harderror.test
b/tests/parallel-tests-harderror.test
index 8680e29..f30fd52 100755
--- a/tests/parallel-tests-harderror.test
+++ b/tests/parallel-tests-harderror.test
@@ -69,13 +69,13 @@ DISABLE_HARD_ERRORS=x $MAKE check
$MAKE check DISABLE_HARD_ERRORS='' && Exit 1
cat test-suite.log
-grep '^FAIL: foo\.test .*exit.*99' test-suite.log
+grep '^ERROR: foo\.test .*exit.*99' test-suite.log
cd sub
# The `-e' is wanted here.
DISABLE_HARD_ERRORS='' $MAKE -e check && Exit 1
cat test-suite.log
-grep '^FAIL: bar\.test .*exit.*99' test-suite.log
+grep '^ERROR: bar\.test .*exit.*99' test-suite.log
cd ..
# Check the distributions.
@@ -90,7 +90,7 @@ $AUTOMAKE Makefile
./config.status Makefile
VERBOSE=yes $MAKE check && Exit 1
grep '^FAIL' test-suite.log && Exit 1
-grep '^FAIL: bar\.test .*exit.*99' sub/test-suite.log
+grep '^ERROR: bar\.test .*exit.*99' sub/test-suite.log
echo 'DISABLE_HARD_ERRORS = zardoz' >> sub/Makefile
VERBOSE=yes $MAKE check
diff --git a/tests/parallel-tests-log-compiler-example.test
b/tests/parallel-tests-log-compiler-example.test
new file mode 100755
index 0000000..746c128
--- /dev/null
+++ b/tests/parallel-tests-log-compiler-example.test
@@ -0,0 +1,71 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test the example of usage of generic and extension-specific
+# LOG_COMPILER and LOG_FLAGS given in the manual.
+
+parallel_tests=yes
+required=python
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AM_PATH_PYTHON
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.pl bar.py baz
+TEST_EXTENSIONS = .pl .py
+PL_LOG_COMPILER = $(PERL)
+AM_PL_LOG_FLAGS = -w
+PY_LOG_COMPILER = $(PYTHON)
+AM_PY_LOG_FLAGS = -v
+LOG_COMPILER = ./wrapper-script
+AM_LOG_FLAGS = -d
+END
+
+echo 'my $a =+ 2; exit (0);' > foo.pl
+echo 'import sys; sys.exit(0);' > bar.py
+: > baz
+
+cat > wrapper-script <<'END'
+#!/bin/sh
+echo "wrapper args: $*"
+END
+chmod a+x wrapper-script
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+st=0
+$MAKE check || st=$?
+cat foo.log
+cat bar.log
+cat baz.log
+test $st -eq 0 || Exit $st
+
+# Check that the wrappers have been run with the expected flags.
+grep '[rR]eversed.*+=.*operator.*foo\.pl' foo.log
+grep '^# *[cC]lear.*sys\.argv' bar.log
+grep '^wrapper args:.* -d .*baz' baz.log
+
+:
diff --git a/tests/parallel-tests-log-override-2.test
b/tests/parallel-tests-log-override-2.test
index 649360c..dfd6f2f 100755
--- a/tests/parallel-tests-log-override-2.test
+++ b/tests/parallel-tests-log-override-2.test
@@ -67,6 +67,7 @@ do
$MAKE -e check >stdout || { cat stdout; Exit 1; }
cat stdout
ls -l
+ count_test_results total=2 pass=1 fail=0 skip=1 xfail=0 xpass=0 error=0
cat pass.log
cat skip.log
cat partial.log
@@ -74,14 +75,11 @@ do
test ! -f pass2.log
test ! -f skip2.log
test ! -f fail.log
- grep '^PASS: .*pass\.test' stdout
- grep '^SKIP: .*skip\.test' stdout
+ grep '^PASS: pass\.test$' stdout
+ grep '^SKIP: skip\.test$' stdout
$FGREP 'skip.test' partial.log
$FGREP '% skipped test %' partial.log
- for t in pass2 skip2 fail; do
- $FGREP "$t.test" stdout && Exit 1
- $FGREP "$t.test" partial.log && Exit 1
- done
+ $EGREP '(pass2|skip2|fail)\.test' stdout partial.log && Exit 1
rm -f *.log
done
diff --git a/tests/parallel-tests-log-override-recheck.test
b/tests/parallel-tests-log-override-recheck.test
index 8a50d29..d8cab8a 100755
--- a/tests/parallel-tests-log-override-recheck.test
+++ b/tests/parallel-tests-log-override-recheck.test
@@ -65,7 +65,7 @@ TEST_SUITE_LOG=my.log $MAKE -e recheck >stdout \
&& { cat stdout; Exit 1; }
cat stdout
ls -l
-grep '^2 of 2 .*failed' stdout
+count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
for x in stdout my.log; do
$FGREP foo.test $x && Exit 1
$FGREP bar.test $x
@@ -77,7 +77,7 @@ BAZ_EXIT_STATUS=0 TEST_SUITE_LOG=my2.log $MAKE -e recheck
>stdout \
&& { cat stdout; Exit 1; }
cat stdout
ls -l
-grep '^1 of 2 .*failed' stdout
+count_test_results total=2 pass=1 fail=0 skip=0 xfail=0 xpass=0 error=1
$FGREP foo.test stdout && Exit 1
$FGREP bar.test stdout
$FGREP baz.test stdout
diff --git a/tests/parallel-tests-no-color-in-log.test
b/tests/parallel-tests-no-color-in-log.test
new file mode 100755
index 0000000..5d33ac5
--- /dev/null
+++ b/tests/parallel-tests-no-color-in-log.test
@@ -0,0 +1,63 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Colorized output from the testsuite report shouldn't end up in log files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+esc=''
+
+# Check that grep can parse nonprinting characters.
+# BSD 'grep' works from a pipe, but not a seekable file.
+# GNU or BSD 'grep -a' works on files, but is not portable.
+case `echo "$esc" | $FGREP "$esc"` in
+ "$esc") ;;
+# FIXME: creative quoting below to please maintainer-check.
+ *) echo "$me: f""grep can't parse nonprinting characters" >&2; Exit 77;;
+esac
+
+TERM=ansi; export TERM
+
+cat >>configure.in <<END
+AC_OUTPUT
+END
+
+cat >Makefile.am <<'END'
+LOG_COMPILER = $(SHELL)
+AUTOMAKE_OPTIONS = color-tests parallel-tests
+TESTS = pass fail skip xpass xfail error
+XFAIL_TESTS = xpass xfail
+END
+
+# FIXME: creative quoting to please maintainer-check.
+echo exit '0' > pass
+echo exit '0' > xpass
+echo exit '1' > fail
+echo exit '1' > xfail
+echo exit '77' > skip
+echo exit '99' > error
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+./configure
+mv config.log config-log # Avoid possible false positives below.
+AM_COLOR_TESTS=always $MAKE -e check && Exit 1
+$FGREP "$esc" *.log && Exit 1
+
+:
diff --git a/tests/parallel-tests-no-spurious-summary.test
b/tests/parallel-tests-no-spurious-summary.test
new file mode 100755
index 0000000..9ef7715
--- /dev/null
+++ b/tests/parallel-tests-no-spurious-summary.test
@@ -0,0 +1,68 @@
+#! /bin/sh
+# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that `:test-results:' directives in test scripts' output doesn't
+# originate spurious results in the testsuite summary.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+echo :test-result:XFAIL
+echo :test-result: SKIP
+echo :test-result:ERROR
+exit 0
+END
+cat > bar.test <<'END'
+#! /bin/sh
+echo :test-result: ERROR
+echo :test-result:FAIL
+echo :test-result: XPASS
+exit 0
+END
+chmod a+x foo.test bar.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+st=0
+$MAKE check >stdout || st=$?
+cat stdout
+cat test-suite.log
+cat foo.log
+cat bar.log
+test $st -eq 0 || Exit $st
+
+grep '^:test-result:XFAIL$' foo.log
+grep '^:test-result: SKIP$' foo.log
+grep '^:test-result:FAIL$' bar.log
+grep '^:test-result: XPASS$' bar.log
+
+count_test_results total=2 pass=2 fail=0 skip=0 xfail=0 xpass=0 error=0
+
+:
diff --git a/tests/parallel-tests-once.test b/tests/parallel-tests-once.test
new file mode 100755
index 0000000..d676a12
--- /dev/null
+++ b/tests/parallel-tests-once.test
@@ -0,0 +1,48 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Tests shouldn't be run multiple times by a simple "make check" in a
+# clean directory. An early implementation of the `.trs' intermediate
+# files incurred a similar problem.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+test -f foo.run && Exit 1
+: > foo.run
+END
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check
+test -f foo.run # Sanity check.
+
+:
diff --git a/tests/parallel-tests-unreadable-log.test
b/tests/parallel-tests-unreadable-log.test
deleted file mode 100755
index 572f03d..0000000
--- a/tests/parallel-tests-unreadable-log.test
+++ /dev/null
@@ -1,60 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Check that the testsuite driver copes well with unreadable test logs.
-
-parallel_tests=yes
-. ./defs || Exit 1
-
-cat >> configure.in << 'END'
-AC_OUTPUT
-END
-
-cat > Makefile.am << 'END'
-TESTS = foo.test bar.test
-XFAIL_TESTS = bar.test
-END
-
-cat > foo.test << 'END'
-#! /bin/sh
-exit 0
-END
-cat > bar.test << 'END'
-#! /bin/sh
-exit 1
-END
-chmod a+x foo.test bar.test
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE -a
-
-# The testsuite driver will use this variable, so ensure it sanitizes
-# it and do not allow in spurious values from the environment.
-line=PASS; export line
-
-./configure
-
-$MAKE foo.log
-$MAKE bar.log
-chmod a-r foo.log bar.log
-test ! -r foo.log || Exit 77
-$MAKE test-suite.log >stdout && { cat stdout; Exit 1; }
-cat stdout
-grep '^2 of 2 tests failed *$' stdout
-grep '^2 of 2 tests failed\. *$' test-suite.log
-
-:
diff --git a/tests/parallel-tests-unreadable.test
b/tests/parallel-tests-unreadable.test
new file mode 100755
index 0000000..47edc9a
--- /dev/null
+++ b/tests/parallel-tests-unreadable.test
@@ -0,0 +1,75 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that the testsuite driver copes well with unreadable `.log'
+# and `.trs' files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+: > t
+chmod a-r t && test ! -r t || skip_ "you can still read unreadable files"
+rm -f t
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test
+END
+
+cat > foo.test << 'END'
+#! /bin/sh
+echo foofoofoo
+exit 0
+END
+
+cat > bar.test << 'END'
+#! /bin/sh
+echo barbarbar
+exit 77
+END
+
+chmod a+x foo.test bar.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+for files in \
+ 'foo.log bar.log' \
+ 'foo.trs bar.trs' \
+ 'foo.trs bar.log' \
+ 'foo.log bar.trs' \
+; do
+ $MAKE check
+ rm -f test-suite.log
+ chmod a-r $files
+ $MAKE test-suite.log || { ls -l; Exit 1; }
+ ls -l
+ grep '^foofoofoo$' foo.log
+ grep '^:test-result: PASS' foo.trs
+ grep '^barbarbar$' bar.log
+ grep '^:test-result: SKIP' bar.trs
+ grep '^SKIP: bar.test' test-suite.log
+ grep '^barbarbar$' test-suite.log
+ $EGREP 'foo\.test|foofoofoo' test-suite.log && Exit 1
+done
+
+:
diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test
index f046c3f..2d9e2a2 100755
--- a/tests/parallel-tests.test
+++ b/tests/parallel-tests.test
@@ -14,14 +14,14 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-# Check parallel-tests features:
-# - VERBOSE
-# - clean
-# - TEST_SUITE_LOG
-# - dependencies between tests
-# - TESTS
-# - TEST_LOGS
-# - RECHECK_LOGS
+# Basic checks on parallel-tests support:
+# - console output
+# - log files, and what goes in 'test-suite.log'
+# - make clean
+# - dependencies between tests
+# - TESTS redefinition at runtime
+# - TEST_LOGS redefinition at runtime
+# - RECHECK_LOGS redefinition at runtime
parallel_tests=yes
. ./defs || Exit 1
@@ -31,7 +31,6 @@ AC_OUTPUT
END
cat > Makefile.am << 'END'
-TEST_SUITE_LOG = mylog.log
TESTS = foo.test bar.test baz.test
XFAIL_TESTS = bar.test
foo.log: bar.log
@@ -68,35 +67,36 @@ $AUTOMAKE -a
$MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
-# There should be two errors: bar.test is a hard error.
-test `grep -c '^FAIL' stdout` -eq 2
-test -f mylog.log
-test `grep -c '^FAIL' mylog.log` -eq 2
+count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
+test -f test-suite.log
+cat test-suite.log
+test `grep -c '^FAIL:' test-suite.log` -eq 1
+test `grep -c '^ERROR:' test-suite.log` -eq 1
+$EGREP '^(X?PASS|XFAIL|SKIP)' test-suite.log && Exit 1
test -f baz.log
test -f bar.log
test -f foo.log
-# The summary should be formatted correctly.
-grep 'failedn' stdout && Exit 1
-# clean should remove all log files (but not more).
-: > unrelated.log
$MAKE clean
test ! -f baz.log
test ! -f bar.log
test ! -f foo.log
-test ! -f mylog.log
-test -f unrelated.log
+test ! -f test-suite.log
# Check dependencies: baz.test needs to run before bar.test,
# but foo.test is not needed.
# Note that this usage has a problem: the summary will only
# take bar.log into account, because the $(TEST_SUITE_LOG) rule
# does not "see" baz.log. Hmm.
-env TESTS='bar.test' $MAKE -e check && Exit 1
+env TESTS='bar.test' $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^FAIL: baz\.test$' stdout
+grep '^ERROR: bar\.test$' stdout
+
test -f baz.log
test -f bar.log
test ! -f foo.log
-test -f mylog.log
+test -f test-suite.log
# Upon a lazy rerun, foo.test should be run, but the others shouldn't.
# Note that the lazy rerun still exits with a failure, due to the previous
@@ -107,10 +107,12 @@ test -f mylog.log
env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
cat stdout
test -f foo.log
-grep foo.test stdout
+grep '^PASS: foo\.test$' stdout
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
# Now, explicitly retry with all test logs already updated, and ensure
# that the summary is still displayed.
@@ -119,7 +121,9 @@ cat stdout
grep foo.test stdout && Exit 1
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
# Lazily rerunning only foo should only rerun this one test.
env RECHECK_LOGS=foo.log $MAKE -e check > stdout && { cat stdout; Exit 1; }
@@ -127,13 +131,9 @@ cat stdout
grep foo.test stdout
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
-
-# Test VERBOSE.
-env VERBOSE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; }
-cat stdout
-grep 'this is.*bar.test' stdout
-grep 'this is.*baz.test' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
$MAKE clean
env TEST_LOGS=baz.log $MAKE -e check > stdout && { cat stdout; Exit 1; }
diff --git a/tests/parallel-tests3.test b/tests/parallel-tests3.test
index 9f0ce42..8153eeb 100755
--- a/tests/parallel-tests3.test
+++ b/tests/parallel-tests3.test
@@ -70,7 +70,7 @@ $MAKE -j4 check >> stdout
cd ..
kill $!
cat parallel/stdout
-test `grep -c PASS parallel/stdout` -eq 8
+test `grep -c '^PASS:' parallel/stdout` -eq 8
# Wait long enough so that there are no open files any more
# when the post-test cleanup runs.
diff --git a/tests/parallel-tests6.test b/tests/parallel-tests6.test
index 5992250..e0c535e 100755
--- a/tests/parallel-tests6.test
+++ b/tests/parallel-tests6.test
@@ -36,6 +36,8 @@ $AUTOMAKE -a
./configure
$MAKE check >stdout || { cat stdout; Exit 1; }
cat stdout
-grep '0 tests' stdout
+for x in TOTAL PASS FAIL XPASS FAIL SKIP ERROR; do
+ grep "^# $x: *0$" stdout
+done
Exit 0
diff --git a/tests/parallel-tests9.test b/tests/parallel-tests9.test
index e6191d9..f86562a 100755
--- a/tests/parallel-tests9.test
+++ b/tests/parallel-tests9.test
@@ -58,21 +58,14 @@ $AUTOMAKE -a
./configure
$MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
+count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
$MAKE recheck >stdout && { cat stdout; Exit 1; }
cat stdout
-grep foo.test stdout && Exit 1
-grep bar.test stdout || Exit 1
-grep baz.test stdout || Exit 1
-grep '2 of 2.*failed' stdout
-
-# If we cannot read the log file, then redo it as well.
-chmod a-r foo.log
-if test ! -r foo.log; then
- $MAKE recheck >stdout && { cat stdout; Exit 1; }
- cat stdout
- grep foo.test stdout || Exit 1
-fi
+count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
+grep 'foo\.test' stdout && Exit 1
+grep '^ERROR: bar\.test$' stdout
+grep '^FAIL: baz\.test$' stdout
# Ensure that recheck builds check_SCRIPTS, and that
# recheck reruns nothing if check has not been run.
diff --git a/tests/plain-functions.sh b/tests/plain-functions.sh
new file mode 100644
index 0000000..c142f05
--- /dev/null
+++ b/tests/plain-functions.sh
@@ -0,0 +1,39 @@
+# -*- shell-script -*-
+#
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Helper functions used by "plain" tests of the Automake testsuite
+# (i.e., tests that don't use any test protocol).
+
+# Print warnings (e.g., about skipped and failed tests) to this file
+# number. Override by putting, say:
+# stderr_fileno_=9; export stderr_fileno_; exec 9>&2;
+# in the definition of AM_TESTS_ENVIRONMENT.
+# This is useful when using automake's parallel tests mode, to print the
+# reason for skip/failure to console, rather than to the *.log files.
+: ${stderr_fileno_=2}
+
+# Copied from Gnulib's `tests/init.sh'.
+warn_ () { echo "$@" 1>&$stderr_fileno_; }
+fail_ () { warn_ "$me: failed test: $@"; Exit 1; }
+skip_ () { warn_ "$me: skipped test: $@"; Exit 77; }
+fatal_ () { warn_ "$me: hard error: $@"; Exit 99; }
+framework_failure_ () { warn_ "$me: set-up failure: $@"; Exit 99; }
+
+# For compatibility with TAP functions.
+skip_all_ () { skip "$@"; }
+
+:
diff --git a/tests/remake-subdir-long-time.test
b/tests/remake-subdir-long-time.test
new file mode 100755
index 0000000..18ba732
--- /dev/null
+++ b/tests/remake-subdir-long-time.test
@@ -0,0 +1,116 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that remake rules from subdirectories do not hang or cycle
+# endlessly, even with build systems that takes several seconds to
+# rebuild the Makefiles.
+# This test tries to ensure a long-enough rebuild time by introducing
+# an explicit delay in the build process.
+# Suggestion by Ralf Wildenhues.
+
+. ./defs || Exit 1
+
+set -e
+
+cat > configure.in <<END
+AC_INIT([$me], [1.0])
+AM_INIT_AUTOMAKE([foreign -Wall -Werror])
+AC_CONFIG_FILES([Makefile sub/Makefile])
+AC_SUBST([MAGIC], [magic])
+AC_OUTPUT
+END
+
+echo SUBDIRS = sub > Makefile.am
+mkdir sub
+: > sub/Makefile.am
+
+# Both aclocal and automake are expected to run one and just one time.
+# Create and use wrappers that will verify that.
+
+ocwd=`pwd` || fatal_ "cannot get current working directory"
+
+mkdir bin
+
+cat > bin/automake <<END
+#!/bin/sh
+set -e
+PATH='$PATH'; export PATH
+sentinel='$ocwd/automake-has-run'
+if test -f "\$sentinel"; then
+ echo "Automake has been run more than one time" >&2
+ exit 1
+else
+ echo automake has run > "\$sentinel"
+fi
+$sleep; $sleep;
+exec $AUTOMAKE \${1+"\$@"}
+END
+chmod a+x bin/automake
+
+cat > bin/aclocal <<END
+#!/bin/sh
+set -e
+PATH='$PATH'; export PATH
+sentinel='$ocwd/aclocal-has-run'
+if test -f "\$sentinel"; then
+ echo "Aclocal has been run more than one time" >&2
+ exit 1
+else
+ echo aclocal has run > "\$sentinel"
+fi
+$sleep; $sleep;
+exec $ACLOCAL \${1+"\$@"}
+END
+chmod a+x bin/aclocal
+
+# Just to be sure.
+cp bin/automake bin/automake-$APIVERSION
+cp bin/aclocal bin/aclocal-$APIVERSION
+
+PATH=$ocwd/bin$PATH_SEPARATOR$PATH; export PATH
+
+AUTOMAKE=automake ACLOCAL=aclocal; export AUTOMAKE ACLOCAL
+
+$ACLOCAL # Should use or just-defined wrapper.
+$AUTOMAKE # Likewise.
+$AUTOCONF
+
+# Sanity check: the wrappers have been used.
+test -f automake-has-run
+test -f aclocal-has-run
+rm -f automake-has-run aclocal-has-run
+
+./configure
+# Sanity check: Makefile doesn't get updated uselessly.
+ACLOCAL=false AUTOMAKE=false AUTOCONF=false $MAKE -e
+
+$sleep
+sed "s|magic|magic2|" configure.in > t
+mv -f t configure.in
+
+cd sub
+AUTOMAKE="$AUTOMAKE" ACLOCAL="$ACLOCAL" $MAKE -e Makefile
+cd ..
+
+# For debugging.
+ls -l . sub
+grep -i magic configure Makefile.in Makefile sub/Makefile.in sub/Makefile
+# Sanity checks.
+$FGREP magic2 configure
+$FGREP magic2 Makefile
+$FGREP magic2 sub/Makefile
+
+:
diff --git a/tests/self-check-env-sanitize.test
b/tests/self-check-env-sanitize.test
index 169f003..1f7dba2 100755
--- a/tests/self-check-env-sanitize.test
+++ b/tests/self-check-env-sanitize.test
@@ -23,10 +23,20 @@
set -x
exec 5>&1
-for var in me parallel_tests required original_AUTOMAKE original_ACLOCAL; do
+vars='
+ me
+ parallel_tests
+ required
+ use_tap
+ original_AUTOMAKE
+ original_ACLOCAL
+'
+
+for var in $vars; do
env "$var=foo" $SHELL -c '. ./defs' foo.test && exit 1
env "$var=foo" $SHELL -c '. ./defs' foo.test 2>&1 1>&5 \
- | grep "foo\.test:.* variable \`$var'.* in the environment.*unsafe" ||
exit 1
+ | grep "foo\.test:.* variable \`$var'.* in the environment.*unsafe" \
+ || exit 1
done
:
diff --git a/tests/self-check-is_newest.test b/tests/self-check-is_newest.test
index f179229..832d352 100755
--- a/tests/self-check-is_newest.test
+++ b/tests/self-check-is_newest.test
@@ -39,4 +39,37 @@ stat c d || : # for debugging
is_newest c d
+# Should work on directories too, both empty and not-empty. An older
+# implementation of `is_newest' failed if the first argument was a
+# directory containing files newer than itself (see automake bug#9147).
+mkdir u x
+touch x/foo
+$sleep
+touch x/foo
+$sleep
+mkdir v y
+touch y/foo
+$sleep
+touch y/foo
+
+stat u v x y x/foo y/foo || : # For debugging.
+
+for older in u x; do
+ for newer in v y; do
+ is_newest $newer $older
+ is_newest $older $newer && Exit 1
+ done
+done
+
+is_newest x/foo x
+is_newest x x/foo && Exit 1
+
+touch -r x u
+is_newest x u
+is_newest u x
+
+# A couple of mild "stress" tests.
+is_newest y x u v
+is_newest y u x/foo a b c
+
:
diff --git a/tests/self-check-tap.test b/tests/self-check-tap.test
new file mode 100755
index 0000000..43a8bac
--- /dev/null
+++ b/tests/self-check-tap.test
@@ -0,0 +1,34 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Sanity check for the automake testsuite.
+# Make sure that $using_tap gets automatically defined by `./defs-static',
+# but can be overridden by the individual tests.
+
+. ./defs-static || exit 1
+
+set -ex
+
+$SHELL -c '. ./defs-static && test $using_tap = yes' foo.tap
+$SHELL -c '. ./defs-static && test $using_tap = no' foo.test
+$SHELL -c '. ./defs-static && test $using_tap = no' tap
+$SHELL -c '. ./defs-static && test $using_tap = no' tap.test
+$SHELL -c '. ./defs-static && test $using_tap = no' foo-tap
+
+$SHELL -c 'using_tap=no; . ./defs-static; test $using_tap = no' foo.tap
+$SHELL -c 'using_tap=yes; . ./defs-static; test $using_tap = yes' foo.test
+
+:
diff --git a/tests/specflg8.test b/tests/specflg8.test
index 1defd0b..d2c6714 100755
--- a/tests/specflg8.test
+++ b/tests/specflg8.test
@@ -32,25 +32,18 @@ END
# different flags.
cat > Makefile.am << 'END'
-FALSESOURCE = false.c
+TRUESOURCE = true.c
bin_PROGRAMS = false true
-true_SOURCES = $(FALSESOURCE)
-true_CPPFLAGS = -DAM_TRUE
-false_SOURCES = $(FALSESOURCE)
-false_CPPFLAGS = -DAM_FALSE
+true_SOURCES = $(TRUESOURCE)
+true_CPPFLAGS = -DEXIT_CODE=0
+false_SOURCES = $(TRUESOURCE)
+false_CPPFLAGS = -DEXIT_CODE=1
END
-cat > false.c << 'END'
-#include <stdio.h>
-int
-main (int argc, char *argv[])
+cat > true.c << 'END'
+int main (void)
{
-#ifdef AM_TRUE
- puts ("true");
-#else
- puts ("false");
-#endif
- return 0;
+ return EXIT_CODE;
}
END
@@ -60,5 +53,11 @@ $AUTOMAKE -a
./configure
$MAKE
-./true | grep true
-./false | grep false
+
+./true
+./false && Exit 1
+
+test -f ./true-true.o
+test -f ./true-true.o
+
+:
diff --git a/tests/tap-autonumber.test b/tests/tap-autonumber.test
new file mode 100755
index 0000000..12dec11
--- /dev/null
+++ b/tests/tap-autonumber.test
@@ -0,0 +1,75 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - unnumbered tests are OK, as long as their final total number
+# agrees with the plan
+# - test results without number get automatically numbered in the
+# console progress output
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..14
+ok
+ok foo
+ok - foo2
+not ok
+not ok bar
+not ok - bar2
+; See that we can intermingle different kind of results without
+; messing up the autonumbering
+ok
+ok # TODO
+not ok # TODO who cares?
+ok
+not ok
+ok muuu # SKIP
+not ok
+ok
+END
+
+TESTS=all.test $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=14 pass=6 fail=5 xpass=1 xfail=1 skip=1 error=0
+
+cat > exp <<'END'
+PASS: all.test 1
+PASS: all.test 2 foo
+PASS: all.test 3 - foo2
+FAIL: all.test 4
+FAIL: all.test 5 bar
+FAIL: all.test 6 - bar2
+PASS: all.test 7
+XPASS: all.test 8 # TODO
+XFAIL: all.test 9 # TODO who cares?
+PASS: all.test 10
+FAIL: all.test 11
+SKIP: all.test 12 muuu # SKIP
+FAIL: all.test 13
+PASS: all.test 14
+END
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-bad-prog.tap b/tests/tap-bad-prog.tap
new file mode 100755
index 0000000..2dee975
--- /dev/null
+++ b/tests/tap-bad-prog.tap
@@ -0,0 +1,86 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - missing, unreadable, or not-executable test scripts cause proper
+# error reports
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TESTS = none.test noread.test noexec.test
+none.test:
+END
+
+cat > noexec.test <<'END'
+#!/bin/sh
+echo 1..1
+ok 1
+END
+
+cp noexec.test noread.test
+chmod a-r noread.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+if $MAKE check >stdout; then r='not ok'; else r='ok'; fi
+cat stdout
+result_ "$r" '"make check" returns non-zero exit status'
+
+# FIXME: maybe grep for stricter error messages in the next checks?
+
+command_ok_ "non-existent test is reported" \
+ grep '^ERROR: none\.test' stdout
+
+desc="non-executable test is reported"
+if ./noexec.test; then
+ skip_ -r "any file is executable" "$desc"
+else
+ command_ok_ "$desc" -- grep '^ERROR: noexec\.test' stdout
+fi
+
+desc="non-readable test is reported"
+if test -r noread.test; then
+ skip_ -r "any file is readable" "$desc"
+else
+ command_ok_ "$desc" -- grep '^ERROR: noread\.test' stdout
+fi
+
+# Check that no spurious test results is reported. This is lower-priority
+# (and in fact the check currently fails.
+if count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3
+then
+ r='ok'
+else
+ r='not ok'
+fi
+result_ "$r" 'no spurious results # TODO still get "missing plan"'
+
+:
diff --git a/tests/tap-bailout.test b/tests/tap-bailout.test
new file mode 100755
index 0000000..8354b27
--- /dev/null
+++ b/tests/tap-bailout.test
@@ -0,0 +1,138 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP test protocol support:
+# - "Bail out!" magic
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+: > exp
+
+#------------------------------------------------------------------
+
+# Bailout without explanation.
+
+cat > a.test <<END
+1..4
+ok 1
+not ok 2
+Bail out!
+not ok 3
+ok 4 # SKIP
+END
+
+cat >> exp <<END
+PASS: a.test 1
+FAIL: a.test 2
+ERROR: a.test - Bail out!
+END
+
+# pass += 1, fail +=1, error += 1
+
+#------------------------------------------------------------------
+
+# Bailout with explanation.
+
+cat > b.test <<END
+1..7
+ok 1 # SKIP
+ok 2 # TODO
+not ok 3 # TODO
+Bail out! We're out of disk space.
+ok 4
+not ok 5
+not ok 6 # TODO
+ok 7 # TODO
+END
+
+cat >> exp <<END
+SKIP: b.test 1 # SKIP
+XPASS: b.test 2 # TODO
+XFAIL: b.test 3 # TODO
+ERROR: b.test - Bail out! We're out of disk space.
+END
+
+# skip += 1, xpass += 1, xfail += 1, error += 1
+
+#------------------------------------------------------------------
+
+# Bail out before the test plan.
+
+cat > c.test <<END
+ok 1
+ok 2
+Bail out! BOOOH!
+1..2
+END
+
+cat >> exp <<END
+PASS: c.test 1
+PASS: c.test 2
+ERROR: c.test - Bail out! BOOOH!
+END
+
+# pass += 2, error += 1
+
+#------------------------------------------------------------------
+
+# Bailout on the first line.
+
+cat > d.test <<END
+Bail out! mktemp -d: Permission denied
+ok 1
+END
+
+echo 'ERROR: d.test - Bail out! mktemp -d: Permission denied' >> exp
+
+# error += 1
+
+#------------------------------------------------------------------
+
+# TAP input comprised only of a bailout directive.
+
+cat > e.test <<END
+Bail out!
+END
+
+echo "ERROR: e.test - Bail out!" >> exp
+
+# error += 1
+
+#------------------------------------------------------------------
+
+# Doing the sums above, we have:
+test_counts='total=12 pass=3 fail=1 xpass=1 xfail=1 skip=1 error=5'
+
+TESTS='a.test b.test c.test d.test e.test' $MAKE -e check >stdout \
+ && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results $test_counts
+
+LC_ALL=C sort exp > t
+mv -f t exp
+
+# We need the sort below to account for parallel make usage.
+grep ': [abcde]\.test' stdout | LC_ALL=C sort > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-basic.test b/tests/tap-basic.test
new file mode 100755
index 0000000..26a749d
--- /dev/null
+++ b/tests/tap-basic.test
@@ -0,0 +1,177 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP support:
+# - LOG_COMPILER support;
+# - basic support for TODO and SKIP directives, and "Bail out!" magic;
+# - testsuite progress output on console;
+# - runtime overriding of TESTS and TEST_LOGS;
+# - correct counts of test results (both in summary and in progress
+# output on console).
+# Note that some of the features checked here are checked in other
+# test cases too, usually in a more thorough and detailed way.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+## Defining LOG_COMPILER should work and not intefere with the
+## tap-driver script.
+TEST_LOG_COMPILER = cat
+TESTS = success.test
+
+ok.test:
+ echo '1..3' > address@hidden
+ echo 'ok 1' >> address@hidden
+ echo 'not ok 2 # TODO' >>address@hidden
+ echo 'ok 3 # SKIP' >>address@hidden
+ cat address@hidden ;: For debugging.
+ mv -f address@hidden $@
+END
+
+cat > success.test << 'END'
+1..20
+ok 1
+ok 2 two
+ok 3 - three
+ok 4 four four
+not ok 5
+not ok 6 six
+not ok 7 - seven
+not ok 8 eight eight
+ok 9 # TODO
+ok 10 ten # TODO
+ok 11 - eleven # TODO
+ok 12 twelve twelve # TODO
+not ok 13 # TODO
+not ok 14 fourteen # TODO
+not ok 15 - fifteen # TODO
+not ok 16 sixteen sixteen # TODO
+ok 17 # SKIP
+ok 18 eighteen # SKIP
+ok 19 - nineteen # SKIP
+ok 20 twenty twenty # SKIP
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# Basilar usage and testsuite progress output.
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=20 pass=4 fail=4 xpass=4 xfail=4 skip=4 error=0
+
+test -f success.log
+test -f test-suite.log
+
+cat > exp << 'END'
+PASS: success.test 1
+PASS: success.test 2 two
+PASS: success.test 3 - three
+PASS: success.test 4 four four
+FAIL: success.test 5
+FAIL: success.test 6 six
+FAIL: success.test 7 - seven
+FAIL: success.test 8 eight eight
+XPASS: success.test 9 # TODO
+XPASS: success.test 10 ten # TODO
+XPASS: success.test 11 - eleven # TODO
+XPASS: success.test 12 twelve twelve # TODO
+XFAIL: success.test 13 # TODO
+XFAIL: success.test 14 fourteen # TODO
+XFAIL: success.test 15 - fifteen # TODO
+XFAIL: success.test 16 sixteen sixteen # TODO
+SKIP: success.test 17 # SKIP
+SKIP: success.test 18 eighteen # SKIP
+SKIP: success.test 19 - nineteen # SKIP
+SKIP: success.test 20 twenty twenty # SKIP
+END
+
+$FGREP ': success.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+# Override TESTS from the command line.
+
+rm -f *.log *.test
+
+cat > bail.test <<'END'
+1..1
+Bail out!
+ok 1
+END
+
+TESTS=bail.test $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
+
+test ! -f success.log
+test -f bail.log
+test -f test-suite.log
+
+grep '^ERROR: bail\.test - Bail out!' stdout
+grep '^PASS:' stdout && Exit 1
+test `$FGREP -c ': bail.test' stdout` -eq 1
+$FGREP 'success.test' stdout && Exit 1
+
+# Override TEST_LOGS from the command line, making it point to a test
+# (ok.test) that have to be generated at make time.
+
+rm -f *.log *.test
+
+TEST_LOGS=ok.log $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=1 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+test -f ok.test
+test -f ok.log
+test ! -f success.log
+test ! -f bail.log
+test -f test-suite.log
+
+$EGREP '(bail|success)\.test' stdout && Exit 1
+
+cat > exp << 'END'
+PASS: ok.test 1
+XFAIL: ok.test 2 # TODO
+SKIP: ok.test 3 # SKIP
+END
+
+$FGREP ': ok.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-color.test b/tests/tap-color.test
new file mode 100755
index 0000000..dbd6629
--- /dev/null
+++ b/tests/tap-color.test
@@ -0,0 +1,165 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - colorization of TAP results and diagnostic messages
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+TERM=ansi; export TERM
+
+esc=''
+# Escape `[' for grep, below.
+red="$esc\[0;31m"
+grn="$esc\[0;32m"
+lgn="$esc\[1;32m"
+blu="$esc\[1;34m"
+mgn="$esc\[0;35m"
+std="$esc\[m"
+
+# Check that grep can parse nonprinting characters.
+# BSD 'grep' works from a pipe, but not a seekable file.
+# GNU or BSD 'grep -a' works on files, but is not portable.
+case `echo "$std" | grep .` in
+ "$std") ;;
+ *) echo "$me: grep can't parse nonprinting characters" >&2; Exit 77;;
+esac
+
+cat > Makefile.am << 'END'
+AUTOMAKE_OPTIONS = color-tests
+AM_TEST_LOG_DRIVER_FLAGS = --comments
+TEST_LOG_COMPILER = cat
+TESTS = all.test skip.test bail.test badplan.test noplan.test \
+ few.test many.test order.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test << 'END'
+1..5
+ok 1 - foo
+# Hi! I shouldn't be colorized!
+not ok 2 - bar # TODO td
+ok 3 - baz # SKIP sk
+not ok 4 - quux
+ok 5 - zardoz # TODO
+END
+
+cat > skip.test << 'END'
+1..0 # SKIP whole script
+END
+
+cat > bail.test << 'END'
+1..1
+ok 1
+Bail out!
+END
+
+cat > badplan.test << 'END'
+# foo
+1..1
+ok 1
+END
+
+cat > noplan.test << 'END'
+ok 1
+END
+
+cat > few.test << 'END'
+1..2
+ok 1
+END
+
+cat > many.test << 'END'
+1..1
+ok 1
+ok 2
+END
+
+cat > order.test << 'END'
+1..1
+ok 5
+END
+
+AM_COLOR_TESTS=always $MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+test_color ()
+{
+ # Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "^${grn}PASS${std}: all\.test 1 - foo$"
+ cat stdout | grep "^${lgn}XFAIL${std}: all\.test 2 - bar # TODO td$"
+ cat stdout | grep "^${blu}SKIP${std}: all\.test 3 - baz # SKIP sk$"
+ cat stdout | grep "^${red}FAIL${std}: all\.test 4 - quux$"
+ cat stdout | grep "^${red}XPASS${std}: all\.test 5 - zardoz # TODO$"
+ cat stdout | grep "^${blu}SKIP${std}: skip\.test - whole script$"
+ cat stdout | grep "^${grn}PASS${std}: bail\.test 1$"
+ cat stdout | grep "^${mgn}ERROR${std}: bail\.test - Bail out!$"
+ cat stdout | grep "^${mgn}ERROR${std}: badplan\.test - test plan in middle
of output$"
+ cat stdout | grep "^${mgn}ERROR${std}: noplan\.test - missing test plan$"
+ cat stdout | grep "^${mgn}ERROR${std}: few.test - too few tests run
(expected 2, got 1)$"
+ cat stdout | grep "^${mgn}ERROR${std}: many.test - too many tests run
(expected 1, got 2)$"
+ cat stdout | grep "^${mgn}ERROR${std}: many.test 2 # UNPLANNED$"
+ cat stdout | grep "^${mgn}ERROR${std}: order.test 5 # OUT-OF-ORDER
(expecting 1)$"
+ # Diagnostic messages shouldn't be colorized.
+ cat stdout | grep "^# all\.test: Hi! I shouldn't be colorized!$"
+ :
+}
+
+test_no_color ()
+{
+ # With make implementations that, like Solaris make, in case of errors
+ # print the whole failing recipe on standard output, we should content
+ # ourselves with a laxer check, to avoid false positives.
+ # Keep this in sync with lib/am/check.am:$(am__color_tests).
+ if $FGREP '= Xalways || test -t 1 ' stdout; then
+ # Extra verbose make, resort to laxer checks.
+ # But we also want to check that the testsuite summary is not unduly
+ # colorized.
+ (
+ set +e # In case some grepped regex below isn't matched.
+ # Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "TOTAL.*:"
+ cat stdout | grep "PASS.*:"
+ cat stdout | grep "FAIL.*:"
+ cat stdout | grep "SKIP.*:"
+ cat stdout | grep "XFAIL.*:"
+ cat stdout | grep "XPASS.*:"
+ cat stdout | grep "ERROR.*:"
+ cat stdout | grep "^#"
+ cat stdout | grep 'test.*expected'
+ cat stdout | grep 'test.*not run'
+ cat stdout | grep '===='
+ cat stdout | grep '[Ss]ee .*test-suite\.log'
+ cat stdout | grep '[Tt]estsuite summary'
+ ) | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ else
+ cat stdout | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ fi
+}
+
+AM_COLOR_TESTS=always $MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+test_color
+
+$MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+test_no_color
+
+:
diff --git a/tests/tap-common-setup.test b/tests/tap-common-setup.test
new file mode 100755
index 0000000..de8ff2c
--- /dev/null
+++ b/tests/tap-common-setup.test
@@ -0,0 +1,47 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Auxiliary test to set up common data used by many tests on TAP support.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in << END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TEST_LOG_COMPILER = cat
+TESTS = all.test
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+rm -rf autom4te*.cache
+
+# So that the data files we've created won't be removed at exit.
+keep_testdirs=yes
+
+:
diff --git a/tests/tap-deps.test b/tests/tap-deps.test
new file mode 100755
index 0000000..b86d235
--- /dev/null
+++ b/tests/tap-deps.test
@@ -0,0 +1,91 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP test protocol support:
+# - dependencies between test scripts
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+# The tests are *deliberately* listed in inversed order here.
+TESTS = c.test b.test a.test
+b.log: a.log
+c.log: b.log
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > a.test << 'END'
+#!/bin/sh
+echo 1..2
+echo ok 1
+sleep '3' # FIXME: quotes to please maintainer-check
+echo ok 2
+: > a.run
+END
+
+cat > b.test << 'END'
+#!/bin/sh
+echo 1..2
+if test -f a.run; then
+ echo ok 1
+else
+ echo not ok 1
+fi
+sleep '3' # FIXME: quotes to please maintainer-check
+echo ok 2
+: > b.run
+END
+
+cat > c.test << 'END'
+#!/bin/sh
+echo 1..1
+test -f b.run || { echo 'Bail out!'; exit 1; }
+echo ok 1
+rm -f a.run b.run
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=5 pass=5 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+cat > exp << 'END'
+PASS: a.test 1
+PASS: a.test 2
+PASS: b.test 1
+PASS: b.test 2
+PASS: c.test 1
+END
+
+grep ': [abc]\.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+# TODO: it would be nice to also redo the checks forcing parallel make...
+
+:
diff --git a/tests/tap-diagnostic.test b/tests/tap-diagnostic.test
new file mode 100755
index 0000000..73ab45e
--- /dev/null
+++ b/tests/tap-diagnostic.test
@@ -0,0 +1,124 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - diagnostic messages (TAP lines with leading "#")
+# - flags '--comments' and '--no-comments' of the TAP test driver
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+metacharacters=\''"\$!&()[]<>#;^?*'
+
+cat > all.test <<END
+1..4
+# Hi! I'm a comment.
+# Tests begin.
+ok 1
+not ok 2 - foo # TODO
+ok 3 - bar # SKIP
+# Tests end.
+ok - zardoz
+# Shell metacharacters here: $metacharacters
+.# Leading characters before "#", not a TAP diagnostic line.
+x # Leading characters before "#", not a TAP diagnostic line.
+ # Leading whitespace before "#", not a TAP diagnostic line.
+${tab}# Leading whitespace before "#", not a TAP diagnostic line.
+ ${tab} # Leading whitespace before "#", not a TAP diagnostic line.
+END
+
+cat > exp <<END
+# all.test: Hi! I'm a comment.
+# all.test: Tests begin.
+PASS: all.test 1
+XFAIL: all.test 2 - foo # TODO
+SKIP: all.test 3 - bar # SKIP
+# all.test: Tests end.
+PASS: all.test 4 - zardoz
+# all.test: Shell metacharacters here: $metacharacters
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+$EGREP -i "#.*all\\.test|a comment|(Tests|Shell) " stdout && Exit 1
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+echo 'AM_TEST_LOG_DRIVER_FLAGS = --comments' >> Makefile
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+$FGREP ' all.test' stdout > got
+cat exp
+cat got
+diff exp got
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+TEST_LOG_DRIVER_FLAGS="--no-comments" $MAKE -e check >stdout \
+ || { cat stdout; Exit 1; }
+cat stdout
+$EGREP -i "#.*all\\.test|a comment|(Tests|Shell) " stdout && Exit 1
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+# The "#"-prepended lines here shouldn't be parsed as test results.
+cat > all.test <<END
+1..1
+ok
+# ok
+#ok
+# not ok
+#not ok
+# Bail out!
+#Bail out!
+# SKIP
+#SKIP
+# TODO
+#TODO
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+# Diagnostic without leading whitespace, or with extra leading whitespace,
+# is ok. Be laxer in the grepping checks, to allow for whitespace
+# normalization by the TAP driver.
+
+ws="[ $tab]"
+ws0p="${ws}*"
+ws1p="${ws}${ws0p}"
+
+cat > all.test <<END
+1..1
+ok 1
+#foo
+#bar${tab}
+# zardoz ${tab}
+# ${tab} ${tab}${tab}foo bar${tab}baz ${tab}
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+grep "^# all.test:${ws0p}foo$" stdout
+grep "^# all.test:${ws0p}bar${ws0p}$" stdout
+grep "^# all.test:${ws1p}zardoz${ws0p}$" stdout
+grep "^# all.test:${ws1p}foo${ws1p}bar${ws1p}baz${ws0p}$" stdout
+
+test `grep -c '^# all\.test:' stdout` -eq 4
+
+:
diff --git a/tests/tap-doc.test b/tests/tap-doc.test
new file mode 100755
index 0000000..2920d4e
--- /dev/null
+++ b/tests/tap-doc.test
@@ -0,0 +1,92 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that an example given in the documentation really works.
+# See section "Simple Tests" subsection "Script-based Testsuites".
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_PROG_CC
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.sh zardoz.tap bar.sh mu.tap
+TEST_EXTENSIONS = .sh .tap
+TAP_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+END
+
+cat > foo.sh <<'END'
+#!/bin/sh
+exit 0
+END
+
+cat > bar.sh <<'END'
+#!/bin/sh
+exit 77
+END
+
+cat > zardoz.tap << 'END'
+#!/bin/sh
+echo 1..4
+echo 'ok 1 - Daemon started'
+echo 'ok 2 - Daemon responding'
+echo 'ok 3 - Daemon uses /proc # SKIP /proc is not mounted'
+echo 'ok 4 - Daemon stopped'
+END
+
+cat > mu.tap << 'END'
+#!/bin/sh
+echo 1..2
+echo 'ok'
+echo 'not ok # TODO frobnication not yet implemented'
+END
+
+chmod a+x *.sh *.tap
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+cat > exp <<'END'
+PASS: foo.sh
+PASS: zardoz.tap 1 - Daemon started
+PASS: zardoz.tap 2 - Daemon responding
+SKIP: zardoz.tap 3 - Daemon uses /proc # SKIP /proc is not mounted
+PASS: zardoz.tap 4 - Daemon stopped
+SKIP: bar.sh
+PASS: mu.tap 1
+XFAIL: mu.tap 2 # TODO frobnication not yet implemented
+END
+
+sed -n '/^PASS: foo\.sh/,/^XFAIL: mu\.tap/p' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-doc2.test b/tests/tap-doc2.test
new file mode 100755
index 0000000..63bdc23
--- /dev/null
+++ b/tests/tap-doc2.test
@@ -0,0 +1,139 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that an example given in the documentation really works.
+# See section "Using the TAP test protocol", subsection "Use TAP
+# with the Automake test harness".
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+# Ensure we are run from the right directory.
+# (The last thing we want is to delete some random user files.)
+test -f ../defs-static && test -f ../defs || Exit 99
+rm -f *
+
+cat > Makefile.am <<'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/build-aux/tap-driver
+TESTS = foo.test bar.test baz.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > configure.ac <<'END'
+AC_INIT([GNU Try Tap], [1.0], address@hidden)
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE([foreign parallel-tests -Wall -Werror])
+AC_CONFIG_FILES([Makefile])
+AC_REQUIRE_AUX_FILE([tap-driver])
+AC_PATH_PROG([PERL], [perl])
+test -n "$PERL" || AC_MSG_ERROR([perl not found])
+$PERL -MTAP::Parser -e 1 || AC_MSG_ERROR([TAP::Parser not found])
+AC_OUTPUT
+END
+
+cat > foo.test <<'END'
+#!/bin/sh
+echo 1..4 # Number of tests to be executed.
+echo 'ok 1 - Swallows fly'
+echo 'not ok 2 - Caterpillars fly # TODO metamorphosis in progress'
+echo 'ok 3 - Pigs fly # SKIP not enough acid'
+echo '# I just love word plays...'
+echo 'ok 4 - Flies fly too :-)'
+END
+
+cat > bar.test <<'END'
+#!/bin/sh
+echo 1..3
+echo 'not ok 1 - Bummer, this test has failed.'
+echo 'ok 2 - This passed though.'
+echo 'Bail out! Ennui kicking in, sorry...'
+echo 'ok 3 - This will not be seen.'
+END
+
+cat > baz.test <<'END'
+#!/bin/sh
+echo 1..1
+echo ok 1
+# Exit with error, even if all the test case has been successful.
+exit 7
+END
+
+chmod a+x *.test
+
+mkdir build-aux
+cp "$top_testsrcdir"/lib/tap-driver build-aux
+
+(export AUTOMAKE ACLOCAL AUTOCONF && $AUTORECONF -vi) || Exit 1
+
+./configure --help # Sanity check.
+./configure || skip_ "configure failed"
+
+case $MAKE in *\ -j*) skip_ "can't work easily with concurrent make";; esac
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+cat > exp <<'END'
+PASS: foo.test 1 - Swallows fly
+XFAIL: foo.test 2 - Caterpillars fly # TODO metamorphosis in progress
+SKIP: foo.test 3 - Pigs fly # SKIP not enough acid
+PASS: foo.test 4 - Flies fly too :-)
+FAIL: bar.test 1 - Bummer, this test has failed.
+PASS: bar.test 2 - This passed though.
+ERROR: bar.test - Bail out! Ennui kicking in, sorry...
+PASS: baz.test 1
+ERROR: baz.test - exited with status 7
+END
+
+sed -n '/^PASS: foo\.test/,/^ERROR: baz\.test/p' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+grep '^Please report to address@hidden' stdout
+
+env \
+ TESTS='foo.test baz.test' \
+ TEST_LOG_DRIVER_FLAGS='--comments --ignore-exit' \
+ $MAKE -e check >stdout || { cat stdout; Exit 1; }
+
+cat > exp <<'END'
+PASS: foo.test 1 - Swallows fly
+XFAIL: foo.test 2 - Caterpillars fly # TODO metamorphosis in progress
+SKIP: foo.test 3 - Pigs fly # SKIP not enough acid
+# foo.test: I just love word plays...
+PASS: foo.test 4 - Flies fly too :-)
+PASS: baz.test 1
+END
+
+sed -n '/^PASS: foo\.test/,/^PASS: baz\.test/p' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+# Sanity check the distribution.
+cat > bar.test <<'END'
+#!/bin/sh
+echo 1..1
+echo ok 1
+END
+echo AM_TEST_LOG_DRIVER_FLAGS = --ignore-exit >> Makefile.in
+./config.status Makefile
+$MAKE distcheck
+
+:
diff --git a/tests/tap-empty-diagnostic.test b/tests/tap-empty-diagnostic.test
new file mode 100755
index 0000000..2dcc880
--- /dev/null
+++ b/tests/tap-empty-diagnostic.test
@@ -0,0 +1,47 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - empty diagnostic messages are discarder
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+sed 's/\$$//' > all.test <<END
+1..1$
+ok 1$
+#$
+# $
+#${tab}$
+# ${tab} $tab${tab}$
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+grep '^PASS:.*all\.test' stdout # Sanity check.
+grep '#.*all\.test' stdout && Exit 1
+grep "all\.test[ $tab]*:[ $tab]*$" stdout && Exit 1
+
+:
diff --git a/tests/tap-empty.test b/tests/tap-empty.test
new file mode 100755
index 0000000..557b7c7
--- /dev/null
+++ b/tests/tap-empty.test
@@ -0,0 +1,55 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: the following situations should be flagged as errors:
+# - empty TAP input
+# - blank TAP input
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Empty TAP input.
+: > empty.test
+
+# Blank TAP input (one empty line).
+echo > blank.test
+
+# Blank TAP input (one whitespace-only line).
+cat > white.test <<END
+ ${tab}
+END
+
+# Blank TAP input (few blank and whitespace-only lines).
+cat > white2.test <<END
+
+
+ ${tab} ${tab}${tab}
+
+${tab}
+
+END
+
+for input in empty blank white white2; do
+ cp $input.test all.test
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
+ grep '^ERROR: all\.test - missing test plan$' stdout
+done
+
+:
diff --git a/tests/tap-escape-directive.test b/tests/tap-escape-directive.test
new file mode 100755
index 0000000..2f8cb73
--- /dev/null
+++ b/tests/tap-escape-directive.test
@@ -0,0 +1,45 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - "escape" TODO and SKIP directives (by escaping the "#" character)
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..2
+ok \# TODO
+ok \# SKIP
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+grep '^PASS: all\.test 1 .*TODO' stdout
+grep '^PASS: all\.test 2 .*SKIP' stdout
+
+:
diff --git a/tests/tap-exit.test b/tests/tap-exit.test
new file mode 100755
index 0000000..381adff
--- /dev/null
+++ b/tests/tap-exit.test
@@ -0,0 +1,54 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - an exit status != 0 of a test script causes an hard error
+# - the `--ignore-exit' option causes the TAP test driver to ignore
+# exit statuses of the test scripts.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+echo TESTS = > Makefile.am
+for st in 1 2 77 99; do
+ unindent > exit${st}.test <<END
+ #!/bin/sh
+ echo 1..1
+ echo ok 1
+ exit $st
+END
+ echo TESTS += exit${st}.test >> Makefile.am
+done
+
+chmod a+x *.test
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=8 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=4
+
+grep '^ERROR: exit1\.test - exited with status 1$' stdout
+grep '^ERROR: exit2\.test - exited with status 2$' stdout
+grep '^ERROR: exit77\.test - exited with status 77$' stdout
+grep '^ERROR: exit99\.test - exited with status 99$' stdout
+
+echo TEST_LOG_DRIVER_FLAGS = --ignore-exit >> Makefile
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=4 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+:
diff --git a/tests/tap-fancy.test b/tests/tap-fancy.test
new file mode 100755
index 0000000..3348db7
--- /dev/null
+++ b/tests/tap-fancy.test
@@ -0,0 +1,133 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: some unusual forms for valid TAP input.
+# See also related test 'tap-fancy2.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+#
+# From manpage Test::Harness::TAP(3):
+#
+# ``Lines written to standard output matching /^(not )?ok\b/ must be
+# interpreted as test lines. All other lines must not be considered
+# test output.''
+#
+# Unfortunately, the exact format of TODO and SKIP directives is not as
+# clearly described in that manpage; but a simple reverse-engineering of
+# the prove(1) utility shows that it is probably given by the perl regex
+# /#\s*(TODO|SKIP)\b/.
+#
+
+cat > all.test <<END
+1..21
+
+ok? a question
+not ok? a question
+
+ok+plus
+not ok+plus
+
+ok-minus
+not ok-minus
+
+ok#55
+not ok#55
+
+ok${tab} ${tab}9
+ok ${tab}${tab} 10
+
+not ok${tab} ${tab}11
+not ok ${tab}${tab} 12
+
+ok# SKIP
+ok${tab}#SKIP--who cares?
+ok?#SKIP!
+ok!#SKIP?
+
+not ok# TODO
+not ok${tab}#TODO--who cares?
+not ok?#TODO!
+not ok!#TODO?
+
+ok~#TODO
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=21 pass=6 fail=6 xfail=4 xpass=1 skip=4 error=0
+
+#
+# "Weird" characters support.
+#
+
+# The "#" character might cause confusion w.r.t. TAP directives (TODO,
+# SKIP), so we don't attempt to use it.
+weirdchars=\''"$!&()[]<>;^?*/@%=,.:'
+
+cat > all.test <<END
+1..6
+ok $weirdchars
+not ok $weirdchars
+ok $weirdchars # TODO
+not ok $weirdchars # TODO
+ok $weirdchars # SKIP
+Bail out! $weirdchars
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=6 pass=1 fail=1 xfail=1 xpass=1 skip=1 error=1
+$FGREP "PASS: all.test 1 $weirdchars" stdout
+$FGREP "FAIL: all.test 2 $weirdchars" stdout
+$FGREP "XPASS: all.test 3 $weirdchars" stdout
+$FGREP "XFAIL: all.test 4 $weirdchars" stdout
+$FGREP "SKIP: all.test 5 $weirdchars" stdout
+$FGREP "ERROR: all.test - Bail out! $weirdchars" stdout
+
+#
+# Trailing backslashes does not confuse the parser.
+#
+
+bs='\'
+
+cat > all.test <<END
+1..6
+ok $bs
+not ok $bs
+ok # TODO $bs
+not ok # TODO $bs
+ok # SKIP $bs
+Bail out! $bs
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=6 pass=1 fail=1 xfail=1 xpass=1 skip=1 error=1
+grep '^PASS: all\.test 1 \\$' stdout
+grep '^FAIL: all\.test 2 \\$' stdout
+grep '^XPASS: all\.test 3 # TODO \\$' stdout
+grep '^XFAIL: all\.test 4 # TODO \\$' stdout
+grep '^SKIP: all\.test 5 # SKIP \\$' stdout
+grep '^ERROR: all\.test - Bail out! \\$' stdout
+
+:
diff --git a/tests/tap-fancy2.test b/tests/tap-fancy2.test
new file mode 100755
index 0000000..ddd64ea
--- /dev/null
+++ b/tests/tap-fancy2.test
@@ -0,0 +1,144 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: more unusual forms for valid TAP input.
+# See also related test 'tap-fancy.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+#
+# From manpage Test::Harness::TAP(3):
+#
+# ``Lines written to standard output matching /^(not )?ok\b/ must be
+# interpreted as test lines. All other lines must not be considered
+# test output.''
+#
+# Unfortunately, the exact format of TODO and SKIP directives is not as
+# clearly described in that manpage; but a simple reverse-engineering of
+# the prove(1) utility shows that it is probably given by the perl regex
+# /#\s*(TODO|SKIP)\b/.
+#
+
+# To avoid problems with backslashes in echo arguments.
+xecho () { printf '%s\n' "$*"; }
+
+# There are 34 values for $str ...
+for str in \
+ \' \
+ '"' \
+ '`' \
+ '#' \
+ '$' \
+ '!' \
+ '\' \
+ '/' \
+ '&' \
+ '%' \
+ '(' \
+ ')' \
+ '|' \
+ '^' \
+ '~' \
+ '?' \
+ '*' \
+ '+' \
+ '-' \
+ ',' \
+ ':' \
+ ';' \
+ '=' \
+ '<' \
+ '>' \
+ '@' \
+ '[' \
+ ']' \
+ '{' \
+ '}' \
+ '\\' \
+ '...' \
+ '?[a-zA-Z0-9]*' \
+ '*.*' \
+; do
+ # ... each of them add 1 pass, 1 fail, ...
+ xecho "ok${str}"
+ xecho "not ok${str}"
+ # ... and (generally) 4 skips, 4 xfails, and 4 xpasses ...
+ for settings in \
+ 'result="ok" directive=SKIP' \
+ 'result="not ok" directive=TODO' \
+ 'result="ok" directive=TODO' \
+ ; do
+ eval "$settings"
+ xecho "${result}# ${directive}${str}"
+ # ... but 6 skips, 6 xpasses and 6 xfails are to be removed, since
+ # they might not work with $str = '#' or $str = '\' ...
+ if test x"$str" != x'#' && test x"$str" != x'\'; then
+ xecho "${result}${str}#${directive}"
+ xecho "${result}${str}# ${tab}${tab} ${directive}"
+ xecho "${result}${str}#${directive}${str}"
+ fi
+ done
+done > all.test
+
+# Sanity check against a previous use of unportable usages of backslashes
+# with the "echo" builtin.
+if grep '[^\\]\\#' all.test; then
+ framework_failure_ "writing backslashes in all.test"
+fi
+
+# ... so that we finally have:
+pass=34
+fail=34
+xfail=130 # = 4 * 34 - 6
+xpass=130 # = 4 * 34 - 6
+skip=130 # = 4 * 34 - 6
+error=0
+total=`expr $pass + $fail + $xfail + $xpass + $skip`
+
+# Even nastier! But accordingly to the specifics, it should still work.
+for result in 'ok' 'not ok'; do
+ echo "${result}{[(<#${tab}TODO>)]}" >> all.test
+done
+echo "ok{[(<#${tab}SKIP>)]}" >> all.test
+
+# We have to update some test counts.
+xfail=`expr $xfail + 1`
+xpass=`expr $xpass + 1`
+skip=`expr $skip + 1`
+total=`expr $total + 3`
+
+# And add the test plan!
+echo 1..$total >> all.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+$EGREP '^(PASS|FAIL|SKIP).*#.*TODO' stdout && Exit 1
+$EGREP '^X?(PASS|FAIL).*#.*SKIP' stdout && Exit 1
+
+count_test_results total=$total pass=$pass fail=$fail skip=$skip \
+ xpass=$xpass xfail=$xfail error=$error
+
+:
diff --git a/tests/tap-functions.sh b/tests/tap-functions.sh
new file mode 100644
index 0000000..0104d39
--- /dev/null
+++ b/tests/tap-functions.sh
@@ -0,0 +1,239 @@
+# -*- shell-script -*-
+#
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Helper functions used by TAP-producing tests of the Automake testsuite.
+
+#
+# IMPORTANT: All the functions defined in this file can *not* be used
+# from within a subshell, unless explicitly noted otherwise.
+#
+
+# The count of the TAP test results seen so far.
+tap_count_=0
+
+# The first "test -n" tries to avoid extra forks when possible.
+if test -n "${ZSH_VERSION}${BASH_VERSION}" \
+ || (eval 'test $((1 + 1)) = 2') >/dev/null 2>&1
+then
+ # Use of 'eval' needed to protect dumber shells from parsing errors.
+ eval 'incr_tap_count_ () { tap_count_=$(($tap_count_ + 1)); }'
+else
+ incr_tap_count_ () { tap_count_=`expr $tap_count_ + 1`; }
+fi
+
+# plan_ NUMBER-OF-PLANNED-TESTS
+# -----------------------------
+# Print a TAP plan for the given number of tests. This must be called
+# before reporting any test result; in fact, it must be called before
+# emitting anything on standard output.
+plan_ ()
+{
+ echo "1..$1"
+ have_tap_plan_=yes
+}
+
+# late_plan_
+# ----------
+# Print a TAP plan that accounts for the number of tests seen so far.
+# This must be called after all the tests result have been reported;
+# in fact, after this has been called, nothing more can be print on
+# standard output.
+late_plan_ ()
+{
+ echo "1..$tap_count_"
+ have_tap_plan_=yes
+}
+
+# Initialize it to avoid interferences from the environment.
+have_tap_plan_=no
+
+# diag_ [EXPLANATION]
+# ------------------
+# Report the given text as TAP diagnostic.
+diag_ ()
+{
+ test $# -eq 0 || echo "# $*"
+}
+
+# warn_ [EXPLANATION]
+# ------------------
+# Give a warning (using TAP diagnostic).
+warn_ ()
+{
+ diag_ "WARNING:" ${1-"(unknown warning)"} ${1+"$@"}
+}
+
+# result_ RESULT [DESCRIPTION...]
+# -------------------------------
+# Report a test case with the given result.
+result_ ()
+{
+ incr_tap_count_
+ tap_result_=$1; shift
+ echo "$tap_result_" $tap_count_ ${1+"- $*"}
+}
+
+# tap_with_directive_ RESULT DIRECTIVE [-r REASON] [DESCRIPTION...]
+# -----------------------------------------------------------------
+# Write a tap result with the given directive (can be "TODO" or "SKIP").
+# The REASON, if given is appended after the directive. This function is
+# for internal use only.
+result_with_directive_ ()
+{
+ incr_tap_count_
+ tap_result_=$1; shift
+ tap_directive_=$1; shift
+ case $1 in -r) tap_reason_=" $2" shift 2;; *) tap_reason_="";; esac
+ echo "$tap_result_" $tap_count_ ${1+"- $*"} \
+ "# ${tap_directive_}${tap_reason_}"
+}
+
+# ok_ [DESCRIPTION...]
+# --------------------
+# Report a successful test.
+ok_ ()
+{
+ result_ 'ok' ${1+"$@"}
+}
+
+# not_ok_ [DESCRIPTION...]
+# ------------------------
+# Report a failed test.
+not_ok_ ()
+{
+ result_ 'not ok' ${1+"$@"}
+}
+
+# skip_ [-r REASON] [DESCRIPTION...]
+# ----------------------------------
+# Report a skipped test. If the `-r' option is present, its argument is
+# give as the reason of the skip.
+skip_ ()
+{
+ result_with_directive_ 'ok' SKIP ${1+"$@"}
+}
+
+# skip_row_ COUNT [-r REASON] [DESCRIPTION...]
+# --------------------------------------------
+# Report a COUNT of skipped test, with the given reason and descriptions
+# (if any). Useful to avoid cascade failures in case a fair number of
+# tests depend on an earlier one that failed.
+skip_row_ ()
+{
+ skip_count_=$1; shift
+ for i_ in `seq $skip_count_`; do skip_ ${1+"$@"}; done
+}
+
+# xfail_ [-r REASON] [DESCRIPTION...]
+# ----------------------------------
+# Report a test that failed expectedly. If the `-r' option is present, its
+# argument is give as the reason why the failure is expected.
+xfail_ ()
+{
+ result_with_directive_ 'not ok' TODO ${1+"$@"}
+}
+
+# xpass_ [-r REASON] [DESCRIPTION...]
+# -----------------------------------
+# Report a test that failed unexpectedly. If the `-r' option is present, its
+# argument is give as the reason why the failure is expected.
+xpass_ ()
+{
+ result_with_directive_ 'ok' TODO ${1+"$@"}
+}
+
+# skip_all_ [REASON ...]
+# ----------------------
+# Skip all the tests in a test script. Must be used before calling `plan_'
+# or reporting any test result. Can't be used from within a subshell.
+skip_all_ ()
+{
+ echo "1..0 # SKIP" ${1+"$@"}
+ Exit 0
+}
+
+# bailout_ [REASON ...]
+# ---------------------
+# Stop the execution of the current test suite right now, due to an
+# unrecoverable error. Can be called at any point, but cannot be used
+# from within a subshell.
+bailout_ ()
+{
+ echo 'Bail out!' ${1+"$@"}
+ Exit 99
+}
+
+# fatal_ [REASON ...]
+# -------------------
+# Same as `bailout_'; for compatibility with `plain-functions.sh'.
+fatal_ ()
+{
+ bailout_ ${1+"$@"}
+}
+
+# framework_failure_ [REASON ...]
+# -------------------------------
+# Stop the execution of the current test suite right now, due to an
+# unrecoverable error in the set-up of the test case. Can be called
+# at any point, but cannot be used from within a subshell.
+framework_failure_ ()
+{
+ bailout_ "set-up failure"${1+": $*"}
+}
+
+# command_ok_ TEST-DESCRIPTION [--] CMD [ARGS...]
+# -----------------------------------------------
+# Report a passed test if the given command returns with success,
+# a failed test otherwise.
+command_ok_ ()
+{
+ tap_desc_=$1; shift
+ test x"$1" != x"--" || shift
+ if "$@"; then
+ ok_ "$tap_desc_"
+ else
+ not_ok_ "$tap_desc_"
+ fi
+}
+
+# command_not_ok_ TEST-DESCRIPTION [--] CMD [ARGS...]
+# ---------------------------------------------------
+# Report a failed test if the given command returns with success,
+# a failed test otherwise.
+command_not_ok_ ()
+{
+ tap_desc_=$1; shift
+ test x"$1" != x"--" || shift
+ if "$@"; then
+ not_ok_ "$tap_desc_"
+ else
+ ok_ "$tap_desc_"
+ fi
+}
+
+
+# reset_test_count_ COUNT
+# -----------------------
+# Reset the count of the TAP test results seen so far to COUNT.
+# This function is for use in corner cases only (e.g., when `ok_' and
+# `not_ok_' must be used inside a subshell). Be careful when using it!
+reset_test_count_ ()
+{
+ tap_count_=$1
+}
+
+:
diff --git a/tests/tap-global-log.test b/tests/tap-global-log.test
new file mode 100755
index 0000000..a4f6428
--- /dev/null
+++ b/tests/tap-global-log.test
@@ -0,0 +1,122 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - which log files get copied in the global log?
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > ok.test << 'END'
+1..5
+ok 1
+ok 2
+ok 3
+not seen in global log
+ok 4
+ok 5
+END
+
+cat > top << 'END'
+1..6
+ok 1
+ok 2
+ok 3
+END
+
+cat > bot << 'END'
+ok 5
+ok 6
+END
+
+cat top - bot > skip.test << 'END'
+ok # SKIP
+::skip::
+END
+
+cat top - bot > todo.test << 'END'
+not ok # TODO
+::todo::
+END
+
+cat top - bot > fail.test << 'END'
+not ok
+::fail::
+END
+
+cat top - bot > xpass.test << 'END'
+ok # TODO
+::xpass::
+END
+
+cat top - bot > bail.test << 'END'
+::bail::
+Bail out!
+END
+
+cat top - bot > error.test << 'END'
+::error::
+1..7
+END
+
+# Created with "dd if=/dev/urandom count=1 | base64-encode"
+cat > hodgepodge <<'END'
+1+0 records in
+1+0 records out
+512 bytes (512 B) copied, 0.000241092 s, 2.1 MB/s
+gdivnV4VhL4DOzhE3zULJuun3PwqqQqMdATVcZbIQkNgyRvNBoqqHMBQEs7QsjDbp2nK+Szz
+EcelGyvXmHrW7yImaom6Yrg95k31VCmp/pGDRnTDwDrcOPJiv9jDReEmTAQkPuqLO+mFNly+
+DDHM9fNbzGumstsQ3wq3DOXz1pCV3JXwhjeaHSboeEbmr55bX0XHLSKaecctA0rXDXEyZWZ/
+ODlawSrAXzw0H7a+xBwjnNXZ3zYiwk3x+WQrPqNjb+qXiLLTxAKzx2/KnaFhxkPlte5jPRNB
+FciDolL+H/10LsCdSzLOrGnY2zH6vL2JMZfxjnb73zWFcdWWE01LTD7wpN5O1MP3+N47lcVe
+lWbkD04cJvhwxLElYSO24B743GG5EyGYt9SeZRE6xbgwq3fVOS8KqjwGxwi4adSBTtw0CV8W
+S/6n8dck1vBvjA+qpk0zMSYSqc3+jzW9UiGTmTEIwfw80p/lGNsfjcNBJ86nFkWUnHmrsi8k
+Dv57sK70mTg239g08f5Uvdga/5UreMBSgB0hUj5sbq57r7B1fsVr0Kag468la8zKy3ZEZ0gX
+++sbaJ9WGHhnKvjooeH+4Y6HwAFsdINde++FlCmp4EuNKKEEuXbSKLaOTy3+6pJ2DYdvRCL+
+frZwxH4hcrw8qh+8IakB02viewZS/qT57v4=
+END
+
+exec 5>misc.test
+echo 'ok # SKIP' >&5
+cat hodgepodge >&5
+echo 'not ok # TODO' >&5
+echo 'ok' >&5
+exec 5>&-
+
+cat > skipall.test << 'END'
+1..0 # SKIP all
+END
+
+# We don't care about the exit status of "make check" here.
+TESTS="`echo *.test`" $MAKE -e check || :
+cat test-suite.log
+
+grep 'ok\.test|not seen' test-suite.log && Exit 1
+
+for s in skip todo fail xpass bail error; do
+ $FGREP "::$s::" test-suite.log
+done
+
+grep '^1\.\.0 # SKIP all$' test-suite.log
+
+case `cat test-suite.log` in
+ *"`cat hodgepodge`"*) ;;
+ *) Exit 1;;
+esac
+
+:
diff --git a/tests/tap-global-result.test b/tests/tap-global-result.test
new file mode 100755
index 0000000..c49fae4
--- /dev/null
+++ b/tests/tap-global-result.test
@@ -0,0 +1,104 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - which global test result derives from different test results
+# mixed in a single script?
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > ok.test <<END
+1..3
+ok 1
+not ok 2 # TODO
+ok 3 # SKIP
+END
+
+cat > skip.test <<'END'
+1..3
+ok 1 # SKIP
+ok 2 # SKIP
+ok 3 # SKIP
+END
+
+cat > skipall.test <<'END'
+1..0 # SKIP
+not ok 1
+END
+
+cat > fail.test <<'END'
+1..1
+not ok 1
+END
+
+(sed '1s/.*/1..4/' ok.test && echo 'not ok 4') > fail2.test
+
+cat > xpass.test <<'END'
+1..1
+ok 1 # TODO
+END
+
+(sed '1s/.*/1..4/' ok.test && echo 'ok 4 # TODO') > xpass2.test
+
+echo 'Bail out!' > error.test
+(cat ok.test && echo 'Bail out!') > error2.test
+
+cat > hodgepodge.test <<'END'
+1..2
+not ok 1
+ok 2 # TODO
+Bail out!
+END
+
+cat > hodgepodge-all.test <<'END'
+1..4
+ok 1
+ok 2 # SKIP
+not ok 2 # TODO
+not ok 3
+ok 4 # TODO
+Bail out!
+END
+
+# TODO: add scripts with TAP errors (multiple plans, out-of-order
+# tests, etc).
+
+TESTS="`echo *.test`" $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+for tst in ok skip skipall fail fail2 xpass xpass2 error error2 \
+ hodgepodge hodgepodge-all; do
+ sed -e 2q $tst.log > $tst.res
+done
+
+cat *.res # For debugging.
+
+grep '^PASS:' ok.res
+grep '^SKIP:' skip.res
+grep '^SKIP:' skipall.res
+grep '^FAIL:' fail.res
+grep '^FAIL:' fail2.res
+grep '^FAIL:' xpass.res
+grep '^FAIL:' xpass2.res
+grep '^ERROR:' error.res
+grep '^ERROR:' error2.res
+grep '^ERROR:' hodgepodge.res
+grep '^ERROR:' hodgepodge-all.res
+
+:
diff --git a/tests/tap-html.test b/tests/tap-html.test
new file mode 100755
index 0000000..b5a1beb
--- /dev/null
+++ b/tests/tap-html.test
@@ -0,0 +1,80 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - "check-html" and "recheck-html" targets
+# - reStructuredText -> HTML conversion
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_COMPILER = cat
+TESTS = ok.test ko.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > ok.test << 'END'
+1..3
+ok 1 - expected success
+not ok 2 - expectred failure # TODO
+ok 3 - skipped test # SKIP
+END
+
+cat > ko.test << 'END'
+1..2
+not ok 1 - unexpected failure
+ok 2 - spurious success # TODO
+Bail out! hard error
+END
+
+no_rst2html_error ()
+{
+ $EGREP '(test-suite\.log:[0-9]|WARNING|SEVERE)' output && Exit 1
+ :
+}
+
+# Overriding TESTS should work with check-html.
+TESTS=ok.test $MAKE -e check-html >output 2>&1 || { cat output; Exit 1; }
+cat output
+no_rst2html_error
+test -f test-suite.html
+
+mv output stdout
+count_test_results total=3 pass=1 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+: > older
+$sleep
+$MAKE check-html >output 2>&1 && { cat output; Exit 1; }
+cat output
+no_rst2html_error
+# test-suite.html should be remake if if it was up-to-date.
+test -f test-suite.html
+is_newest test-suite.html older
+
+mv output stdout
+count_test_results total=6 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=1
+
+rm -f ok.test
+$MAKE recheck-html >output 2>&1 && { cat output; Exit 1; }
+cat output
+no_rst2html_error
+
+mv output stdout
+count_test_results total=3 pass=0 fail=1 xpass=1 xfail=0 skip=0 error=1
+
+:
diff --git a/tests/tap-log.test b/tests/tap-log.test
new file mode 100755
index 0000000..eecb2a2
--- /dev/null
+++ b/tests/tap-log.test
@@ -0,0 +1,164 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - log file creation
+# - log file removal
+# - stdout and stderr of a script go in its log file
+# - TEST_SUITE_LOG redefinition, at either automake or make time
+# - VERBOSE environment variable support
+# Keep in sync with 'test-log.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TESTS = pass.test skip.test xfail.test fail.test xpass.test error.test
+TEST_SUITE_LOG = global.log
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Custom markers, for use in grepping checks.
+cmarker=::: # comment marker
+pmarker=%%% # plain maker
+
+cat > pass.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker pass $pmarker" >&2
+echo "# $cmarker pass $cmarker" >&2
+echo "ok 1"
+END
+
+cat > skip.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker skip $pmarker"
+echo "# $cmarker skip $cmarker"
+echo "ok 1 # SKIP"
+END
+
+cat > xfail.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker xfail $pmarker" >&2
+echo "# $cmarker xfail $cmarker" >&2
+echo "not ok 1 # TODO"
+END
+
+cat > fail.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker fail $pmarker"
+echo "# $cmarker fail $cmarker"
+echo "not ok 1"
+END
+
+cat > xpass.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker xpass $pmarker" >&2
+echo "# $cmarker xpass $cmarker" >&2
+echo "ok 1 # TODO"
+END
+
+cat > error.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker error $pmarker"
+echo "# $cmarker error $cmarker"
+echo 'Bail out!'
+END
+
+chmod a+x *.test
+
+TEST_SUITE_LOG=my.log $MAKE -e check && Exit 1
+ls -l # For debugging.
+test ! -f test-suite.log
+test ! -f global.log
+test -f my.log
+st=0
+for result in pass fail xfail xpass skip error; do
+ cat $result.log # For debugging.
+ $FGREP "$pmarker $result $pmarker" $result.log || st=1
+ $FGREP "$cmarker $result $cmarker" $result.log || st=1
+done
+test $st -eq 0 || Exit 1
+cat my.log # For debugging.
+for result in xfail fail xpass skip error; do
+ cat $result.log # For debugging.
+ $FGREP "$pmarker $result $pmarker" my.log || st=1
+ $FGREP "$cmarker $result $cmarker" my.log || st=1
+done
+test `$FGREP -c "$pmarker" my.log` -eq 5
+test `$FGREP -c "$cmarker" my.log` -eq 5
+
+# Passed test scripts shouldn't be mentioned in the global log.
+$EGREP '(^pass|[^x]pass)\.test' my.log && Exit 1
+# But failing (expectedly or not) and skipped ones should.
+$FGREP 'xfail.test' my.log
+$FGREP 'skip.test' my.log
+$FGREP 'fail.test' my.log
+$FGREP 'xpass.test' my.log
+$FGREP 'error.test' my.log
+
+touch error2.log test-suite.log global.log
+TEST_SUITE_LOG=my.log $MAKE -e mostlyclean
+ls -l # For debugging.
+test ! -f my.log
+test ! -f pass.log
+test ! -f fail.log
+test ! -f xfail.log
+test ! -f xpass.log
+test ! -f skip.log
+test ! -f error.log
+# "make mostlyclean" shouldn't remove unrelated log files.
+test -f error2.log
+test -f test-suite.log
+test -f global.log
+
+rm -f *.log
+
+VERBOSE=yes $MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+cat global.log
+test ! -f my.log
+test ! -f test-suite.log
+# Check that VERBOSE causes the global testsuite log to be
+# emitted on stdout.
+out=`cat stdout`
+log=`cat global.log`
+case $out in *"$log"*) ;; *) Exit 1;; esac
+
+touch error2.log test-suite.log my.log
+$MAKE clean
+ls -l # For debugging.
+test ! -f global.log
+test ! -f pass.log
+test ! -f fail.log
+test ! -f xfail.log
+test ! -f xpass.log
+test ! -f skip.log
+test ! -f error.log
+# "make clean" shouldn't remove unrelated log files.
+test -f error2.log
+test -f test-suite.log
+test -f my.log
+
+rm -f *.log
+
+:
diff --git a/tests/tap-merge-stdout-stderr.test
b/tests/tap-merge-stdout-stderr.test
new file mode 100755
index 0000000..787ad51
--- /dev/null
+++ b/tests/tap-merge-stdout-stderr.test
@@ -0,0 +1,73 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - The Automake TAP driver has an option that instruct it to read TAP
+# input also from the stderr of the test command, not only its stdout.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat > Makefile.am << 'END'
+AM_TEST_LOG_DRIVER_FLAGS = --comments --merge
+TESTS = all.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+#!/bin/sh
+echo 1..4
+echo ok 1 >&2
+echo ok 2
+echo "not ok 3 # TODO" >&2
+echo "ok 4 # SKIP"
+echo "# foo foo foo" >&2
+END
+chmod a+x all.test
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=1 skip=1 error=0
+grep '^# all\.test: foo foo foo' stdout
+
+cat > all.test <<END
+#!/bin/sh
+echo 1..1
+echo ok 1
+echo 'Bail out!' >&2
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1
+
+# See that the option `--no-merge' can override the effect of `--merge'.
+
+TEST_LOG_DRIVER_FLAGS=--no-merge $MAKE -e check >stdout \
+ || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+
+
+:
diff --git a/tests/tap-message-0.test b/tests/tap-message-0.test
new file mode 100755
index 0000000..bce333f
--- /dev/null
+++ b/tests/tap-message-0.test
@@ -0,0 +1,66 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - having "0" as a test description (or TODO or SKIP message) should
+# be supported
+# Note that a bug in some versions of TAP::Parser causes this not to be
+# generally true!
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test << 'END'
+1..10
+ok 1 0
+ok - 0
+not ok 3 0 # TODO
+not ok - 0 # TODO
+ok 5 0 # SKIP
+ok - 0 # SKIP
+not ok 7 # TODO 0
+ok 8 # SKIP 0
+not ok 9 0 # TODO 0
+ok 10 0 # SKIP 0
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=10 pass=2 fail=0 xpass=0 xfail=4 skip=4 error=0
+
+cat > exp << 'END'
+PASS: all.test 1 0
+PASS: all.test 2 - 0
+XFAIL: all.test 3 0 # TODO
+XFAIL: all.test 4 - 0 # TODO
+SKIP: all.test 5 0 # SKIP
+SKIP: all.test 6 - 0 # SKIP
+XFAIL: all.test 7 # TODO 0
+SKIP: all.test 8 # SKIP 0
+XFAIL: all.test 9 0 # TODO 0
+SKIP: all.test 10 0 # SKIP 0
+END
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-more.test b/tests/tap-more.test
new file mode 100755
index 0000000..7d0bf86
--- /dev/null
+++ b/tests/tap-more.test
@@ -0,0 +1,164 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# More on TAP support:
+# - more than one TAP-generating test script in $(TESTS)
+# - VPATH builds
+# - use with parallel make (if supported)
+# - basic use of diagnostic messages (lines beginning with "#")
+# - flags for TAP driver defined through AC_SUBST in configure.ac
+# - messages generated by the testsuite harness reference the
+# correct test script(s)
+# - "make distcheck" works
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_SUBST([AM_TEST_LOG_DRIVER_FLAGS], ['--comments'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TESTS = 1.test 2.test 3.test
+EXTRA_DIST = $(TESTS) tap-driver
+END
+
+cat > 1.test <<'END'
+#! /bin/sh
+echo 1..2
+echo ok 1 - mu
+if test -f not-skip; then
+ echo "not ok 2 zardoz"
+else
+ echo "ok 2 zardoz # SKIP"
+fi
+END
+
+cat > 2.test <<'END'
+#! /bin/sh
+echo 1..3
+echo "ok"
+echo "not ok # TODO not implemented"
+echo "ok 3"
+END
+
+cat > 3.test <<END
+#! /bin/sh
+echo 1..1
+echo ok - blah blah blah
+echo '# Some diagnostic'
+if test -f bail-out; then
+ echo 'Bail out! Kernel Panic'
+else
+ :
+fi
+END
+
+chmod a+x [123].test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+# Try a VPATH and by default serial build first, and then an in-tree
+# and by default parallel build.
+for try in 0 1; do
+
+ if test $try -eq 0; then
+ # VPATH serial build.
+ mkdir build
+ cd build
+ srcdir=..
+ run_make=$MAKE
+ elif test $try -eq 1; then
+ # In-tree parallel build.
+ srcdir=.
+ case $MAKE in
+ *\ -j*)
+ # Degree of parallelism already specified by the user: do
+ # not override it.
+ run_make=$MAKE
+ ;;
+ *)
+ # Some make implementations (e.g., HP-UX) don't grok `-j',
+ # some require no space between `-j' and the number of jobs
+ # (e.g., older GNU make versions), and some *do* require a
+ # space between `-j' and the number of jobs (e.g., Solaris
+ # dmake). We need a runtime test to see what works.
+ echo 'all:' > Makefile
+ for run_make in "$MAKE -j3" "$MAKE -j 3" "$MAKE"; do
+ $run_make && break
+ done
+ rm -f Makefile
+ ;;
+ esac
+ else
+ fatal_ "internal error, invalid value of '$try' for \$try"
+ fi
+
+ $srcdir/configure
+ ls -l # For debugging.
+
+ # Success.
+
+ $run_make check >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=6 pass=4 fail=0 xpass=0 xfail=1 skip=1 error=0
+ grep '^PASS: 1\.test 1 - mu$' stdout
+ grep '^SKIP: 1\.test 2 zardoz # SKIP$' stdout
+ test `$FGREP -c '1.test' stdout` -eq 2
+ grep '^PASS: 2\.test 1$' stdout
+ grep '^XFAIL: 2\.test 2 # TODO not implemented$' stdout
+ grep '^PASS: 2\.test 3$' stdout
+ test `$FGREP -c '2.test' stdout` -eq 3
+ grep '^PASS: 3\.test 1 - blah blah blah$' stdout
+ grep '^# 3\.test: Some diagnostic$' stdout
+ test `$FGREP -c '3.test' stdout` -eq 2
+
+ # Failure.
+
+ # Use `echo' here, since Solaris 10 /bin/sh would try to optimize
+ # a `:' away after the first iteration, even if it is redirected.
+ echo dummy > not-skip
+ echo dummy > bail-out
+ $run_make check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=7 pass=4 fail=1 xpass=0 xfail=1 skip=0 error=1
+ grep '^PASS: 1\.test 1 - mu$' stdout
+ grep '^FAIL: 1\.test 2 zardoz$' stdout
+ test `$FGREP -c '1.test' stdout` -eq 2
+ grep '^PASS: 2\.test 1$' stdout
+ grep '^XFAIL: 2\.test 2 # TODO not implemented$' stdout
+ grep '^PASS: 2\.test 3$' stdout
+ test `$FGREP -c '2.test' stdout` -eq 3
+ grep '^PASS: 3\.test 1 - blah blah blah$' stdout
+ grep '^# 3\.test: Some diagnostic$' stdout
+ grep '^ERROR: 3\.test - Bail out! Kernel Panic$' stdout
+ test `$FGREP -c '3.test' stdout` -eq 3
+
+ cd $srcdir
+
+done
+
+$MAKE distcheck
+
+:
diff --git a/tests/tap-more2.test b/tests/tap-more2.test
new file mode 100755
index 0000000..810f9b8
--- /dev/null
+++ b/tests/tap-more2.test
@@ -0,0 +1,93 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# More on TAP support:
+# - more LOG_COMPILER at once for TAP tests
+# - binary programs in $(TESTS)
+# - interactions with `check_*' variables
+
+required='cc native'
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_PROG_CC
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .sh .tap
+
+tap_driver = $(PERL) $(srcdir)/tap-driver
+
+LOG_DRIVER = $(tap_driver)
+SH_LOG_DRIVER = $(tap_driver)
+TAP_LOG_DRIVER = $(tap_driver)
+
+TAP_LOG_COMPILER = cat
+SH_LOG_COMPILER = $(SHELL)
+LOG_COMPILER =
+
+EXTRA_DIST = baz.tap
+
+check_SCRIPTS = bar.sh
+bar.sh: Makefile
+ echo '#!/bin/sh' > address@hidden
+ echo 'echo 1..1' >> address@hidden
+ echo 'echo "not ok 1 # TODO"' >> address@hidden
+ chmod a-w address@hidden && mv -f address@hidden $@
+CLEANFILES = bar.sh
+
+check_PROGRAMS = foo-test
+foo_test_SOURCES = foo.c
+
+TESTS = foo-test $(check_SCRIPTS) baz.tap
+
+EXTRA_DIST += tap-driver
+END
+
+cat > foo.c <<'END'
+#include <stdio.h>
+int main (void)
+{
+ printf ("1..1\n");
+ printf ("ok 1\n");
+ return 0;
+}
+END
+
+cat > baz.tap << 'END'
+1..1
+ok 1 # SKIP
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+for target in check distcheck; do
+ $MAKE $target >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=3 pass=1 fail=0 xpass=0 xfail=1 skip=1 error=0
+done
+
+:
diff --git a/tests/tap-no-disable-hard-error.test
b/tests/tap-no-disable-hard-error.test
new file mode 100755
index 0000000..678e184
--- /dev/null
+++ b/tests/tap-no-disable-hard-error.test
@@ -0,0 +1,50 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - "Bail out!" magic and TAP parse errors are not disabled nor turned
+# into simple failures by the definition DISABLE_HARD_ERRORS.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+DISABLE_HARD_ERRORS = yes
+TEST_LOG_COMPILER = cat
+TESTS = bail.test few.test noplan.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > bail.test <<END
+1..1
+Bail out!
+END
+
+cat > few.test <<END
+1..1
+END
+
+cat > noplan.test <<END
+# nothing here
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3
+
+:
diff --git a/tests/tap-no-merge-stdout-stderr.test
b/tests/tap-no-merge-stdout-stderr.test
new file mode 100755
index 0000000..25a06fd
--- /dev/null
+++ b/tests/tap-no-merge-stdout-stderr.test
@@ -0,0 +1,53 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - By default, TAP input is only from the stdout (and not the stderr)
+# of the test command.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER_FLAGS = --comments
+TESTS = all.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+#!/bin/sh
+echo 1..2
+echo 'not ok' >&2
+echo 'not ok 1 # TODO' >&2
+echo 'ok 1'
+echo '# foo foo foo'
+echo '# bar bar bar' >&2
+echo 'ok 44 # TODO' >&2
+echo 'Bail out!' >&2
+echo 'ok 2 # SKIP'
+END
+
+chmod a+x all.test
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=1 fail=0 xpass=0 xfail=0 skip=1 error=0
+$FGREP 'foo foo foo' stdout
+$FGREP 'bar bar bar' stdout && Exit 1
+
+:
diff --git a/tests/tap-no-spurious-summary.test
b/tests/tap-no-spurious-summary.test
new file mode 100755
index 0000000..bc7fd51
--- /dev/null
+++ b/tests/tap-no-spurious-summary.test
@@ -0,0 +1,47 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - `:test-results:' directives in test scripts' output doesn't
+# originate spurious results in the testsuite summary
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..1
+:test-result: PASS
+:test-result: FAIL
+:test-result: XPASS
+:test-result: XFAIL
+:test-result: SKIP
+:test-result: ERROR
+:test-result: UNKNOWN
+ok 1
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+for result in PASS FAIL XPASS XFAIL SKIP ERROR UNKNOWN; do
+ grep "^ *:test-result: $result$" all.log
+done
+
+:
diff --git a/tests/tap-no-spurious.test b/tests/tap-no-spurious.test
new file mode 100755
index 0000000..91b115f
--- /dev/null
+++ b/tests/tap-no-spurious.test
@@ -0,0 +1,102 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - don't spuriously recognize lines that are "almost" TAP lines as
+# real TAP lines
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo 1..5 > all.test
+
+# The only recognized directives are "TODO" and "SKIP".
+# So the following should count as passed tests.
+cat >> all.test <<END
+ok 1 # XFAIL
+ok 2 # SKIPPED
+ok 3 # TO DO
+ok 4 # TODOALL
+ok 5 # FIXME
+END
+
+# According to documentation of Test::Harness::TAP(3):
+#
+# ``Lines written to standard output matching /^(not )?ok\b/
+# must be interpreted as test lines. All other lines must
+# not be considered test output.''
+
+cat >> all.test <<END
+ ok
+ ok 1
+${tab}ok
+${tab}ok 1
+ok1
+ok23
+ not ok
+ not ok 1
+${tab}not ok
+${tab}not ok 1
+not ok1
+not ok23
+notok
+notok 1
+not${tab}ok
+not${tab}ok 1
+not ok
+not ok 1
+no ok
+no ok 1
+# ok
+# not ok
+# ok 1
+# not ok 1
+#ok
+#not ok
+#ok 1
+#not ok 1
+END
+
+# The prove(1) utility doesn't bail out on these, so our driver
+# shouldn't either.
+cat >> all.test <<'END'
+bailout
+bailout!
+bail out
+bail out!
+Bailout
+Bailout!
+Bail out
+ Bail out
+ Bail out!
+#Bail out!
+# Bail out!
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=5 pass=5 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+:
diff --git a/tests/tap-not-ok-skip.test b/tests/tap-not-ok-skip.test
new file mode 100755
index 0000000..f154d23
--- /dev/null
+++ b/tests/tap-not-ok-skip.test
@@ -0,0 +1,39 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - a "not ok # SKIP" line should count as a failure, for consistency
+# with the prove(1) utility.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..4
+not ok # SKIP
+not ok 2 # SKIP
+not ok - foo # SKIP
+not ok 4 - bar # SKIP
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=4 pass=0 fail=4 skip=0 xpass=0 xfail=0 error=0
+
+:
diff --git a/tests/tap-numeric-description.test
b/tests/tap-numeric-description.test
new file mode 100755
index 0000000..ea01038
--- /dev/null
+++ b/tests/tap-numeric-description.test
@@ -0,0 +1,69 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP result lines whose description is a number.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Some random numbers to be used as test names. The definitions below are
+# selected so that $x<n> != <n> for every n >= 1. We can't use positional
+# parameters for this unfortunately, since Solaris 10 /bin/sh doesn't allow
+# the dereferencing of parameters from ${10} onwards.
+x1=5 x2=7 x3=45 x4=11 x5=7 x6=3 x7=6 x8=9 x9=1000 x10=1
+
+cat > all.test <<END
+1..10
+ok 1 ${x1}
+ok - ${x2}
+not ok 3 ${x3}
+not ok - ${x4}
+ok 5 ${x5} # SKIP
+ok - ${x6} # SKIP
+not ok 7 ${x7} # TODO
+not ok - ${x8} # TODO
+ok 9 ${x9} # TODO
+ok - ${x10} # TODO
+END
+
+cat > exp <<END
+PASS: all.test 1 ${x1}
+PASS: all.test 2 - ${x2}
+FAIL: all.test 3 ${x3}
+FAIL: all.test 4 - ${x4}
+SKIP: all.test 5 ${x5} # SKIP
+SKIP: all.test 6 - ${x6} # SKIP
+XFAIL: all.test 7 ${x7} # TODO
+XFAIL: all.test 8 - ${x8} # TODO
+XPASS: all.test 9 ${x9} # TODO
+XPASS: all.test 10 - ${x10} # TODO
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=10 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=0
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+
+:
diff --git a/tests/tap-out-of-order.test b/tests/tap-out-of-order.test
new file mode 100755
index 0000000..f8f69c7
--- /dev/null
+++ b/tests/tap-out-of-order.test
@@ -0,0 +1,88 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - out-of-order test results
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > a.test <<END
+1..3
+ok
+not ok 2 # TODO
+ok 71
+END
+
+cat > b.test <<END
+1..5
+ok 1
+ok 2
+ok 4 foo # SKIP mu
+not ok 2 bar # TODO um
+ok 5
+END
+
+cat > c.test <<END
+1..4
+ok 1
+ok foo
+ok 4 - zardoz
+not ok 3
+END
+
+# This is documented as an "test out of sequence" error in the
+# TAP::Parser documentation. Keep it in sync with the example
+# there.
+cat > d.test <<'END'
+1..5
+ok 1
+ok 2
+ok 15
+ok 16
+ok 17
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+
+TESTS='a.test b.test c.test d.test' $MAKE -e check >stdout \
+ && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=17 pass=8 fail=0 xpass=0 xfail=1 skip=0 error=8
+
+test `$FGREP -c ': a.test' stdout` -eq 3
+test `$FGREP -c ': b.test' stdout` -eq 5
+test `$FGREP -c ': c.test' stdout` -eq 4
+test `$FGREP -c ': d.test' stdout` -eq 5
+
+grep '^ERROR: a\.test 71 # OUT-OF-ORDER (expecting 3)$' stdout
+grep '^ERROR: b\.test 4 foo # OUT-OF-ORDER (expecting 3)$' stdout
+grep '^ERROR: b\.test 2 bar # OUT-OF-ORDER (expecting 4)$' stdout
+grep '^ERROR: c\.test 4 - zardoz # OUT-OF-ORDER (expecting 3)$' stdout
+grep '^ERROR: c\.test 3 # OUT-OF-ORDER (expecting 4)$' stdout
+grep '^ERROR: d\.test 15 # OUT-OF-ORDER (expecting 3)$' stdout
+grep '^ERROR: d\.test 16 # OUT-OF-ORDER (expecting 4)$' stdout
+grep '^ERROR: d\.test 17 # OUT-OF-ORDER (expecting 5)$' stdout
+
+:
diff --git a/tests/tap-passthrough-exit.test b/tests/tap-passthrough-exit.test
new file mode 100755
index 0000000..ae8ffa8
--- /dev/null
+++ b/tests/tap-passthrough-exit.test
@@ -0,0 +1,66 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - non-success exit status of a test script is reported in the
+# log file
+# See also related test 'tap-passthrough.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_COMPILER = $(SHELL)
+TESTS =
+END
+
+exit_statuses='1 2 77 99'
+
+for e in $exit_statuses; do
+ unindent > exit-$e.test <<END
+ #!/bin/sh
+ echo 1..1
+ echo ok 1
+ exit $e
+END
+ echo TESTS += exit-$e.test >> Makefile.am
+done
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+st=0
+$MAKE check || st=$?
+for e in $exit_statuses; do cat exit-$e.log; done
+cat test-suite.log
+test $st -gt 0 || Exit 1
+
+for e in $exit_statuses; do
+ for log in exit-$e.log test-suite.log; do
+ grep "^ERROR: exit-$e\\.test - exited with status $e$" $log
+ done
+done
+
+st=0
+env TEST_LOG_DRIVER_FLAGS='--ignore-exit' $MAKE -e check || st=$?
+for e in $exit_statuses; do cat exit-$e.log; done
+cat test-suite.log
+test $st -eq 0 || Exit 1
+
+for e in $exit_statuses; do
+ grep "exit-$e\\.test - exited with status $e$" exit-$e.log
+done
+
+:
diff --git a/tests/tap-passthrough.test b/tests/tap-passthrough.test
new file mode 100755
index 0000000..ba773ed
--- /dev/null
+++ b/tests/tap-passthrough.test
@@ -0,0 +1,166 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - all input (valid TAP lines, invalid TAP lines, non-TAP lines)
+# are passed through in the log file
+# - TAP errors are reported in the log file too
+# See also related test 'tap-passthrough-exit.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+weirdchars=\''"address@hidden&()[]<>#;,:.^?*/'
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+#
+# Only successful tests.
+#
+
+# The whitespace in this test might be normalized in the testsuite
+# progress output, but should be copied verbatim in the log files.
+cat > ok.test <<END
+1..6
+TAP plan in the previous line.
+ok${tab}
+ok 2
+ok - foo
+ok 4 - x
+ This is not a TAP line, but should still be copied in the log file!
+# some diagnostic${tab}
+not ok # TODO low priority
+ok # SKIP who cares?
+$weirdchars
+END
+
+TESTS=ok.test $MAKE -e check || { cat ok.log; Exit 1; }
+cat ok.log
+
+for rx in \
+ '1\.\.6' \
+ 'TAP plan in the previous line\.' \
+ "ok${tab}" \
+ 'ok 2' \
+ 'ok - foo' \
+ 'ok 4 - x' \
+ ' This is not a TAP line, but should still be copied in the log file!' \
+ "# some diagnostic${tab}" \
+ 'not ok # TODO low priority' \
+ 'ok # SKIP who cares?' \
+; do
+ grep "^$rx$" ok.log
+done
+$FGREP "$weirdchars" ok.log
+
+#
+# Mixed failing/successful tests.
+#
+
+cat > tiny.test <<END
+1..1
+ok
+END
+
+cat > ok.test <<END
+1..1
+ok
+only one success here
+END
+
+cat > ko.test <<END
+1..5
+foo foo foo
+ok${tab}
+ok 2
+not ok - foo
+not ok 4 - x
+# diagnostic ko
+ bar${tab}bar${tab}bar
+ok # TODO dunno
+$weirdchars
+END
+
+cat > bail.test <<END
+Bail out! Test is taking too long!
+END
+
+cat > skip.test <<END
+1..0 # Skipped: WWW::Mechanize not installed
+END
+
+cat > err.test <<END
+1..3
+ok 1
+Invalid test count
+ok 23
+Misplaced plan
+1..13
+ok
+Extra test
+ok
+Last line
+END
+
+st=0
+env TESTS='tiny.test ok.test ko.test bail.test skip.test err.test' \
+ $MAKE -e check || st=$?
+cat tiny.log
+cat ok.log
+cat ko.log
+cat bail.log
+cat skip.log
+cat err.log
+test $st -gt 0 || Exit 1
+
+grep '^1\.\.1$' tiny.log
+grep '^ok$' tiny.log
+grep '^only one success here$' ok.log
+
+for rx in \
+ '1\.\.5' \
+ 'foo foo foo' \
+ "ok${tab}" \
+ 'ok 2' \
+ 'not ok - foo' \
+ 'not ok 4 - x' \
+ '# diagnostic ko' \
+ " bar${tab}bar${tab}bar" \
+ 'ok # TODO dunno' \
+; do
+ grep "^$rx$" ko.log
+done
+$FGREP "$weirdchars" ko.log
+
+grep '^Bail out! Test is taking too long!$' bail.log
+grep '^1\.\.0 # Skipped: WWW::Mechanize not installed$' skip.log
+
+for rx in \
+ '^1\.\.3$' \
+ '^Invalid test count$' \
+ '^ok 23$' \
+ '^Misplaced plan$' \
+ '^1\.\.13$' \
+ '^ERROR:.* test plan in middle of output' \
+ '^Extra test$' \
+ '^Last line$' \
+ '^ERROR:.* [tT]oo many tests run.*expected 3, got 4' \
+ '^ERROR:.* err\.test 23 .*OUT[ -]OF[ -]ORDER.*expecting 2' \
+; do
+ grep "$rx" err.log
+done
+
+:
diff --git a/tests/tap-plan-corner.test b/tests/tap-plan-corner.test
new file mode 100755
index 0000000..4215556
--- /dev/null
+++ b/tests/tap-plan-corner.test
@@ -0,0 +1,63 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - some corner cases for TAP plan
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > leading-repeated-1.test <<END
+1..1
+1..1
+ok 1
+END
+
+cat > trailing-repeated-1.test <<END
+ok 1
+1..1
+1..1
+END
+
+cat > leading-repeated-2.test <<END
+1..2
+ok 1
+1..2
+ok 2
+END
+
+cat > trailing-repeated-2.test <<END
+ok 1
+1..2
+ok 2
+1..2
+END
+
+for i in 1 2; do
+ for kind in leading trailing; do
+ cp $kind-repeated-$i.test all.test
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=`expr $i + 2` pass=$i fail=0 \
+ xpass=0 xfail=0 skip=0 error=2
+ grep '^ERROR: all\.test - test plan in middle of output$' stdout
+ grep '^ERROR: all\.test - multiple test plans$' stdout
+ done
+done
+
+:
diff --git a/tests/tap-plan-corner2.test b/tests/tap-plan-corner2.test
new file mode 100755
index 0000000..54fd5e9
--- /dev/null
+++ b/tests/tap-plan-corner2.test
@@ -0,0 +1,49 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - more corner cases for TAP plan
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# The leading blank line is meant.
+cat > leading-blank.test <<END
+
+1..2
+ok 1
+ok 2
+END
+
+# The trailing blank line is meant.
+cat > trailing-blank.test <<END
+ok 1
+ok 2
+1..2
+
+END
+
+for kind in leading trailing; do
+ cp $kind-blank.test all.test
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1
+ grep '^ERROR: all\.test - test plan in middle of output$' stdout
+done
+
+:
diff --git a/tests/tap-plan-errors.test b/tests/tap-plan-errors.test
new file mode 100755
index 0000000..fd12e1b
--- /dev/null
+++ b/tests/tap-plan-errors.test
@@ -0,0 +1,92 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: the following situations should be flagged as errors:
+# - unmatched test plan (too few tests run)
+# - misplaced test plan
+# - multiple test plans
+# - missing test plan
+# Checks about unplanned tests are performed in 'tap-unplanned.test'.
+# More checks about corner-cases in TAP plans are performed in
+# 'tap-plan-corner.test' and 'tap-plan-corner2.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+my_check ()
+{
+ cat > all.test
+ test -n "$err" || fatal_ "\$err not set before calling my_check"
+ cat all.test # For debugging.
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results "$@"
+ grep "^ERROR: all\\.test - $err$" stdout
+ unset err
+}
+
+err='too few tests run (expected 2, got 1)'
+my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+1..2
+ok 1
+END
+
+err='too few tests run (expected 12, got 3)'
+my_check total=4 pass=2 fail=0 xpass=0 xfail=1 skip=0 error=1 <<END
+ok 1
+ok 2
+not ok 3 # TODO
+1..12
+END
+
+err='too few tests run (expected 1, got 0)'
+my_check total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+1..1
+END
+
+err='test plan in middle of output'
+my_check total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+oops
+1..2
+ok 1
+ok 2
+END
+
+err='test plan in middle of output'
+my_check total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+ok 1
+ok 2
+1..2
+oops
+END
+
+# The two test plans here are deliberately equal.
+err='multiple test plans'
+my_check total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+1..2
+ok 1
+ok 2
+1..2
+END
+
+err='missing test plan'
+my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+ok 1
+END
+
+:
diff --git a/tests/tap-plan.test b/tests/tap-plan.test
new file mode 100755
index 0000000..79e9bdd
--- /dev/null
+++ b/tests/tap-plan.test
@@ -0,0 +1,51 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - test scripts with the test plan at the beginning
+# - test scripts with the test plan at the end
+# - test scripts without a test plan
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > top.test <<END
+1..3
+ok 1
+ok 2 # SKIP
+ok 3
+# a trailing comment
+END
+
+cat > bot.test <<END
+ok 1
+# a comment
+ok 2
+not ok 3 # TODO
+# another comment
+ok
+1..4
+END
+
+# Check that the plans doesn't cause any problem
+
+TESTS='top.test bot.test' $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=7 pass=5 xfail=1 skip=1 fail=0 xpass=0 error=0
+
+:
diff --git a/tests/tap-realtime.test b/tests/tap-realtime.test
new file mode 100755
index 0000000..4a38f14
--- /dev/null
+++ b/tests/tap-realtime.test
@@ -0,0 +1,101 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - testsuite progress on console should happen mostly "in real time";
+# i.e., it's not acceptable for the driver to wait the end of the
+# script to start displaying results from it.
+# FIXME: this test uses expect(1) to ensure line buffering from make and
+# children, and is pretty hacky and complex; is there a better way to
+# accomplish the checks done here?
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >expect-check <<'END'
+eval spawn $env(SHELL) -c ":"
+expect eof
+END
+expect -f expect-check || {
+ echo "$me: failed to find a working expect program" >&2
+ Exit 77
+}
+rm -f expect-check
+
+cat > Makefile.am << 'END'
+TESTS = all.test
+END
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+#! /bin/sh
+echo 1..3
+
+# FIXME: creative quoting to placate maintainer-check
+sleep="sleep "3
+
+echo ok 1 - foo
+$sleep
+test -f ok-1 || { echo 'Bail out!'; exit 1; }
+
+echo ok 2 - bar
+$sleep
+test -f ok-2 || { echo 'Bail out!'; exit 1; }
+
+echo ok 3 - baz
+$sleep
+test -f ok-3 || { echo 'Bail out!'; exit 1; }
+
+: > all-is-well
+END
+
+chmod a+x all.test
+
+cat > expect-make <<'END'
+eval spawn $env(MAKE) check
+expect {
+ "PASS: all.test 1 - foo" {
+ open "ok-1" "w"
+ exp_continue
+ }
+ "PASS: all.test 2 - bar" {
+ open "ok-2" "w"
+ exp_continue
+ }
+ "PASS: all.test 3 - baz" {
+ open "ok-3" "w"
+ exp_continue
+ }
+ "Testsuite summary" {
+ exit 0
+ }
+ timeout {
+ puts "expect timed out"
+ exit 1
+ }
+ default {
+ puts "expect error"
+ exit 1
+ }
+}
+END
+
+# Expect should simulate a tty as stdout, which should ensure a
+# line-buffered output.
+MAKE=$MAKE expect -f expect-make
+test -f all-is-well
+
+:
diff --git a/tests/tap-recheck-logs.test b/tests/tap-recheck-logs.test
new file mode 100755
index 0000000..6e1b646
--- /dev/null
+++ b/tests/tap-recheck-logs.test
@@ -0,0 +1,119 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - RECHECK_LOGS
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_COMPILER = cat
+TESTS = foo.test bar.test baz.test
+baz.log: zardoz
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+: > zardoz
+
+cat > foo.test <<'END'
+1..2
+ok 1
+ok 2
+END
+
+cat > bar.test <<'END'
+1..1
+not ok 1
+END
+
+cat > baz.test <<'END'
+1..1
+Bail out!
+END
+
+# Even the tests that are not re-run should contribute to the testsuite
+# summary when obtained by "make check RECHECK_LOGS=".
+grep_summary ()
+{
+ grep '^# TOTAL: *4$' stdout
+ grep '^# PASS: *2$' stdout
+ grep '^# XPASS: *0$' stdout
+ grep '^# FAIL: *1$' stdout
+ grep '^# XFAIL: *0$' stdout
+ grep '^# SKIP: *0$' stdout
+ grep '^# ERROR: *1$' stdout
+}
+
+$MAKE -e check && Exit 1
+test -f foo.log
+test -f bar.log
+test -f baz.log
+
+rm -f foo.log bar.log
+
+env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+test -f foo.log
+test -f bar.log
+grep '^PASS: foo\.test 1$' stdout
+grep '^PASS: foo\.test 2$' stdout
+grep '^FAIL: bar\.test 1$' stdout
+grep 'baz\.test' stdout && Exit 1
+grep_summary
+
+$sleep
+touch foo.test
+# We re-run only a successful test, but the tests that failed in the
+# previous run should still be taken into account, and cause an overall
+# failure.
+env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^PASS: foo\.test 1$' stdout
+grep '^PASS: foo\.test 2$' stdout
+grep 'ba[rz]\.test' stdout && Exit 1
+is_newest foo.log foo.test
+grep_summary
+
+$sleep
+touch zardoz
+env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^ERROR: baz\.test' stdout
+$EGREP '(foo|bar)\.test' stdout && Exit 1
+is_newest baz.log zardoz
+grep_summary
+
+# Now, explicitly retry with all test logs already updated, and ensure
+# that the summary is still displayed.
+env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+$EGREP '(foo|bar|baz)\.test' stdout && Exit 1
+grep_summary
+
+# The following should re-run foo.test (and only foo.test), even if its
+# log file is up-to-date.
+: > older
+env RECHECK_LOGS=foo.log $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^PASS: foo\.test 1$' stdout
+grep '^PASS: foo\.test 2$' stdout
+grep 'ba[rz]\.test' stdout && Exit 1
+is_newest foo.log older
+grep_summary
+
+:
diff --git a/tests/tap-recheck.test b/tests/tap-recheck.test
new file mode 100755
index 0000000..14b02ad
--- /dev/null
+++ b/tests/tap-recheck.test
@@ -0,0 +1,217 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test the 'recheck' target for TAP test protocol.
+# Keep in sync with 'test-driver-custom-multitest-recheck.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TESTS = a.test b.test c.test d.test
+END
+
+cat > a.test << 'END'
+#! /bin/sh
+echo 1..2
+echo ok 1
+echo ok 2
+: > a.run
+END
+
+cat > b.test << 'END'
+#! /bin/sh
+: > b.run
+echo 1..2
+echo ok 1
+if test -f b.ok; then
+ echo ok 2
+else
+ echo 'Bail out!'
+fi
+END
+
+cat > c.test << 'END'
+#! /bin/sh
+echo 1..3
+if test -f c.pass; then
+ echo 'ok - c is ok :-)'
+else
+ echo 'not ok - c is ko :-('
+fi
+if test -f c.xfail; then
+ echo 'not ok 2 # TODO'
+else
+ echo 'ok 2 # TODO'
+fi
+echo 'not ok 3 - blah blah # TODO need better diagnostic'
+: > c.run
+END
+
+cat > d.test << 'END'
+#! /bin/sh
+test -f ./d.count && . ./d.count
+echo 1..${test_count-2}
+echo ok 1 '# SKIP: who cares ...'
+(. ./d.extra) || echo 'not ok 2 - d.extra failed'
+: > d.run
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+do_recheck ()
+{
+ case $* in
+ --fail) on_bad_rc='&&';;
+ --pass) on_bad_rc='||';;
+ *) fatal_ "invalid usage of function 'do_recheck'";;
+ esac
+ rm -f *.run
+ eval "\$MAKE recheck >stdout $on_bad_rc { cat stdout; ls -l; Exit 1; }; :"
+ cat stdout; ls -l
+}
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ : A "make recheck" in a clean tree should run no tests.
+ do_recheck --pass
+ cat test-suite.log
+ test ! -r a.run
+ test ! -r a.log
+ test ! -r b.run
+ test ! -r b.log
+ test ! -r c.run
+ test ! -r c.log
+ test ! -r d.run
+ test ! -r d.log
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+ : Run the tests for the first time.
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ ls -l
+ # All the test scripts should have run.
+ test -f a.run
+ test -f b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
+
+ : Let us make b.test pass.
+ echo OK > b.ok
+ do_recheck --fail
+ # a.test has been successful the first time, so no need to re-run it.
+ # Similar considerations apply to similar checks, below.
+ test ! -r a.run
+ test -f b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
+
+ : Let us make the first part of c.test pass.
+ echo OK > c.pass
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
+
+ : Let us make also the second part of c.test pass.
+ echo KO > c.xfail
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
+
+ : Nothing changed, so only d.test should be run.
+ for i in 1 2; do
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
+ done
+
+ : Let us make d.test run more testcases, and experience _more_ failures.
+ echo 'test_count=9' > d.count
+ unindent > d.extra <<'END'
+ echo 'ok # SKIP s'
+ echo 'not ok - f 1'
+ echo 'ok - p 1'
+ echo 'not ok - f 2'
+ echo 'ok # TODO xp'
+ echo 'not ok - f 3'
+ echo 'not ok - f 4'
+ echo 'ok - p 2'
+ echo 'ok' # Extra test.
+ echo 'Bail out!'
+END
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
+
+ : Let us finally make d.test pass.
+ echo 'test_count=1' > d.count
+ echo : > d.extra
+ do_recheck --pass
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
+
+ : All tests have been successful or skipped, nothing should be re-run.
+ do_recheck --pass
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test ! -r d.run
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/tap-setup.sh b/tests/tap-setup.sh
new file mode 100755
index 0000000..0d1a1e3
--- /dev/null
+++ b/tests/tap-setup.sh
@@ -0,0 +1,48 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Auxiliary shell script that copies the common data and files used by
+# many tests on TAP support into the current directory. It should be
+# sourced by client test scripts, and assumes the auxiliary test
+# `tap-common-setup.test' has been run beforehand (it will error out
+# if this is not the case).
+
+# Check that we are running from a proper directory: last thing we want
+# is to overwrite some random user files.
+test -f ../defs-static && test -f ../defs \
+ || fatal_ "running from a wrong directory"
+
+test ! -f Makefile.am || mv Makefile.am Makefile.am~ \
+ || fatal_ "failed to save Makefile.am"
+
+test -d ../tap-common-setup.dir && cp -Rp ../tap-common-setup.dir/* . \
+ || fatal_ "couldn't get precomputed data files"
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+if test -f Makefile.am~; then
+ mv -f Makefile.am~ Makefile.am \
+ || fatal_ "failed to restore Makefile.am"
+ echo 'TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver' >> Makefile.am \
+ || fatal_ "failed to update Makefile.am"
+ $AUTOMAKE Makefile \
+ || fatal_ "failed to remake Makefile.in"
+ ./config.status Makefile \
+ || fatal_ "failed to remake Makefile"
+fi
+
+:
diff --git a/tests/tap-signal.test b/tests/tap-signal.test
new file mode 100755
index 0000000..fb75c83
--- /dev/null
+++ b/tests/tap-signal.test
@@ -0,0 +1,49 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - a test script terminated by a signal causes an hard error
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+echo TESTS = > Makefile.am
+for sig in 1 2 13 15; do
+ unindent > signal-$sig.test <<END
+ #!/bin/sh
+ echo 1..1
+ echo ok 1
+ kill -$sig \$\$
+ echo "Bail out! \$0 not killed?"
+END
+ echo TESTS += signal-$sig.test >> Makefile.am
+done
+
+chmod a+x *.test
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+for append in '' 'TEST_LOG_DRIVER_FLAGS = --ignore-exit'; do
+ echo "$append" >> Makefile
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=8 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=4
+ for sig in 1 2 13 15; do
+ grep "^ERROR: signal-$sig\\.test - terminated by signal.*" stdout
+ done
+done
+
+:
diff --git a/tests/tap-skip-whole-whitespace.test
b/tests/tap-skip-whole-whitespace.test
new file mode 100755
index 0000000..bf83241
--- /dev/null
+++ b/tests/tap-skip-whole-whitespace.test
@@ -0,0 +1,48 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - normalization of whitespace in console testsuite progress associated
+# with a SKIP directive in the TAP plan
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > foo.test <<END
+1..0${tab}${tab} #${tab}SKIP ${tab}Strip leading & trailing ${tab}${tab}
+END
+
+cat > bar.test <<END
+1..0 #SKIP Preserve ${tab} middle${tab}${tab}space
+END
+
+cat > baz.test <<END
+1..0 # SKIP${tab} Strip${tab}external preserve ${tab}middle ${tab}${sp}
+END
+
+TESTS='foo.test bar.test baz.test' $MAKE -e check > stdout \
+ || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=0 fail=0 error=0 xpass=0 xfail=0 skip=3
+
+grep "SKIP: foo\\.test - Strip leading & trailing$" stdout
+grep "SKIP: bar\\.test - Preserve ${tab} middle${tab}${tab}space" stdout
+grep "SKIP: baz\\.test - Strip${tab}external preserve ${tab}middle" stdout
+
+:
diff --git a/tests/tap-skip-whole.test b/tests/tap-skip-whole.test
new file mode 100755
index 0000000..6b14be4
--- /dev/null
+++ b/tests/tap-skip-whole.test
@@ -0,0 +1,83 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP test protocol support:
+# - special plan format to skip all the tests in a script
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+weirdchars=\''"$!&()[]<>#;^?*/@%=,.:'
+
+cat > foo.test <<END
+1..0
+not ok 1
+not ok 2
+END
+
+cat > bar.test <<END
+1..0$tab $tab
+ok 1
+Bail out! some random failure
+END
+
+# It is undefined whether the comment after the plan below should
+# count as an explanation; still, the test should be skipped.
+cat > baz.test <<END
+1..0 # WWW::Mechanize not installed
+ok 1
+END
+
+cat > wget.test <<END
+1..0 # SKIP wget(1) not installed
+not ok 1 # TODO
+ok 2
+ok 3 # SKIP
+END
+
+cat > curl.test <<END
+1..0 # skip: Can't connect to gnu.org!
+not ok 1
+ok 2 # TODO
+END
+
+cat > mu.test <<END
+1..0 # Skip $weirdchars
+# Various TAP errors in here shouldn't be seen, as the test is skipped.
+1..2
+ok 1
+ok 22
+not ok 333
+1..7
+END
+
+env TESTS='foo.test bar.test baz.test wget.test curl.test mu.test' \
+ $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=6 pass=0 fail=0 xpass=0 xfail=0 skip=6 error=0
+
+grep '^SKIP: foo\.test$' stdout
+grep '^SKIP: bar\.test$' stdout
+grep '^SKIP: baz\.test' stdout # Deliberately laxer, see above for why.
+grep '^SKIP: wget\.test .* wget(1) not installed$' stdout
+grep '^SKIP: curl\.test .* Can'\''t connect to gnu\.org!$' stdout
+grep '^SKIP: mu\.test' stdout | $FGREP "$weirdchars" stdout
+test `grep -c ': .*\.test' stdout` -eq 6
+
+:
diff --git a/tests/tap-summary-aux.sh b/tests/tap-summary-aux.sh
new file mode 100755
index 0000000..a5cad53
--- /dev/null
+++ b/tests/tap-summary-aux.sh
@@ -0,0 +1,361 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Auxiliary script for tests on TAP support: checking testsuite summary.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+br='============================================================================'
+
+case $use_colors in
+ yes|no) ;;
+ *) fatal_ "invalid \$use_colors value '$use_colors'"
+esac
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat > configure.in <<END
+AC_INIT([GNU AutoTAP], [5.12], address@hidden)
+AM_INIT_AUTOMAKE([parallel-tests])
+AC_SUBST([PERL], ['$PERL'])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TEST_LOG_COMPILER = cat
+TESTS = all.test
+END
+
+# The following shell variables are influential for this function:
+# - expect_failure
+# - use_colors
+do_check ()
+{
+ case $#,$1 in
+ 1,--pass) expect_failure=no;;
+ 1,--fail) expect_failure=yes;;
+ *) fatal_ "invalid usage of 'do_check'";;
+ esac
+ shift
+ cat > summary.exp
+ cat all.test
+ st=0
+ if test $use_colors = yes; then
+ make_cmd="env AM_COLOR_TESTS=always $MAKE -e"
+ else
+ make_cmd=$MAKE
+ fi
+ $make_cmd check > stdout || st=$?
+ cat stdout
+ if test $expect_failure = yes; then
+ test $st -gt 0 || Exit 1
+ else
+ test $st -eq 0 || Exit 1
+ fi
+ $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
+ || fatal_ "cannot extract testsuite summary"
+ cat summary.exp
+ cat summary.got
+ if test $use_colors = yes; then
+ # Use cmp, not diff, because the files might contain binary data.
+ compare=cmp
+ else
+ compare=diff
+ fi
+ $compare summary.exp summary.got || Exit 1
+}
+
+if test $use_colors = yes; then
+ red='[0;31m'
+ grn='[0;32m'
+ lgn='[1;32m'
+ blu='[1;34m'
+ mgn='[0;35m'
+ brg='[1m'
+ std='[m'
+ echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
+else
+ red= grn= lgn= blu= mgn= brg= std=
+fi
+
+ success_header="\
+${grn}${br}${std}
+${grn}Testsuite summary for GNU AutoTAP 5.12${std}
+${grn}${br}${std}"
+
+ success_footer=${grn}${br}${std}
+
+ failure_header="\
+${red}${br}${std}
+${red}Testsuite summary for GNU AutoTAP 5.12${std}
+${red}${br}${std}"
+
+ failure_footer="\
+${red}${br}${std}
+${red}See ./test-suite.log${std}
+${red}Please report to address@hidden
+${red}${br}${std}"
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+./configure
+
+# 1 pass.
+{ echo 1..1 && echo ok; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 skip.
+{ echo 1..1 && echo 'ok # SKIP'; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 xfail.
+{ echo 1..1 && echo 'not ok # TODO'; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 fail.
+{ echo 1..1 && echo not ok; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 xpass.
+{ echo 1..1 && echo 'ok # TODO'; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+# 1 hard error.
+{ echo 1..1 && echo 'Bail out!'; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# 3 non-failing results.
+cat > all.test <<END
+1..3
+ok
+not ok # TODO
+ok # SKIP
+END
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 pass, 1 skip, 1 fail.
+cat > all.test <<END
+1..3
+ok
+ok # SKIP
+not ok
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 pass, 1 xfail, 1 xpass.
+cat > all.test <<END
+1..3
+ok
+ok # TODO
+not ok # TODO
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+# 1 skip, 1 xfail, 1 error.
+cat > all.test <<END
+1..3
+ok # SKIP
+not ok # TODO
+Bail out!
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# 1 of each kind
+cat > all.test <<END
+1..6
+ok
+not ok
+ok # TODO
+not ok # TODO
+ok # SKIP
+Bail out!
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 6${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+${red}# FAIL: 1${std}
+${red}# XPASS: 1${std}
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# Prepare some common data for later.
+for i in 0 1 2 3 4 5 6 7 8 9; do
+ for j in 0 1 2 3 4 5 6 7 8 9; do
+ echo "ok"
+ echo "not ok # TODO"
+ echo "ok # SKIP"
+ done
+done > tap
+
+# Lots of non-failures (300 per kind).
+(cat tap && cat tap && cat tap) > all.test
+test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
+echo 1..900 >> all.test # Test plan.
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 900${std}
+${grn}# PASS: 300${std}
+${blu}# SKIP: 300${std}
+${lgn}# XFAIL: 300${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 failure and lots of non-failures means failure.
+(cat tap && echo "not ok" && cat tap) > all.test
+test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
+echo 1..601 >> all.test # Test plan.
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 601${std}
+${grn}# PASS: 200${std}
+${blu}# SKIP: 200${std}
+${lgn}# XFAIL: 200${std}
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 error and lots of non-failures means failure.
+(cat tap && sed 30q tap && echo 'Bail out!') > all.test
+test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
+echo 1..331 >> all.test # Test plan.
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 331${std}
+${grn}# PASS: 110${std}
+${blu}# SKIP: 110${std}
+${lgn}# XFAIL: 110${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+:
diff --git a/tests/tap-summary-color.test b/tests/tap-summary-color.test
new file mode 100755
index 0000000..2dacb54
--- /dev/null
+++ b/tests/tap-summary-color.test
@@ -0,0 +1,23 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - colorized testsuite summary
+
+. ./defs-static
+
+use_colors=yes
+. "$testsrcdir"/tap-summary-aux.sh
diff --git a/tests/tap-summary.test b/tests/tap-summary.test
new file mode 100755
index 0000000..b49aba7
--- /dev/null
+++ b/tests/tap-summary.test
@@ -0,0 +1,23 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - colorized testsuite summary
+
+. ./defs-static
+
+use_colors=no
+. "$testsrcdir"/tap-summary-aux.sh
diff --git a/tests/tap-todo-skip-together.test
b/tests/tap-todo-skip-together.test
new file mode 100755
index 0000000..14b8ed5
--- /dev/null
+++ b/tests/tap-todo-skip-together.test
@@ -0,0 +1,39 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - TODO and SKIP directives on the same line: the first one wins
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+1..2
+ok 1 # SKIP TODO
+not ok 2 # TODO SKIP
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+grep '^SKIP: all\.test 1 ' stdout
+grep '^XFAIL: all\.test 2 ' stdout
+
+:
diff --git a/tests/tap-todo-skip-whitespace.test
b/tests/tap-todo-skip-whitespace.test
new file mode 100755
index 0000000..695e98c
--- /dev/null
+++ b/tests/tap-todo-skip-whitespace.test
@@ -0,0 +1,100 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - normalization of whitespace in console testsuite progress associated
+# with TODO and SKIP directives
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > stub.tap <<END
+1 # TODO
+2 # TODO foo?
+3 # TODO: bar!
+4 aa # TODO
+5 bb # TODO fnord 5
+6 cc # TODO:${tab}fnord 6
+7 - x # TODO
+8 - y # TODO fnord $tab 8
+9 - z # TODO: fnord 9 $tab
+10# TODO x0
+11$tab# TODO x1
+12 $tab$tab # TODO x2
+13 asd# TODO x3
+14 sad$tab# TODO x4
+15 das$tab$tab # TODO x5
+END
+
+cat > stub.exp <<END
+1 # TODO
+2 # TODO foo?
+3 # TODO: bar!
+4 aa # TODO
+5 bb # TODO fnord 5
+6 cc # TODO:${tab}fnord 6
+7 - x # TODO
+8 - y # TODO fnord $tab 8
+9 - z # TODO: fnord 9
+10 # TODO x0
+11 # TODO x1
+12 # TODO x2
+13 asd # TODO x3
+14 sad # TODO x4
+15 das # TODO x5
+END
+
+plan=1..15
+
+my_make_check ()
+{
+ xpass=0 xfail=0 skip=0
+ case $1 in
+ xpass|xfail|skip) eval $1=15;;
+ *) fatal_ "bad argument '$1' for my_make_check";;
+ esac
+ cat all.test
+ $MAKE check >stdout || : # Don't care about the exit status in this test.
+ cat stdout
+ count_test_results total=15 pass=0 fail=0 error=0 \
+ xpass=$xpass xfail=$xfail skip=$skip
+ # Don't be too strict w.r.t. possible normalization of "TODO: foo" into
+ # "TODO : foo" (as is done by, e.g., the 'TAP::Parser' perl module).
+ LC_ALL=C grep '^[A-Z][A-Z]*:' stdout \
+ | sed -e 's/# \(TODO\|SKIP\) *:/# \1:/' > got
+ cat exp
+ cat got
+ diff exp got
+}
+
+# For "TODO" directives leading to XPASS results.
+(echo $plan && sed -e 's/^/ok /' stub.tap) > all.test
+sed -e 's/^/XPASS: all.test /' stub.exp > exp
+my_make_check xpass
+
+# For "TODO" directives leading to XFAIL results.
+(echo $plan && sed -e 's/^/not ok /' stub.tap) > all.test
+sed -e 's/^/XFAIL: all.test /' stub.exp > exp
+my_make_check xfail
+
+# For "SKIP" directives.
+(echo $plan && sed -e 's/^/ok /' -e 's/TODO/SKIP/' stub.tap) > all.test
+sed -e 's/TODO/SKIP/' -e 's/^/SKIP: all.test /' stub.exp > exp
+my_make_check skip
+
+:
diff --git a/tests/tap-todo-skip.test b/tests/tap-todo-skip.test
new file mode 100755
index 0000000..d00dddd
--- /dev/null
+++ b/tests/tap-todo-skip.test
@@ -0,0 +1,91 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - TODO and SKIP directives are case-insensitive
+# - TODO and SKIP directives can be followed optionally by a colon ":"
+# and by an optional explanation.
+# - our driver isn't fooled into recognizing TODO and SKIP directives
+# spuriously
+# - the reasons for TODO and SKIP, if present, are nicely printed in
+# the testsuite progress output
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# ----------------------------------------------------- #
+# Check all possible combinations of: #
+# - uppercase/lowercase #
+# - with/without colon character ":" #
+# - with/without explanatory message #
+# in TODO and SKIP directives. #
+# ----------------------------------------------------- #
+
+# There are 2 * 2^6 + 2 * 2^6 = 256 tests.
+echo 1..256 > all.test
+
+# These nested loops are clearer without indentation.
+
+for c1 in t T; do
+for c2 in o O; do
+for c3 in d D; do
+for c4 in o O; do
+for ex in '' ':' ' foo' ': foo'; do
+ echo "not ok # $c1$c2$c3$c4$ex"
+ echo "not ok# $c1$c2$c3$c4$ex"
+done; done; done; done; done >> all.test
+
+for c1 in s S; do
+for c2 in k K; do
+for c3 in i I; do
+for c4 in p P; do
+for ex in '' ':' ' foo' ': foo'; do
+ echo "ok # $c1$c2$c3$c4$ex"
+ echo "ok# $c1$c2$c3$c4$ex"
+done; done; done; done; done >> all.test
+
+cat all.test # For debugging.
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=256 pass=0 fail=0 xpass=0 xfail=128 skip=128 error=0
+
+# -------------------------------------------------------- #
+# TODO ans SKIP directives aren't recognized spuriously. #
+# -------------------------------------------------------- #
+
+cat > all.test <<'END'
+1..9
+ok TODO
+ok - TODO
+ok 3 TODO
+ok 4 - TODO
+ok SKIP
+ok - SKIP
+ok 7 SKIP
+ok 8 - SKIP
+ok 9
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=9 pass=9 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+:
diff --git a/tests/tap-unplanned.test b/tests/tap-unplanned.test
new file mode 100755
index 0000000..177e36e
--- /dev/null
+++ b/tests/tap-unplanned.test
@@ -0,0 +1,159 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - unplanned tests are properly reported as errors
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+1..1
+ok 1
+ok 2
+END
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=3 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=2
+grep '^ERROR: all\.test - too many tests run (expected 1, got 2)$' stdout
+grep '^ERROR: all\.test 2 # UNPLANNED$' stdout
+
+cat > all.test <<END
+1..2
+ok 1
+ok 2
+ok 3
+END
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=2
+grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout
+grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
+
+# Interaction with XFAIL_TESTS.
+cat > all.test <<END
+1..2
+not ok 1
+ok 2 # SKIP
+ok 3
+not ok 4
+ok 5 # SKIP
+END
+XFAIL_TESTS=all.test $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=6 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=4
+grep '^ERROR: all\.test - too many tests run (expected 2, got 5)$' stdout
+grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
+grep '^ERROR: all\.test 4 # UNPLANNED$' stdout
+grep '^ERROR: all\.test 5 # UNPLANNED$' stdout
+
+cat > all.test <<END
+1..1
+ok 1
+
+ok
+ok 3
+ok foo
+ok 5 - bar bar
+
+not ok
+not ok 7
+not ok foo
+not ok 9 - bar bar
+
+ok # TODO
+ok 11 # TODO
+ok foo # TODO
+ok 13 - bar bar # TODO
+
+not ok # TODO
+not ok 15 # TODO
+not ok foo # TODO
+not ok 17 - bar bar # TODO
+
+ok # SKIP
+ok 19 # SKIP
+ok foo # SKIP
+ok 21 - bar bar # SKIP
+END
+
+cat > t <<END
+
+2
+3
+4 foo
+5 - bar bar
+
+6
+7
+8 foo
+9 - bar bar
+
+10
+11
+12 foo
+13 - bar bar
+
+14
+15
+16 foo
+17 - bar bar
+
+18
+19
+20 foo
+21 - bar bar
+
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=22 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=21
+
+echo 'PASS: all.test 1' > exp
+sed -e '/^$/d' -e 's/.*/ERROR: all.test & # UNPLANNED/' t >> exp
+echo 'ERROR: all.test - too many tests run (expected 1, got 21)' >> exp
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+# Note that, if the TAP input has a trailing plan, it is not possible
+# to flag unplanned tests as such, since we do not know they're unplanned
+# until the plan is reached; still, we should give at least an error
+# message about the unmatched number of tests once we've got the plan.
+
+for x in 'ok' 'ok 3' 'not ok' 'not ok # TODO' 'ok # TODO' 'ok # SKIP'; do
+ unindent > all.test <<END
+ ok 1
+ ok 2 # SKIP
+ $x
+ 1..2
+END
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ test `$FGREP -c ': all.test' stdout` -eq 4
+ $EGREP '^PASS: all\.test 1($| )' stdout
+ $EGREP '^SKIP: all\.test 2($| )' stdout
+ $EGREP ': all\.test 3($| )' stdout
+ grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout
+done
+
+:
diff --git a/tests/tap-whitespace-normalization.test
b/tests/tap-whitespace-normalization.test
new file mode 100755
index 0000000..a902438
--- /dev/null
+++ b/tests/tap-whitespace-normalization.test
@@ -0,0 +1,165 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: whitespace normalization (or lack thereof) in the testsuite
+# progress output on console. We keep all these checks here in a single
+# script so that a potential cosmetic change in the output format won't
+# force us to tweak dozens of other tests (hopefully).
+# See also related test 'tap-todo-skip-whitespace.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_COMPILER = cat
+TESTS =
+END
+
+: > exp
+
+spaces_a=${sp}${tab}${tab}${sp}${sp}${tab}
+spaces_b=${tab}${tab}${sp}${tab}${sp}${sp}${sp}
+
+#-----------------------------------------------------------------------
+
+echo TESTS += numbers.test >> Makefile.am
+
+cat > numbers.test <<END
+1..6
+ok${spaces_a}1
+ok 2${spaces_b}
+ok${spaces_a}3${spaces_b}
+not ok${spaces_b}4
+not ok 5${spaces_a}
+not ok${spaces_b}6${spaces_a}
+END
+
+cat >> exp <<END
+PASS: numbers.test 1
+PASS: numbers.test 2
+PASS: numbers.test 3
+FAIL: numbers.test 4
+FAIL: numbers.test 5
+FAIL: numbers.test 6
+END
+
+#-----------------------------------------------------------------------
+
+echo TESTS += description.test >> Makefile.am
+
+cat > description.test <<END
+1..8
+ok${spaces_a}+foo
+ok +bar${spaces_b}
+ok${spaces_a}+baz${spaces_b}
+not ok${spaces_b}-foo
+not ok -bar${spaces_a}
+not ok${spaces_b}-baz${spaces_a}
+ok u${spaces_b}v${spaces_a}w${sp}
+not ok${spaces_a}x${spaces_a}y${tab}z${tab}
+END
+
+cat >> exp <<END
+PASS: description.test 1 +foo
+PASS: description.test 2 +bar
+PASS: description.test 3 +baz
+FAIL: description.test 4 -foo
+FAIL: description.test 5 -bar
+FAIL: description.test 6 -baz
+PASS: description.test 7 u${spaces_b}v${spaces_a}w
+FAIL: description.test 8 x${spaces_a}y${tab}z
+END
+
+#-----------------------------------------------------------------------
+
+# "Bail out!" magic.
+
+echo TESTS += bailout.test >> Makefile.am
+
+cat > bailout.test <<END
+1..1
+Bail out!${tab}${sp}${sp}${tab}We're out of disk space.
+ok 1
+END
+
+cat >> exp <<END
+ERROR: bailout.test - Bail out! We're out of disk space.
+END
+
+echo TESTS += bailout2.test >> Makefile.am
+
+cat > bailout2.test <<END
+1..1
+Bail out!foo${tab}${sp}
+ok 1
+END
+
+cat >> exp <<END
+ERROR: bailout2.test - Bail out! foo
+END
+
+#-----------------------------------------------------------------------
+
+# Diagnostic lines.
+
+echo AM_TEST_LOG_DRIVER_FLAGS = --comments >> Makefile.am
+echo TESTS += cmnt.test >> Makefile.am
+
+cat > cmnt.test <<END
+1..1
+ok 1
+#Leading whitespace gets added
+# ${tab}${tab} ${tab}Extra leading whitespace is stripped
+# Trailing whitespace is stripped ${tab} ${tab}${tab}
+# Middle${tab}whitespace is${tab} ${tab}${tab} kept
+# ${tab} And only${tab}middle ${tab}whitespace ${tab}${tab} ${tab}
+END
+
+cat >> exp <<END
+PASS: cmnt.test 1
+# cmnt.test: Leading whitespace gets added
+# cmnt.test: Extra leading whitespace is stripped
+# cmnt.test: Trailing whitespace is stripped
+# cmnt.test: Middle${tab}whitespace is${tab} ${tab}${tab} kept
+# cmnt.test: And only${tab}middle ${tab}whitespace
+END
+
+#-----------------------------------------------------------------------
+
+# TODO: we should have more checks here ... (but let's not over-do FTM).
+
+#-----------------------------------------------------------------------
+
+chmod a+x *.test
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Don't care about exit status or number of test results, they should be
+# checked for in many other tests.
+$MAKE check >stdout || :
+cat stdout
+
+LC_ALL=C sort exp > t
+mv -f t exp
+
+# We need the sort below to account for parallel make usage.
+LC_ALL=C grep '[a-z0-9][a-z0-9]*\.test' stdout | LC_ALL=C sort > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-with-and-without-number.test
b/tests/tap-with-and-without-number.test
new file mode 100755
index 0000000..e989cc8
--- /dev/null
+++ b/tests/tap-with-and-without-number.test
@@ -0,0 +1,50 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - numbered and unnumbered test results can coexist
+# - tests without explicit number get automatically numbered in the
+# testsuite progress output on console
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..7
+ok 1 foo
+ok 2 - foo2
+ok - bar
+not ok baz # TODO
+not ok 5 - asd # TODO
+ok 6 # SKIP
+ok zardoz
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=7 pass=4 fail=0 xpass=0 xfail=2 skip=1 error=0
+
+grep '^PASS: all\.test 1 foo$' stdout
+grep '^PASS: all\.test 2 - foo2$' stdout
+grep '^PASS: all\.test 3 - bar$' stdout
+grep '^XFAIL: all\.test 4 baz # TODO$' stdout
+grep '^XFAIL: all\.test 5 - asd # TODO$' stdout
+grep '^SKIP: all\.test 6 # SKIP$' stdout
+grep '^PASS: all\.test 7 zardoz$' stdout
+
+:
diff --git a/tests/tap-xfail-tests.test b/tests/tap-xfail-tests.test
new file mode 100755
index 0000000..22733c8
--- /dev/null
+++ b/tests/tap-xfail-tests.test
@@ -0,0 +1,68 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP and $(XFAIL_TESTS): test results without directives are turned from
+# PASS to XPASS and from FAIL to XFAIL; other results are unchanged.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo 'XFAIL_TESTS = $(TESTS)' >> Makefile
+
+cat > all.test <<END
+1..6
+ok 1
+not ok 2
+ok 3 # TODO
+not ok 4 # TODO
+ok 5 # SKIP
+# The next should count as a failure (and thus will be
+# rendered into an expected failure by XFAIL_TESTS).
+not ok 6 # SKIP
+Bail out!
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=7 pass=0 fail=0 xpass=2 xfail=3 skip=1 error=1
+
+grep '^XPASS: all\.test 1$' stdout
+grep '^XFAIL: all\.test 2$' stdout
+grep '^XPASS: all\.test 3 # TODO' stdout
+grep '^XFAIL: all\.test 4 # TODO' stdout
+grep '^SKIP: all\.test 5 # SKIP' stdout
+grep '^XFAIL: all\.test 6 # SKIP' stdout
+grep '^ERROR: all\.test - Bail out!' stdout
+
+# Check that the exit status of the testsuite is indeed 0 when we
+# would expect success.
+
+cat > all.test <<END
+1..3
+not ok 1
+ok 2 # SKIP
+not ok 3 # TODO
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=0 fail=0 xpass=0 xfail=2 skip=1 error=0
+
+:
diff --git a/tests/test-driver-create-log-dir.test
b/tests/test-driver-create-log-dir.test
index 5713b92..c554793 100755
--- a/tests/test-driver-create-log-dir.test
+++ b/tests/test-driver-create-log-dir.test
@@ -40,14 +40,18 @@ check-local: $(TEST_SUITE_LOG)
test -f sub/foo.log
test -f sub/subsub/bar.log
test -f sub1/baz.log
+ test -f sub/foo.trs
+ test -f sub/subsub/bar.trs
+ test -f sub1/baz.trs
END
cat > checkdir-driver <<'END'
#! /bin/sh
-set -e
+set -e; set -u
while test $# -gt 0; do
case $1 in
--log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
--test-name|--expect-failure|--color-tests|--enable-hard-errors) shift;;
--) shift; break;;
*) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
@@ -55,11 +59,16 @@ while test $# -gt 0; do
shift
done
echo "log: $log_file" # For debugging.
+echo "trs: $trs_file" # Ditto.
case $log_file in */*);; *) exit 1;; esac
-dir=`expr "$log_file" : '\(.*\)/[^/]*'`
-echo "dir: $dir" # For debugging.
-test -d "$dir" || exit 1
-echo dummy > "$log_file"
+dir_log=`expr "$log_file" : '\(.*\)/[^/]*'`
+dir_trs=`expr "$trs_file" : '\(.*\)/[^/]*'`
+echo "dir_log: $dir_log" # For debugging.
+echo "dir_trs: $dir_trs" # Likewise.
+test x"$dir_trs" = x"$dir_log" || exit 1
+test -d "$dir_log" || exit 1
+echo dummy1 > "$log_file"
+echo dummy2 > "$trs_file"
END
chmod a+x checkdir-driver
diff --git a/tests/test-driver-custom-multitest-recheck.test
b/tests/test-driver-custom-multitest-recheck.test
index e0f010f..8bbcc7c 100755
--- a/tests/test-driver-custom-multitest-recheck.test
+++ b/tests/test-driver-custom-multitest-recheck.test
@@ -20,6 +20,7 @@
# support, but also serves as a "usability test" for our APIs.
# See also related tests `test-driver-custom-multitest-recheck2.test'
# and `parallel-tests-recheck-override.test'.
+# Keep in sync with 'tap-recheck.test'.
parallel_tests=yes
. ./defs || Exit 1
@@ -49,7 +50,7 @@ echo PASS:
if test -f b.ok; then
echo PASS:
else
- echo FAIL:
+ echo ERROR:
fi
: > b.run
END
@@ -95,18 +96,6 @@ do_recheck ()
cat stdout; ls -l
}
-do_count ()
-{
- pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR
- eval "$@"
- $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP)' stdout || : # For debugging.
- test `grep -c '^PASS:' stdout` -eq $pass
- test `grep -c '^FAIL:' stdout` -eq $fail
- test `grep -c '^XPASS:' stdout` -eq $xpass
- test `grep -c '^XFAIL:' stdout` -eq $xfail
- test `grep -c '^SKIP:' stdout` -eq $skip
-}
-
for vpath in : false; do
if $vpath; then
mkdir build
@@ -129,7 +118,7 @@ for vpath in : false; do
test ! -r c.log
test ! -r d.run
test ! -r d.log
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
: Run the tests for the first time.
$MAKE check >stdout && { cat stdout; Exit 1; }
@@ -140,7 +129,7 @@ for vpath in : false; do
test -f b.run
test -f c.run
test -f d.run
- do_count pass=3 fail=3 xpass=1 xfail=1 skip=1
+ count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
: Let us make b.test pass.
echo OK > b.ok
@@ -151,7 +140,7 @@ for vpath in : false; do
test -f b.run
test -f c.run
test -f d.run
- do_count pass=2 fail=2 xpass=1 xfail=1 skip=1
+ count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
: Let us make the first part of c.test pass.
echo OK > c.pass
@@ -160,7 +149,7 @@ for vpath in : false; do
test ! -r b.run
test -f c.run
test -f d.run
- do_count pass=1 fail=1 xpass=1 xfail=1 skip=1
+ count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
: Let us make also the second part of c.test pass.
echo KO > c.xfail
@@ -169,7 +158,7 @@ for vpath in : false; do
test ! -r b.run
test -f c.run
test -f d.run
- do_count pass=1 fail=1 xpass=0 xfail=2 skip=1
+ count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
: Nothing changed, so only d.test should be run.
for i in 1 2; do
@@ -178,7 +167,7 @@ for vpath in : false; do
test ! -r b.run
test ! -r c.run
test -f d.run
- do_count pass=0 fail=1 xpass=0 xfail=0 skip=1
+ count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
done
: Let us make d.test run more testcases, and experience _more_ failures.
@@ -190,14 +179,16 @@ for vpath in : false; do
echo XPASS: xp
echo FAIL: f 3
echo FAIL: f 4
+ echo ERROR: e 1
echo PASS: p 2
+ echo ERROR: e 2
END
do_recheck --fail
test ! -r a.run
test ! -r b.run
test ! -r c.run
test -f d.run
- do_count pass=2 fail=4 xpass=1 xfail=0 skip=2
+ count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
: Let us finally make d.test pass.
echo : > d.extra
@@ -206,7 +197,7 @@ END
test ! -r b.run
test ! -r c.run
test -f d.run
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=1
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
: All tests have been successful or skipped, nothing should be re-run.
do_recheck --pass
@@ -214,7 +205,7 @@ END
test ! -r b.run
test ! -r c.run
test ! -r d.run
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
cd $srcdir
diff --git a/tests/test-driver-custom-multitest-recheck2.test
b/tests/test-driver-custom-multitest-recheck2.test
index a18bdd8..0d254dc 100755
--- a/tests/test-driver-custom-multitest-recheck2.test
+++ b/tests/test-driver-custom-multitest-recheck2.test
@@ -56,7 +56,13 @@ END
cat > c.test << 'END'
#! /bin/sh
-echo XPASS: xp
+if test -f c.err; then
+ echo ERROR: xxx
+elif test -f c.ok; then
+ echo PASS: ok
+else
+ echo XPASS: xp
+fi
: > c.run
END
@@ -66,18 +72,6 @@ $ACLOCAL
$AUTOCONF
$AUTOMAKE
-do_count ()
-{
- pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR
- eval "$@"
- $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP)' stdout || : # For debugging.
- test `grep -c '^PASS:' stdout` -eq $pass
- test `grep -c '^FAIL:' stdout` -eq $fail
- test `grep -c '^XPASS:' stdout` -eq $xpass
- test `grep -c '^XFAIL:' stdout` -eq $xfail
- test `grep -c '^SKIP:' stdout` -eq $skip
-}
-
for vpath in : false; do
if $vpath; then
mkdir build
@@ -96,7 +90,7 @@ for vpath in : false; do
test -f a.run
test -f b.run
test -f c.run
- do_count pass=2 fail=1 xpass=1 xfail=0 skip=1
+ count_test_results total=5 pass=2 fail=1 xpass=1 xfail=0 skip=1 error=0
rm -f *.run
@@ -104,7 +98,7 @@ for vpath in : false; do
for var in TESTS TEST_LOGS; do
env "$var=" $MAKE -e recheck >stdout || { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
@@ -115,7 +109,7 @@ for vpath in : false; do
env TESTS=a.test $MAKE -e recheck >stdout \
|| { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
@@ -128,7 +122,7 @@ for vpath in : false; do
test ! -r a.run
test -f b.run
test ! -r c.run
- do_count pass=0 fail=0 xpass=0 xfail=1 skip=1
+ count_test_results total=2 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=0
rm -f *.run
@@ -136,35 +130,57 @@ for vpath in : false; do
TEST_LOGS=b.log $MAKE -e recheck >stdout \
|| { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
TESTS='a.test b.test' $MAKE -e recheck >stdout \
|| { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=0 xfail=0 skip=0
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test ! -r c.run
- # An XPASS should count a failure.
+ : No need to re-run a.test anymore, but c.test should be rerun,
+ : as it contained an XPASS. And this time, make it fail with
+ : an hard error.
+ # Use `echo' here, since Solaris 10 /bin/sh would try to optimize
+ # a `:' away after the first iteration, even if it is redirected.
+ echo dummy > c.err
env TEST_LOGS='a.log c.log' $MAKE -e recheck >stdout \
&& { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=1 xfail=0 skip=0
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
test ! -r a.run
test ! -r b.run
test -f c.run
- rm -f *.run
- env TESTS='c.test b.test' $MAKE -e recheck >stdout \
- && { cat stdout; Exit 1; }
+
+ rm -f *.run *.err
+
+ : c.test contained and hard error the last time, so it should be re-run.
+ : This time, make it pass
+ # Use `echo', not `:'; see comments above for why.
+ echo dummy > c.ok
+ env TESTS='c.test a.test' $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
cat stdout
- do_count pass=0 fail=0 xpass=1 xfail=0 skip=0
+ count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
test ! -r a.run
test ! -r b.run
test -f c.run
+ rm -f *.run *.err *.ok
+
+ : Nothing should be rerun anymore, as all tests have been eventually
+ : succesful.
+ $MAKE recheck >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+
cd $srcdir
done
diff --git a/tests/test-driver-custom-multitest.test
b/tests/test-driver-custom-multitest.test
index a1c1010..4fb023c 100755
--- a/tests/test-driver-custom-multitest.test
+++ b/tests/test-driver-custom-multitest.test
@@ -40,15 +40,9 @@ TESTS = \
pass-fail.t \
pass4-skip.t \
pass3-skip2-xfail.t \
- pass-xpass-fail-xfail-skip.t
+ pass-xpass-fail-xfail-skip-error.t
END
-expected_pass=10
-expected_fail=5
-expected_skip=4
-expected_xfail=2
-expected_xpass=1
-
cat > pass.t << 'END'
echo %% pass %%
echo PASS: pass
@@ -99,13 +93,14 @@ mv || echo "XFAIL: mv cannot read users' mind yet"
exit 127
END
-cat > pass-xpass-fail-xfail-skip.t << 'END'
+cat > pass-xpass-fail-xfail-skip-error.t << 'END'
echo PASS:
echo FAIL:
echo XFAIL:
echo XPASS:
echo SKIP:
-echo %% pass-xpass-fail-xfail-skip %%
+echo ERROR:
+echo %% pass-xpass-fail-xfail-skip-error %%
END
chmod a+x *.t
@@ -132,6 +127,7 @@ for vpath in : false; do
# `trivial-test-driver' script is changed.
$FGREP INVALID.NAME stdout test-suite.log && Exit 1
test -f BAD.LOG && Exit 1
+ test -f BAD.TRS && Exit 1
# These log files must all have been created by the testsuite.
cat pass.log
cat fail.log
@@ -139,27 +135,17 @@ for vpath in : false; do
cat pass-fail.log
cat pass4-skip.log
cat pass3-skip2-xfail.log
- cat pass-xpass-fail-xfail-skip.log
- # For debugging.
- $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP)' stdout
-
- test `grep -c '^PASS:' stdout` -eq $expected_pass
- test `grep -c '^FAIL:' stdout` -eq $expected_fail
- test `grep -c '^XPASS:' stdout` -eq $expected_xpass
- test `grep -c '^XFAIL:' stdout` -eq $expected_xfail
- test `grep -c '^SKIP:' stdout` -eq $expected_skip
-
- grep '^PASS: pass-xpass-fail-xfail-skip.t\, testcase 1' stdout
- grep '^FAIL: pass-xpass-fail-xfail-skip\.t, testcase 2' stdout
- grep '^XFAIL: pass-xpass-fail-xfail-skip\.t, testcase 3' stdout
- grep '^XPASS: pass-xpass-fail-xfail-skip\.t, testcase 4' stdout
- grep '^SKIP: pass-xpass-fail-xfail-skip\.t, testcase 5' stdout
-
- # Check testsuite summary printed on console.
- sed -e 's/[()]/ /g' -e 's/^/ /' stdout > t
- grep ' 6 of 18 ' t
- grep ' 1 unexpected pass' t
- grep ' 4 test.* not run' t
+ cat pass-xpass-fail-xfail-skip-error.log
+
+ count_test_results total=23 pass=10 fail=5 skip=4 xfail=2 xpass=1 error=1
+
+ tst=pass-xpass-fail-xfail-skip-error
+ grep "^PASS: $tst\.t, testcase 1" stdout
+ grep "^FAIL: $tst\.t, testcase 2" stdout
+ grep "^XFAIL: $tst\.t, testcase 3" stdout
+ grep "^XPASS: $tst\.t, testcase 4" stdout
+ grep "^SKIP: $tst\.t, testcase 5" stdout
+ grep "^ERROR: $tst\.t, testcase 6" stdout
# Check that the content of, and only of, the test logs with at least
# one failing test case has been copied into `test-suite.log'. Note
@@ -170,19 +156,14 @@ for vpath in : false; do
grep '%% fail %%' test-suite.log
grep '%% fail2 %%' test-suite.log
grep '%% pass-fail %%' test-suite.log
- grep '%% pass-xpass-fail-xfail-skip %%' test-suite.log
+ grep '%% pass-xpass-fail-xfail-skip-error %%' test-suite.log
test `grep -c '%% ' test-suite.log` -eq 4
TESTS='pass.t pass3-skip2-xfail.t' $MAKE -e check >stdout \
|| { cat stdout; cat test-suite.log; Exit 1; }
cat test-suite.log
cat stdout
- # For debugging.
- $EGREP '(PASS|FAIL|XPASS|XFAIL|SKIP)' stdout
- test `grep -c '^PASS:' stdout` -eq 4
- test `grep -c '^SKIP:' stdout` -eq 2
- test `grep -c '^XFAIL:' stdout` -eq 1
- $EGREP '^(FAIL|XPASS)' stdout && Exit 1
+ count_test_results total=7 pass=4 fail=0 skip=2 xfail=1 xpass=0 error=0
cd $srcdir
diff --git a/tests/test-driver-custom-no-html.test
b/tests/test-driver-custom-no-html.test
index 9aceae6..2c86026 100755
--- a/tests/test-driver-custom-no-html.test
+++ b/tests/test-driver-custom-no-html.test
@@ -35,17 +35,17 @@ END
cat > no-rst <<'END'
#! /bin/sh
+echo ':test-result: SKIP' > foo.trs
+echo ':copy-in-global-log: yes' >> foo.trs
# The genereted log file is deliberately syntactically invalid
# reStructuredText.
-cat > foo.log <<'EoL'
+cat > foo.log <<'EOF'
SKIP: FooBar
=============
-:test-result: SKIP
-
--------------
dummy title
-EoL
+EOF
END
chmod a+x no-rst
@@ -62,6 +62,6 @@ $FGREP 'dummy title' test-suite.log
# Sanity check: trying to produce HTML output should fail.
$MAKE check-html >output 2>&1 && { cat output; Exit 1; }
cat output
-$EGREP 'SEVERE|ERROR' output
+$FGREP SEVERE output
:
diff --git a/tests/test-driver-custom-xfail-tests.test
b/tests/test-driver-custom-xfail-tests.test
index 4f8bb83..e68e04b 100755
--- a/tests/test-driver-custom-xfail-tests.test
+++ b/tests/test-driver-custom-xfail-tests.test
@@ -88,15 +88,17 @@ done
cat > td <<'END'
#! /bin/sh
-set -e
+set -e; set -u
test_name=INVALID
log_file=/dev/null
+trs_file=/dev/null
expect_failure=no
while test $# -gt 0; do
case $1 in
--test-name) test_name=$2; shift;;
--expect-failure) expect_failure=$2; shift;;
--log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
# Ignored.
--color-tests) shift;;
--enable-hard-errors) shift;;
@@ -109,23 +111,22 @@ while test $# -gt 0; do
done
st=0
"$@" || st=$?
-rm -f "$log_file"
case $st,$expect_failure in
0,no)
echo "PASS: $test_name" | tee "$log_file"
- echo ":test-result: PASS" >> "$log_file"
+ echo ":test-result: PASS" > "$trs_file"
;;
1,no)
echo "FAIL: $test_name" | tee "$log_file"
- echo ":test-result: FAIL" >> "$log_file"
+ echo ":test-result: FAIL" > "$trs_file"
;;
0,yes)
echo "XPASS: $test_name" | tee "$log_file"
- echo ":test-result: XPASS" >> "$log_file"
+ echo ":test-result: XPASS" > "$trs_file"
;;
1,yes)
echo "XFAIL: $test_name" | tee "$log_file"
- echo ":test-result: XFAIL" >> "$log_file"
+ echo ":test-result: XFAIL" > "$trs_file"
;;
*)
echo "INTERNAL ERROR" >&2
@@ -141,21 +142,24 @@ $AUTOMAKE
./configure
-VERBOSE=yes $MAKE check
-test -f test-suite.log
-test -f sub1/test-suite.log
-test -f sub2/test-suite.log
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+test `grep -c '^PASS:' stdout` -eq 3
+test `grep -c '^XFAIL:' stdout` -eq 13
for dir in sub1 sub2; do
cd $dir
cp pass.test x1.test
cp x2.test pass.test
- $MAKE check && { cat test-suite.log; Exit 1; }
- cat test-suite.log
- grep '^FAIL: pass\.test$' test-suite.log
- grep '^XPASS: x1\.test$' test-suite.log
- test `grep -c '^FAIL' test-suite.log` -eq 1
- test `grep -c '^XPASS' test-suite.log` -eq 1
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ test "`cat pass.trs`" = ":test-result: FAIL"
+ test "`cat x1.trs`" = ":test-result: XPASS"
+ test "`cat x2.trs`" = ":test-result: XFAIL"
+ grep '^FAIL: pass\.test$' stdout
+ grep '^XPASS: x1\.test$' stdout
+ grep '^XFAIL: x2\.test$' stdout
+ count_test_results total=7 pass=0 xpass=1 fail=1 xfail=5 skip=0 error=0
cd ..
done
diff --git a/tests/test-driver-custom.test b/tests/test-driver-custom.test
index b6a33d2..8305047 100755
--- a/tests/test-driver-custom.test
+++ b/tests/test-driver-custom.test
@@ -59,11 +59,13 @@ log_wflags='@log_wflags@'
test_name=INVALID
log_file=BAD.log
+trs_file=BAD.trs
extra_opts=
while test $# -gt 0; do
case $1 in
--test-name) test_name=$2; shift;;
--log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
# Ignored.
--expect-failure) shift;;
--color-tests) shift;;
@@ -79,6 +81,7 @@ while test $# -gt 0; do
done
echo "$me" "$test_name" $extra_opts > "$log_file"
+: > "$trs_file"
exec "$@"
exit 127
@@ -123,6 +126,7 @@ VERBOSE=yes $MAKE check
ls -l . sub
test ! -r BAD.log
+test ! -r BAD.trs
echo 'chk-wrapper 1.chk --am-chk --chk' > 1.exp
echo 'test-wrapper 2.test -am-test -test' > 2.exp
diff --git a/tests/test-driver-global-log.test
b/tests/test-driver-global-log.test
deleted file mode 100755
index cdaaff6..0000000
--- a/tests/test-driver-global-log.test
+++ /dev/null
@@ -1,110 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Check that the output of testcases having a global outcome that is not
-# "PASS" or "XFAIL" is copied in the global 'test-suite.log'.
-
-parallel_tests=yes
-. ./defs || Exit 1
-
-cat >> configure.in << 'END'
-AC_OUTPUT
-END
-
-# The :test-result: fields should be irrelevant for the decision
-# of whether a test output is to be copied in the test-suite.log.
-cat > pass.test <<END
-PASS: pass.test
-:test-result: SKIP
-:test-result: FAIL
-:test-result: XFAIL
-:test-result: XPASS
-:test-result: ERROR
-not seen (pass)
-END
-
-cat > xfail.test <<'END'
-XFAIL: xfail.test
-not seen (xfail)
-END
-
-echo SKIP: skip.test > skip.test
-echo FAIL: fail.test > fail.test
-echo XPASS: xpass.test > xpass.test
-echo :test-result: PASS > fake-pass.test
-echo "$tab $tab$tab" > empty.test
-
-cat > Makefile.am << 'END'
-TEST_LOG_DRIVER = ./dummy-driver
-TESTS = pass.test skip.test fail.test xfail.test xpass.test \
- fake-pass.test empty.test
-END
-
-
-cat > dummy-driver <<'END'
-#!/bin/sh
-set -e
-while test $# -gt 0; do
- case $1 in
- --log-file) log_file=$2; shift;;
- --test-name) test_name=$2; shift;;
- --expect-failure|--color-tests|--enable-hard-errors) shift;;
- --) shift; break;;
- *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
- esac
- shift
-done
-cp $1 $log_file
-END
-chmod a+x dummy-driver
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE
-
-./configure
-
-$MAKE check && { cat test-suite.log; Exit 1; }
-cat test-suite.log
-grep '^SKIP: skip\.test$' test-suite.log
-grep '^FAIL: fail.test$' test-suite.log
-grep '^XPASS: xpass.test$' test-suite.log
-grep '^:test-result: PASS$' test-suite.log
-grep "^$tab $tab$tab$" test-suite.log
-$EGREP 'not seen' test-suite.log && Exit 1
-test `grep -c ':test-result:' test-suite.log` -eq 1
-
-cat > pass.test <<END
-PASS: pass.test
-:test-result: SKIP
-:test-result: XFAIL
-not seen (pass)
-END
-
-cat > xfail.test <<END
-XFAIL: xfail.test
-:test-result: SKIP
-:test-result: PASS
-not seen (xfail)
-END
-
-TESTS='pass.test skip.test xfail.test' $MAKE -e check \
- || { cat test-suite.log; Exit 1; }
-cat test-suite.log
-grep '^SKIP: skip\.test$' test-suite.log
-$EGREP ':test-result:|not seen' test-suite.log && Exit 1
-
-:
diff --git a/tests/test-driver-recheck.test b/tests/test-driver-recheck.test
deleted file mode 100755
index 2ae75b9..0000000
--- a/tests/test-driver-recheck.test
+++ /dev/null
@@ -1,126 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test the "make recheck" semantics for custom test driver, as
-# documented in detail in the Automake manual.
-
-parallel_tests=yes
-. ./defs || Exit 1
-
-cat >> configure.in << 'END'
-AC_OUTPUT
-END
-
-cat > Makefile.am << 'END'
-LOG_DRIVER = ./dummy-driver
-TEST_EXTENSIONS =
-TESTS =
-END
-
-rechecked=''
-notrechecked=''
-
-for R in PASS XFAIL SKIP; do
- echo $R: > $R-1
- echo " $R:" > $R-2
- echo $R:foo > $R-3
- echo "$tab $R: bar baz" > $R-4
- # The :test-result: fields should be irrelevant for the decision of
- # whether "make recheck" should or should not re-run a test.
- unindent > $R-5 <<END
- $R:
- :test-result: FAIL
- :test-result: XPASS
- :test-result: ERROR
-END
- notrechecked="$notrechecked $R-1 $R-2 $R-3 $R-4 $R-5"
- echo $R > BAD-$R-1
- echo $R. > BAD-$R-2
- echo :$R: > BAD-$R-3
- echo $R asd > BAD-$R-4
- rechecked="$rechecked BAD-$R-1 BAD-$R-2 BAD-$R-3 BAD-$R-4"
-done
-
-for R in FAIL XPASS UNKNOWN; do
- echo $R: > $R-1
- echo $R:foo > $R-2
- echo $R: bar baz > $R-3
- echo $R > $R-4
- echo $R asd > $R-5
- # The :test-result: fields should be irrelevant for the decision of
- # whether "make recheck" should or should not re-run a test.
- unindent > $R-6 <<END
- :test-result: PASS
- :test-result: SKIP
- :test-result: XFAIL
-END
- (echo $R: && cat $R-6) > $R-7
- rechecked="$rechecked $R-1 $R-2 $R-3 $R-4 $R-5 $R-6 $R-7"
-done
-
-: > EMPTY
-echo " $tab $tab" > WHITE
-rechecked="$rechecked EMPTY WHITE"
-
-tests="$rechecked $notrechecked"
-
-for t in $tests; do echo $t; done | sed 's/.*/TESTS += &/' >> Makefile.am
-
-cat Makefile.am # For debugging.
-
-cat > dummy-driver <<'END'
-#!/bin/sh
-set -e
-while test $# -gt 0; do
- case $1 in
- --log-file) log_file=$2; shift;;
- --test-name) test_name=$2; shift;;
- --expect-failure|--color-tests|--enable-hard-errors) shift;;
- --) shift; break;;
- *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
- esac
- shift
-done
-: > $test_name.run
-cp $1 $log_file
-END
-chmod a+x dummy-driver
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE
-
-./configure
-
-# The ':test-result:' fields should be ignored by "make recheck",
-# but should cause the testsuite report to detect errors.
-$MAKE check && Exit 1
-ls -l
-for t in $tests; do test -f $t.run; done
-rm -f *.run
-
-# But now the tests that actually get re-run have only ':test-result:'
-# fields indicating success, so "make recheck" must pass. Still, the
-# next "make recheck" call should still re-run the same set of tests.
-for iteration in 1 2; do
- $MAKE recheck
- ls -l
- for t in $rechecked; do test -f $t.run; done
- for t in $notrechecked; do test ! -r $t.run; done
- rm -f *.run
-done
-
-:
diff --git a/tests/test-driver-strip-vpath.test
b/tests/test-driver-strip-vpath.test
index f1b316c..5b21cc0 100755
--- a/tests/test-driver-strip-vpath.test
+++ b/tests/test-driver-strip-vpath.test
@@ -51,6 +51,7 @@ set -e
while test $# -gt 0; do
case $1 in
--log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
--test-name) test_name=$2; shift;;
--expect-failure|--color-tests|--enable-hard-errors) shift;;
--) shift; break;;
@@ -64,6 +65,7 @@ case $test_name in
*) exit 1;;
esac
echo dummy > "$log_file"
+echo dummy > "$trs_file"
END
chmod a+x checkstrip-driver
diff --git a/tests/test-driver-trs-suffix-registered.test
b/tests/test-driver-trs-suffix-registered.test
new file mode 100755
index 0000000..7f6522a
--- /dev/null
+++ b/tests/test-driver-trs-suffix-registered.test
@@ -0,0 +1,58 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests support: the following are registered with `.SUFFIXES':
+# - .log
+# - .trs (used by files that store test results and metadata)
+# - .test if $(TEST_EXTENSIONS) is not defined
+# - stuff in $(TEST_EXTENSIONS) otherwise
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+: > Makefile.am
+
+cat > 1.am << 'END'
+TESTS =
+END
+
+cat > 2.am << 'END'
+TEST_EXTENSIONS = .SH .abcdef
+TESTS =
+END
+
+: > test-driver
+
+$ACLOCAL
+
+$AUTOMAKE 1
+$AUTOMAKE 2
+
+sed -e 's/$/ /' 1.in > mk.1
+sed -e 's/$/ /' 2.in > mk.2
+
+grep '^\.SUFFIXES:' mk.1
+grep '^\.SUFFIXES:' mk.2
+
+for suf in test log trs; do
+ grep "^\\.SUFFIXES:.* \\.$suf " mk.1
+done
+
+for suf in SH abcdef log trs; do
+ grep "^\\.SUFFIXES:.* \\.$suf " mk.2
+done
+
+:
diff --git a/tests/test-log.test b/tests/test-log.test
new file mode 100755
index 0000000..1385386
--- /dev/null
+++ b/tests/test-log.test
@@ -0,0 +1,168 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - log file creation
+# - log file removal
+# - stdout and stderr of a test script go in its log file
+# - TEST_SUITE_LOG redefinition, at either automake or make time
+# - VERBOSE environment variable support
+# Keep in sync with 'tap-log.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<END
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = pass.test skip.test xfail.test fail.test xpass.test error.test
+XFAIL_TESTS = xpass.test xfail.test
+TEST_SUITE_LOG = global.log
+END
+
+# Custom markers, for use in grepping checks.
+cmarker=::: # comment marker
+pmarker=%%% # plain maker
+
+cat > pass.test <<END
+#! /bin/sh
+echo "$pmarker pass $pmarker" >&2
+echo "# $cmarker pass $cmarker" >&2
+exit 0
+END
+
+cat > skip.test <<END
+#! /bin/sh
+echo "$pmarker skip $pmarker"
+echo "# $cmarker skip $cmarker"
+exit 1
+END
+
+cat > xfail.test <<END
+#! /bin/sh
+echo "$pmarker xfail $pmarker" >&2
+echo "# $cmarker xfail $cmarker" >&2
+exit 1
+END
+
+cat > fail.test <<END
+#! /bin/sh
+echo 1..1
+echo "$pmarker fail $pmarker"
+echo "# $cmarker fail $cmarker"
+exit 1
+END
+
+cat > xpass.test <<END
+#! /bin/sh
+echo "$pmarker xpass $pmarker" >&2
+echo "# $cmarker xpass $cmarker" >&2
+exit 0
+END
+
+cat > error.test <<END
+#! /bin/sh
+echo "$pmarker error $pmarker"
+echo "# $cmarker error $cmarker"
+exit 99
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+TEST_SUITE_LOG=my.log $MAKE -e check && Exit 1
+ls -l # For debugging.
+test ! -f test-suite.log
+test ! -f global.log
+test -f my.log
+st=0
+for result in pass fail xfail xpass skip error; do
+ cat $result.log # For debugging.
+ $FGREP "$pmarker $result $pmarker" $result.log || st=1
+ $FGREP "$cmarker $result $cmarker" $result.log || st=1
+done
+test $st -eq 0 || Exit 1
+cat my.log # For debugging.
+for result in xfail fail xpass skip error; do
+ cat $result.log # For debugging.
+ $FGREP "$pmarker $result $pmarker" my.log || st=1
+ $FGREP "$cmarker $result $cmarker" my.log || st=1
+done
+test `$FGREP -c "$pmarker" my.log` -eq 5
+test `$FGREP -c "$cmarker" my.log` -eq 5
+
+# Passed test scripts shouldn't be mentioned in the global log.
+$EGREP '(^pass|[^x]pass)\.test' my.log && Exit 1
+# But failing (expectedly or not) and skipped ones should.
+$FGREP 'xfail.test' my.log
+$FGREP 'skip.test' my.log
+$FGREP 'fail.test' my.log
+$FGREP 'xpass.test' my.log
+$FGREP 'error.test' my.log
+
+touch error2.log test-suite.log global.log
+TEST_SUITE_LOG=my.log $MAKE -e mostlyclean
+ls -l # For debugging.
+test ! -f my.log
+test ! -f pass.log
+test ! -f fail.log
+test ! -f xfail.log
+test ! -f xpass.log
+test ! -f skip.log
+test ! -f error.log
+# "make mostlyclean" shouldn't remove unrelated log files.
+test -f error2.log
+test -f test-suite.log
+test -f global.log
+
+rm -f *.log
+
+VERBOSE=yes $MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+cat global.log
+test ! -f my.log
+test ! -f test-suite.log
+# Check that VERBOSE causes the global testsuite log to be
+# emitted on stdout.
+out=`cat stdout`
+log=`cat global.log`
+case $out in *"$log"*) ;; *) Exit 1;; esac
+
+touch error2.log test-suite.log my.log
+$MAKE clean
+ls -l # For debugging.
+test ! -f global.log
+test ! -f pass.log
+test ! -f fail.log
+test ! -f xfail.log
+test ! -f xpass.log
+test ! -f skip.log
+test ! -f error.log
+# "make clean" shouldn't remove unrelated log files.
+test -f error2.log
+test -f test-suite.log
+test -f my.log
+
+rm -f *.log
+
+:
diff --git a/tests/test-metadata-global-log.test
b/tests/test-metadata-global-log.test
new file mode 100755
index 0000000..a6f0999
--- /dev/null
+++ b/tests/test-metadata-global-log.test
@@ -0,0 +1,147 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers and parallel-tests harness: check the documented
+# semantics for deciding when the content of a test log file should be
+# copied in the global test-suite.log file. Currently, this is done
+# with the use of the reStructuredText field `:copy-in-global-log:' in
+# the associated `.trs' files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = ./passthrough-driver
+TEST_LOG_COMPILER = $(SHELL) -e
+END
+
+cat > passthrough-driver <<'END'
+#!/bin/sh
+set -e; set -u;
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+echo "$test_name: RUN"
+"$@" >$log_file 2>&1 5>$trs_file
+END
+chmod a+x passthrough-driver
+
+# The `:test-result:' and `:recheck:' fields and the first line of the
+# log file should be be irrelevant for the decision of whether a test
+# output is to be copied in the `test-suite.log'.
+
+cat > no-1.test <<END
+echo :test-result: SKIP >&5
+echo :copy-in-global-log: no >&5
+echo :test-result: FAIL >&5
+echo :test-result: XPASS >&5
+echo not seen 1
+END
+
+# In the last line, with leading and trailing whitespace in the value.
+cat > no-2.test <<END
+echo ":test-result: FAIL" >&5
+echo "not seen 2"
+echo ":recheck: yes" >&5
+echo ":copy-in-global-log:$tab $tab no $tab" >&5
+END
+
+for RES in XPASS FAIL XFAIL SKIP ERROR UNKNOWN; do
+ unindent > $RES.test <<END
+ echo :test-result: $RES >&5
+ echo :copy-in-global-log: no >&5
+ echo not seen $RES
+END
+done
+
+# In the first line, with no whitespace.
+cat > no-3.test <<END
+echo :copy-in-global-log:no >&5
+echo ":test-result: FAIL" >&5
+echo "not seen 3"
+END
+
+# Leading whitespace before the field.
+cat > no-4.test <<END
+echo ":test-result: FAIL" >&5
+echo " $tab $tab$tab :copy-in-global-log: no" >&5
+echo "not seen 4"
+END
+
+cat > yes-1.test <<END
+echo :test-result: PASS >&5
+echo :copy-in-global-log: yes >&5
+echo seen yes 1
+END
+
+# A lacking `:copy-in-global-log:' implies that the content of
+# the log file should be copied.
+cat > yes-2.test <<END
+echo :test-result: PASS >&5
+echo seen yes 2
+END
+
+# Three corner cases.
+
+cat > corn-1.test <<END
+echo seen corn 1
+echo ':copy-in-global-log:' >&5
+END
+
+cat > corn-2.test <<END
+echo seen corn 2
+echo '$tab $tab$tab' >&5
+END
+
+cat > corn-3.test <<'END'
+echo seen corn 31
+echo ':copy-in-global-log:address@hidden' >&5
+echo seen corn 32
+END
+
+echo TESTS = *.test >> Makefile.am
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# We don't care about the exit status of "make check" here, that
+# should be checked in other tests.
+$MAKE check || :
+cat test-suite.log
+grep '^seen yes 1$' test-suite.log
+grep '^seen yes 2$' test-suite.log
+grep '^seen corn 1$' test-suite.log
+grep '^seen corn 2$' test-suite.log
+grep '^seen corn 31$' test-suite.log
+grep '^seen corn 32$' test-suite.log
+$FGREP 'not seen' test-suite.log && Exit 1
+
+:
diff --git a/tests/test-metadata-recheck.test b/tests/test-metadata-recheck.test
new file mode 100755
index 0000000..6a372f3
--- /dev/null
+++ b/tests/test-metadata-recheck.test
@@ -0,0 +1,163 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test the "make recheck" semantics for custom test drivers, as documented
+# in the Automake manual.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+LOG_DRIVER = ./dummy-driver
+TEST_EXTENSIONS =
+TESTS =
+END
+
+#
+# Tests to be re-run by "make recheck"
+#
+
+: > y-1
+echo "foo bar" > y-2
+echo ":recheck:" > y-3
+echo ":recheck:yes" > y-4
+echo ":recheck: who cares" > y-5
+echo ":recheck: $tab y" > y-6
+echo ":recheck: yeah!$tab$tab " > y-7
+cat > y-10 <<END
+:foo:
+:recheck: ???
+END
+cat > y-11 <<END
+:recheck: YES
+:foo:
+END
+cat > y-12 <<END
+foo
+:recheck:yes
+
+bar
+zardoz
+END
+
+echo " $tab $tab$tab :recheck: yes" > y-8
+
+# The :test-result: fields and the fist line of the log should be
+# irrelevant for the decision of whether "make recheck" should or
+# should not re-run a test.
+
+echo ":test-result: PASS" > y-100
+
+echo "PASS: y-101"
+
+cat > y-102 <<END
+PASS: y-102
+===========
+
+:test-result: PASS
+END
+
+#
+# Tests *not* to be re-run by "make recheck"
+#
+
+echo ":recheck:no" > n-1
+echo ":recheck: no " > n-2
+echo ":recheck: $tab no" > n-3
+echo ":recheck: no $tab$tab " > n-4
+cat > n-5 <<END
+:foo:
+:recheck:no
+END
+cat > n-6 <<END
+:recheck: no
+:foo:
+END
+cat > n-7 <<END
+foo
+:recheck: no$tab$tab
+
+bar
+zardoz
+END
+
+echo " $tab $tab$tab :recheck: no" > n-8
+
+# The :test-result: fields should be irrelevant for the decision of
+# whether "make recheck" should or should not re-run a test.
+cat > n-100 <<END
+:test-result: FAIL
+:test-result: XPASS
+:test-result: ERROR
+:test-result: UNKNOWN
+:recheck: no
+END
+
+rechecked=`echo y-[0-9]*`
+
+for t in [yn]-[0-9]*; do echo $t; done \
+ | sed 's/.*/TESTS += &/' >> Makefile.am
+
+cat Makefile.am # For debugging.
+
+cat > dummy-driver <<'END'
+#!/bin/sh
+set -e; set -u
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+: > $test_name.run
+: > $log_file
+cp $1 $trs_file
+END
+chmod a+x dummy-driver
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# The ':test-result:' fields should be ignored by "make recheck",
+# but should cause the testsuite report to detect errors.
+$MAKE check && Exit 1
+ls -l
+for t in $tests; do test -f $t.run; done
+rm -f *.run
+
+# But now the tests that actually get re-run have only ':test-result:'
+# fields indicating success, so "make recheck" must pass. Still, the
+# next "make recheck" call should still re-run the same set of tests.
+for iteration in 1 2; do
+ $MAKE recheck
+ ls -l
+ for t in $rechecked; do test -f $t.run; done
+ find . -name 'n-*.run' | grep . && Exit 1
+done
+
+:
diff --git a/tests/test-metadata-results.test b/tests/test-metadata-results.test
new file mode 100755
index 0000000..effa488
--- /dev/null
+++ b/tests/test-metadata-results.test
@@ -0,0 +1,169 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Parallel testsuite harness: check APIs for the registering of test
+# results in `*.trs' files, as documented in the automake manual.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = ./dummy-driver
+TESTS = foo.test bar.test
+END
+
+cat > dummy-driver <<'END'
+#! /bin/sh
+set -e; set -u
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+: > $log_file
+cp $1 $trs_file
+END
+chmod a+x dummy-driver
+
+mk_check ()
+{
+ st=0
+ $MAKE check >stdout || st=$?
+ cat stdout
+ # Our dummy driver make no testsuite progress report.
+ grep ': .*\.test' stdout && Exit 1
+ # Nor it writes to the log files.
+ test -s foo.log && Exit 1
+ test -s bar.log && Exit 1
+ return $st
+}
+
+# This must be different from the one defined in `test/defs', as that
+# assumes that the driver does proper testsuite progress reporting.
+count_test_results ()
+{
+ total=ERR pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
+ eval "$@"
+ st=0
+ grep "^# TOTAL: *$total$" stdout || rc=1
+ grep "^# PASS: *$pass$" stdout || rc=1
+ grep "^# XFAIL: *$xfail$" stdout || rc=1
+ grep "^# SKIP: *$skip$" stdout || rc=1
+ grep "^# FAIL: *$fail$" stdout || rc=1
+ grep "^# XPASS: *$xpass$" stdout || rc=1
+ grep "^# ERROR: *$error$" stdout || rc=1
+ test $st -eq 0 || Exit 1
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# Basic checks. Also that that "old-style" directives with format
+# "RESULT: test-name" are be ignored now.
+
+cat > foo.test <<END
+FAIL: foo.test
+:test-result: PASS
+END
+echo ERROR: bar.test > bar.test
+mk_check
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+cat > foo.test <<END
+PASS: foo.test
+:test-result: FAIL
+END
+echo SKIP: bar.test > bar.test
+: > bar.test
+mk_check && Exit 1
+count_test_results total=1 pass=0 fail=1 xpass=0 xfail=0 skip=0 error=0
+
+cat > foo.test <<END
+PASS: foo.test
+:test-result: FAIL
+END
+cat > bar.test <<END
+ERROR: foo.test
+:test-result: SKIP
+END
+mk_check && Exit 1
+count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
+
+cat > foo.test <<END
+:test-result: XFAIL
+:test-result: PASS
+:test-result: SKIP
+END
+cat > bar.test <<END
+:test-result: SKIP
+:test-result: PASS
+:test-result: SKIP
+:test-result: PASS
+:test-result: PASS
+END
+mk_check
+count_test_results total=8 pass=4 fail=0 xpass=0 xfail=1 skip=3 error=0
+
+# Check that all results expected to be supported are *really* supported.
+
+cat > foo.test <<END
+:test-result: PASS
+:test-result: SKIP
+:test-result: XFAIL
+:test-result: FAIL
+:test-result: XPASS
+:test-result: ERROR
+END
+: > bar.test
+mk_check && Exit 1
+count_test_results total=6 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=1
+
+cp foo.test bar.test
+mk_check && Exit 1
+count_test_results total=12 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=2
+
+# Check that we are liberal w.r.t. whitespace use.
+
+: > foo.test
+: > bar.test
+for RESULT in PASS FAIL XPASS XFAIL SKIP ERROR; do
+ sed -e 's/^ *//' -e 's/|//g' >> foo.test <<END
+ |:test-result:$RESULT|
+ |:test-result: $tab $RESULT|
+ |:test-result:$RESULT $tab|
+ |:test-result:$tab$tab $RESULT$tab $tab |
+END
+ echo " $tab $tab$tab :test-result: $RESULT" >> bar.test
+done
+cat foo.test # For debugging.
+cat bar.test # Likewise.
+mk_check && Exit 1
+count_test_results total=30 pass=5 fail=5 xpass=5 xfail=5 skip=5 error=5
+
+:
diff --git a/tests/test-missing.test b/tests/test-missing.test
new file mode 100755
index 0000000..a4ac179
--- /dev/null
+++ b/tests/test-missing.test
@@ -0,0 +1,72 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests:
+# - non-existent scripts listed in TESTS get diagnosed
+# See also related test 'test-missing2.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = ok.test zardoz.test
+TEST_LOG_COMPILER = true
+END
+
+: > ok.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check >output 2>&1 && { cat output; Exit 1; }
+cat output
+test -f ok.log
+grep '^PASS: ok\.test' output
+$FGREP 'zardoz.log' output
+test ! -f test-suite.log
+
+TESTS='zardoz2.test' $MAKE -e check >output 2>&1 \
+ && { cat output; Exit 1; }
+cat output
+$FGREP 'zardoz2.log' output
+test ! -f test-suite.log
+
+TEST_LOGS='zardoz3.log' $MAKE -e check >output 2>&1 \
+ && { cat output; Exit 1; }
+cat output
+$FGREP 'zardoz3.log' output
+test ! -f test-suite.log
+
+# The errors should persist even after `test-suite.log'
+# has been created.
+
+: > zardoz.test
+$MAKE check
+rm -f zardoz.test
+
+$MAKE check >output 2>&1 && { cat output; Exit 1; }
+cat output
+$FGREP 'zardoz.log' output
+test ! -f test-suite.log
+
+:
diff --git a/tests/test-missing2.test b/tests/test-missing2.test
new file mode 100755
index 0000000..bc2979a
--- /dev/null
+++ b/tests/test-missing2.test
@@ -0,0 +1,54 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests:
+# - non-existent scripts listed in TESTS get diagnosed, even when
+# all the $(TEST_LOGS) have a dummy dependency.
+# See also related test 'test-missing.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foobar1.test foobar2.test
+$(TEST_LOGS):
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE foobar1.log foobar2.log || Exit 99
+test ! -f foobar1.log || Exit 99
+test ! -f foobar1.trs || Exit 99
+test ! -f foobar2.log || Exit 99
+test ! -f foobar2.trs || Exit 99
+
+$MAKE check >output 2>&1 && { cat output; Exit 1; }
+cat output
+grep 'test-suite\.log.*foobar1\.log' output
+grep 'test-suite\.log.*foobar1\.trs' output
+grep 'test-suite\.log.*foobar2\.log' output
+grep 'test-suite\.log.*foobar2\.trs' output
+test ! -f test-suite.log
+
+:
diff --git a/tests/test-trs-basic.test b/tests/test-trs-basic.test
new file mode 100755
index 0000000..88f1276
--- /dev/null
+++ b/tests/test-trs-basic.test
@@ -0,0 +1,141 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel harness features:
+# - creation and removal of `.trs' auxiliary files
+# - check some internals regarding the use of `.trs' files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<END
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .sh .test
+TESTS = foo.test bar.sh sub/zardoz.test
+TEST_LOG_COMPILER = $(SHELL)
+SH_LOG_COMPILER = $(SHELL)
+
+## Used to check some internal details.
+tb:
+ echo $(am__TEST_BASES) > $@
+END
+
+cat > foo.test << 'END'
+#! /bin/sh
+exit $FOO_STATUS
+END
+: > bar.sh
+mkdir sub
+: > sub/zardoz.test
+
+FOO_STATUS=0; export FOO_STATUS
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+#
+# Check some internal details first.
+#
+
+$MAKE tb
+test x"`cat tb`" = x"foo bar sub/zardoz"
+rm -f tb
+TESTS='foo.test foo2.sh foo3.exe foo4' $MAKE -e tb
+test x"`cat tb`" = x"foo foo2 foo3.exe foo4"
+rm -f tb
+
+#
+# The `test-suite.stamp' file and the `.trs' files get created by
+# "make check" and removed by "make clean" and "make mostlyclean".
+#
+
+: > unrelated.trs
+: > sub/foo.trs
+
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+test -f sub/zardoz.trs
+$MAKE clean
+test ! -f foo.trs
+test ! -f bar.trs
+test ! -f sub/zardoz.trs
+# Unrelated `.trs' files shouldn't be removed.
+test -f unrelated.trs
+test -f sub/foo.trs
+
+# The files should be properly created in case of testsuite failure too.
+FOO_STATUS=1 $MAKE check && Exit 1
+test -f foo.trs
+test -f bar.trs
+test -f sub/zardoz.trs
+$MAKE mostlyclean
+test ! -f foo.trs
+test ! -f bar.trs
+test ! -f sub/zardoz.trs
+# Unrelated `.trs' files shouldn't be removed.
+test -f unrelated.trs
+test -f sub/foo.trs
+
+#
+# Try with a subset of TESTS.
+#
+
+TESTS=foo.test $MAKE -e check
+test -f foo.trs
+test ! -f bar.trs
+test ! -f sub/zardoz.trs
+$MAKE clean
+test ! -f foo.trs
+TESTS='foo.test bar.sh' $MAKE -e check
+test -f foo.trs
+test -f bar.trs
+test ! -f sub/zardoz.trs
+# "make clean" shouldn't remove `.trs' files for tests not in $(TESTS).
+TESTS=bar.sh $MAKE -e clean
+test -f foo.trs
+test ! -f bar.trs
+
+$MAKE clean
+
+#
+# Try with a subset of TEST_LOGS.
+#
+
+TEST_LOGS=sub/zardoz.log $MAKE -e check
+test ! -f foo.trs
+test ! -f bar.trs
+test -f sub/zardoz.trs
+$MAKE clean
+test ! -f sub/zardoz.trs
+TEST_LOGS='foo.log bar.log' $MAKE -e check
+test -f foo.trs
+test -f bar.trs
+test ! -f sub/zardoz.trs
+# "make clean" shouldn't remove `.trs' files for tests whose log
+# is not in $(TEST_LOGS).
+TEST_LOGS=foo.log $MAKE -e clean
+test ! -f foo.trs
+test -f bar.trs
+test ! -f sub/zardoz.trs
+
+:
diff --git a/tests/test-trs-recover.test b/tests/test-trs-recover.test
new file mode 100755
index 0000000..9893424
--- /dev/null
+++ b/tests/test-trs-recover.test
@@ -0,0 +1,167 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel harness features:
+# - recovery from deleted `.trs' files, in various scenarios
+# This test is complex and tricky, but that's acceptable since we are
+# testing semantics that are potentially complex and tricky.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<END
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test baz.test
+TEST_LOG_COMPILER = $(SHELL)
+END
+
+# FIXME: creative quoting below to please maintainer-check.
+echo exit '$TEST_STATUS' > foo.test
+echo exit '$TEST_STATUS' > bar.test
+: > baz.test
+
+TEST_STATUS=0; export TEST_STATUS
+
+# Slower and possible overkill in some situations, but also clearer
+# and safer.
+update_stamp () { $sleep && touch stamp && $sleep; }
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+: Create the required log files.
+$MAKE check
+
+: Recreate by hand.
+rm -f foo.trs bar.trs baz.trs
+$MAKE foo.trs
+test -f foo.trs
+test ! -f bar.trs
+test ! -f baz.trs
+
+: Recreate by hand, several at the same time.
+rm -f foo.trs bar.trs baz.trs
+$MAKE foo.trs bar.trs
+test -f foo.trs
+test -f bar.trs
+test ! -f baz.trs
+
+: Recreate by hand, with a failing test.
+rm -f foo.trs bar.trs
+TEST_STATUS=1 $MAKE bar.trs baz.trs >stdout || { cat stdout; Exit 1; }
+cat stdout
+test ! -f foo.trs
+test -f bar.trs
+test -f baz.trs
+grep '^FAIL: bar\.test' stdout
+$EGREP '^(baz|foo)\.test' stdout && Exit 1
+
+: Recreate with a sweeping "make check", and ensure that also up-to-date
+: '.trs' files are remade.
+update_stamp
+rm -f foo.trs bar.trs
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+is_newest baz.trs stamp
+
+: Recreate with a sweeping "make check" with failing tests. Again,
+: ensure that also up-to-date '.trs' files are remade -- this time we
+: grep the "make check" output verify that.
+rm -f foo.trs bar.trs
+TEST_STATUS=1 $MAKE check >stdout && { cat stdout; Exit 1; }
+test -f foo.trs
+test -f bar.trs
+grep '^FAIL: foo\.test' stdout
+grep '^FAIL: bar\.test' stdout
+grep '^PASS: baz\.test' stdout
+
+: Recreate with a "make check" with redefined TESTS.
+rm -f foo.trs bar.trs baz.trs
+TESTS=foo.test $MAKE -e check
+test -f foo.trs
+test ! -f bar.trs
+test ! -f baz.trs
+
+: Recreate with a "make check" with redefined TEST_LOGS.
+rm -f foo.trs bar.trs baz.trs
+TEST_LOGS=bar.log $MAKE -e check
+test ! -f foo.trs
+test -f bar.trs
+test ! -f baz.trs
+
+: Interactions with "make recheck" are OK.
+rm -f foo.trs bar.trs baz.log baz.trs
+$MAKE recheck >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -f foo.trs
+test -f bar.trs
+test ! -f baz.trs
+test ! -f baz.log
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+grep 'baz\.test' stdout && Exit 1
+count_test_results total=2 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+: Setup for the next check.
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+test -f baz.trs
+
+: Recreate by remaking the global test log, and ensure that up-to-date
+: '.trs' files are *not* remade.
+update_stamp
+rm -f foo.trs bar.trs test-suite.log
+$MAKE test-suite.log >stdout || { cat stdout; Exit 1; }
+cat stdout
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+grep 'baz\.test' stdout && Exit 1
+stat *.trs *.log stamp || : # For debugging.
+# Check that make has updated what it needed to, but no more.
+test -f foo.trs
+test -f bar.trs
+is_newest stamp baz.trs
+is_newest test-suite.log foo.trs bar.trs
+
+: Setup for the next check.
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+test -f baz.trs
+
+: Interactions with lazy test reruns are OK.
+rm -f foo.trs
+update_stamp
+touch bar.test
+RECHECK_LOGS= $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+# Check that make has updated what it needed to, but no more.
+test -f foo.trs
+is_newest bar.trs bar.test
+is_newest stamp baz.trs
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+grep 'baz\.test' stdout && Exit 1
+
+:
diff --git a/tests/test-trs-recover2.test b/tests/test-trs-recover2.test
new file mode 100755
index 0000000..bc94b3b
--- /dev/null
+++ b/tests/test-trs-recover2.test
@@ -0,0 +1,133 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel harness features:
+# - recovery from unreadable `.trs' files, in various scenarios
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<END
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test
+TEST_LOG_COMPILER = true
+END
+
+: > foo.test
+: > bar.test
+
+TEST_STATUS=0; export TEST_STATUS
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+: > t
+chmod a-r t
+test ! -r t || Exit 77
+rm -f t
+
+: Create the required log files.
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+
+: Recreate with a sweeping "make check".
+chmod a-r bar.trs
+$MAKE check
+test -f foo.trs
+test -r foo.trs
+test -f bar.trs
+test -r bar.trs
+
+: Again, this time with one .trs file missing and the other
+: one unreadable.
+rm -f foo.trs
+chmod a-r bar.trs
+$MAKE check
+test -f foo.trs
+test -r foo.trs
+test -f bar.trs
+test -r bar.trs
+
+: Again, but using "make recheck" this time.
+rm -f foo.trs
+chmod a-r bar.trs
+$MAKE recheck >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -f foo.trs
+test -r foo.trs
+test -f bar.trs
+test -r bar.trs
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+
+: More complex interactions with "make recheck" are OK.
+chmod a-r bar.log bar.trs
+$MAKE recheck >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -f bar.trs
+test -r bar.trs
+grep '^PASS: bar\.test' stdout
+grep 'foo\.test' stdout && Exit 1
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+: Recreate by remaking the global test log.
+chmod a-r foo.trs
+rm -f test-suite.log
+$MAKE test-suite.log >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -f foo.trs
+test -r foo.trs
+grep '^PASS: foo\.test' stdout
+grep 'bar\.test' stdout && Exit 1
+# Also test that have only run before should be counted in the final
+# testsuite summary.
+grep '^# TOTAL: *2$' stdout
+
+: Setup for the next check.
+: > baz.test
+sed 's/^TESTS =.*/& baz.test/' Makefile > t
+diff t Makefile && Exit 99
+mv -f t Makefile
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+test -f baz.trs
+
+: Interactions with lazy test reruns are OK.
+chmod a-r foo.trs
+$sleep
+touch stamp
+$sleep
+touch bar.test
+RECHECK_LOGS= $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -r foo.trs
+is_newest bar.trs bar.test
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+grep 'baz\.test' stdout && Exit 1
+# Also test that have only run before should be counted in the final
+# testsuite summary.
+grep '^# TOTAL: *3$' stdout
+
+:
diff --git a/tests/tests-environment-backcompat.test
b/tests/tests-environment-backcompat.test
new file mode 100755
index 0000000..ecf1b16
--- /dev/null
+++ b/tests/tests-environment-backcompat.test
@@ -0,0 +1,61 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# With old serial testsuite driver, TESTS_ENVIRONMENT can be used to
+# define the "test runner", i.e. the program that the test scripts must
+# be run by (with the parallel-tests driver one should use LOG_COMPILER
+# for this). The behaviour tested here is also documented in the manual.
+
+parallel_tests=no
+. ./defs || Exit 1
+
+set -e
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS_ENVIRONMENT = $(PERL) -Mstrict -w
+TESTS = foo.pl bar.pl baz.pl
+XFAIL_TESTS = baz.pl
+EXTRA_DIST = $(TESTS)
+END
+
+echo 'exit (0);' > foo.pl
+echo 'exit (0);' > bar.pl
+
+cat > baz.pl << 'END'
+# With "use strict" enacted, this will cause an error, since the
+# variable `$x' is not declared with `my' nor specified with an
+# explicit package name.
+$x = 0;
+exit ($x);
+END
+
+chmod a+x *.pl
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check
+$MAKE distcheck
+
+:
diff --git a/tests/check-tests_environment.test b/tests/tests-environment.test
similarity index 100%
rename from tests/check-tests_environment.test
rename to tests/tests-environment.test
diff --git a/tests/testsuite-summary-checks.sh
b/tests/testsuite-summary-checks.sh
new file mode 100755
index 0000000..a4de864
--- /dev/null
+++ b/tests/testsuite-summary-checks.sh
@@ -0,0 +1,106 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check the testsuite summary with the parallel-tests harness. This
+# script is meant to be sourced by other test script, so that it can
+# be used to check different scenarios (colorized and non-colorized
+# testsuite output, packages with and without bug-report addresses,
+# testsuites in subdirectories, ...)
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+case $use_colors in
+ yes)
+ AM_COLOR_TESTS=always; export AM_COLOR_TESTS
+ TERM=ansi; export TERM
+ am_opts='parallel-tests color-tests'
+ ;;
+ no)
+ am_opts='parallel-tests'
+ ;;
+ *)
+ fatal_ "invalid use_colors='$use_colors'";;
+esac
+
+cat > configure.in <<END
+AC_INIT([GNU AutoFoo], [7.1], address@hidden)
+AM_INIT_AUTOMAKE([$am_opts])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TEST_EXTENSIONS = .t
+T_LOG_COMPILER = $(SHELL) $(srcdir)/log-compiler
+## Will be defined at runtime.
+TESTS =
+$(TESTS):
+END
+
+cat > log-compiler <<'END'
+#!/bin/sh
+case "$1" in
+ pass*|*/pass*|xpass*|*/xpass*) exit 0;;
+ fail*|*/fail*|xfail*|*/xfail*) exit 1;;
+ skip*|*/skip*) exit 77;;
+ error*|/error*) exit 99;;
+ copy*|*/copy*) cat "$1";;
+ *) exit 99;;
+esac
+END
+
+# Quite complexish, but allow the tests in client scripts to be written
+# in a "data-driven fashion".
+do_check ()
+{
+ cat > summary.exp
+ expect_failure=false
+ xfail_tests=''
+ tests="TESTS='$*'"
+ for t in $*; do
+ case $t in fail*|xpass*|error*) expect_failure=:;; esac
+ case $t in xfail*|xpass*) xfail_tests="$xfail_tests $t";; esac
+ done
+ test -z "$xfail_tests" || xfail_tests="XFAIL_TESTS='$xfail_tests'"
+ st=0
+ eval "env $tests $xfail_tests \$MAKE -e check > stdout || st=\$?"
+ cat stdout
+ if $expect_failure; then
+ test $st -gt 0 || Exit 1
+ else
+ test $st -eq 0 || Exit 1
+ fi
+ $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
+ || fatal_ "cannot extract testsuite summary"
+ cat summary.exp
+ cat summary.got
+ if test $use_colors = yes; then
+ # Use cmp, not diff, because the files might contain binary data.
+ compare=cmp
+ else
+ compare=diff
+ fi
+ $compare summary.exp summary.got || Exit 1
+}
+
+br='============================================================================'
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+:
diff --git a/tests/testsuite-summary-color.test
b/tests/testsuite-summary-color.test
new file mode 100755
index 0000000..9493744
--- /dev/null
+++ b/tests/testsuite-summary-color.test
@@ -0,0 +1,199 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check coloring of the testsuite summary.
+
+. ./defs-static || Exit 1
+
+use_colors=yes
+use_vpath=no
+
+. "$testsrcdir"/testsuite-summary-checks.sh || Exit 99
+
+./configure
+
+# ANSI colors.
+red='[0;31m'
+grn='[0;32m'
+lgn='[1;32m'
+blu='[1;34m'
+mgn='[0;35m'
+brg='[1m';
+std='[m';
+
+success_header="\
+${grn}${br}${std}
+${grn}Testsuite summary for GNU AutoFoo 7.1${std}
+${grn}${br}${std}"
+
+success_footer=${grn}${br}${std}
+
+failure_header="\
+${red}${br}${std}
+${red}Testsuite summary for GNU AutoFoo 7.1${std}
+${red}${br}${std}"
+
+failure_footer="\
+${red}${br}${std}
+${red}See ./test-suite.log${std}
+${red}Please report to address@hidden
+${red}${br}${std}"
+
+do_check '' <<END
+$success_header
+# TOTAL: 0
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check skip.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check xfail.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check fail.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+do_check xpass.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+do_check error.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+do_check pass.t xfail.t skip.t <<END
+$success_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t fail.t skip.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+do_check pass.t xfail.t xpass.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+do_check skip.t xfail.t error.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+do_check pass.t skip.t xfail.t fail.t xpass.t error.t <<END
+$failure_header
+${brg}# TOTAL: 6${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+${red}# FAIL: 1${std}
+${red}# XPASS: 1${std}
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+:
diff --git a/tests/testsuite-summary-count-many.test
b/tests/testsuite-summary-count-many.test
new file mode 100755
index 0000000..b0232c7
--- /dev/null
+++ b/tests/testsuite-summary-count-many.test
@@ -0,0 +1,105 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check test counts in the testsuite summary, with test drivers allowing
+# multiple test results per test script, and for a huge number of tests.
+# Incidentally, this test also checks that the testsuite summary doesn't
+# give any bug-report address if it's not defined.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+for s in trivial-test-driver extract-testsuite-summary; do
+ cp "$testsrcdir/$s" . || fatal_ "failed to fetch auxiliary script $s"
+done
+
+br='============================================================================'
+
+header="\
+${br}
+Testsuite summary for $me 1.0
+${br}"
+
+footer="\
+${br}
+See ./test-suite.log
+${br}"
+
+echo AC_OUTPUT >> configure.in
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+TESTS = all.test
+# Without this, the test driver will be horrendously slow.
+END
+
+cat > all.test <<'END'
+#!/bin/sh
+cat results.txt || { echo ERROR: weird; exit 99; }
+END
+chmod a+x all.test
+
+$PERL -w -e '
+ use warnings FATAL => "all";
+ use strict;
+
+ # FIXME: we would like this to be 1000 or even 10000, but the current
+ # implementation is too slow to handle that :-(
+ my $base = 5;
+ my %count = (
+ TOTAL => $base * 1000,
+ PASS => $base * 700,
+ SKIP => $base * 200,
+ XFAIL => $base * 80,
+ FAIL => $base * 10,
+ XPASS => $base * 7,
+ ERROR => $base * 3,
+ );
+ my @results = qw/PASS SKIP XFAIL FAIL XPASS ERROR/;
+
+ open (RES, ">results.txt") or die "opening results.txt: $!\n";
+ open (CNT, ">count.txt") or die "opening count.txt: $!\n";
+
+ printf CNT "# %-6s %d\n", "TOTAL:", $count{TOTAL};
+ for my $res (@results)
+ {
+ my $uc_res = uc $res;
+ print STDERR "Generating list of $res ...\n";
+ for (1..$count{$res})
+ {
+ print RES "$uc_res: $_\n";
+ }
+ printf CNT "# %-6s %d\n", $res . ":", $count{$res};
+ }
+'
+
+(echo "$header" && cat count.txt && echo "$footer") > summary.exp
+
+$ACLOCAL
+$AUTOMAKE -a
+$AUTOCONF
+
+./configure
+
+($MAKE check || : > make.fail) | tee stdout
+test -f make.fail
+
+$PERL -w extract-testsuite-summary stdout > summary.got
+cat summary.exp
+cat summary.got
+diff summary.exp summary.got || Exit 1
+
+:
diff --git a/tests/testsuite-summary-count.test
b/tests/testsuite-summary-count.test
new file mode 100755
index 0000000..3e24e3a
--- /dev/null
+++ b/tests/testsuite-summary-count.test
@@ -0,0 +1,176 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check test counts in the testsuite summary.
+
+. ./defs-static || Exit 1
+
+use_colors=no
+use_vpath=no
+
+. "$testsrcdir"/testsuite-summary-checks.sh || Exit 99
+
+seq_ ()
+{
+ case $# in
+ 2) l=$1 u=$2;;
+ *) fatal_ "incorrect usage of 'seq_' function";;
+ esac
+ seq $1 $2 || {
+ i=$l
+ while test $i -le $u; do
+ echo $i
+ i=`expr $i + 1`
+ done
+ }
+}
+
+./configure
+
+header="\
+${br}
+Testsuite summary for GNU AutoFoo 7.1
+${br}"
+
+success_footer=${br}
+
+failure_footer="\
+${br}
+See ./test-suite.log
+Please report to address@hidden
+${br}"
+
+# Corner cases.
+
+do_check '' <<END
+$header
+# TOTAL: 0
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t <<END
+$header
+# TOTAL: 1
+# PASS: 1
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check fail.t <<END
+$header
+# TOTAL: 1
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 1
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# Some simpler checks, with low or moderate number of tests.
+
+do_check skip.t skip2.t skip3.t xfail.t xfail2.t <<END
+$header
+# TOTAL: 5
+# PASS: 0
+# SKIP: 3
+# XFAIL: 2
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t pass2.t xfail.t xpass.t error.t error2.t <<END
+$header
+# TOTAL: 6
+# PASS: 2
+# SKIP: 0
+# XFAIL: 1
+# FAIL: 0
+# XPASS: 1
+# ERROR: 2
+$failure_footer
+END
+
+pass_count=22
+skip_count=19
+xfail_count=21
+fail_count=18
+xpass_count=23
+error_count=17
+tests_count=120
+
+pass=` seq_ 1 $pass_count | sed 's/.*/pass-&.t/'`
+skip=` seq_ 1 $skip_count | sed 's/.*/skip-&.t/'`
+xfail=`seq_ 1 $xfail_count | sed 's/.*/xfail-&.t/'`
+fail=` seq_ 1 $fail_count | sed 's/.*/fail-&.t/'`
+xpass=`seq_ 1 $xpass_count | sed 's/.*/xpass-&.t/'`
+error=`seq_ 1 $error_count | sed 's/.*/error-&.t/'`
+
+do_check $pass $skip $xfail $fail $xpass $error <<END
+$header
+# TOTAL: $tests_count
+# PASS: $pass_count
+# SKIP: $skip_count
+# XFAIL: $xfail_count
+# FAIL: $fail_count
+# XPASS: $xpass_count
+# ERROR: $error_count
+$failure_footer
+END
+
+# Mild stress test with a lot of test scripts.
+
+tests_count=1888
+pass_count=1403
+skip_count=292
+xfail_count=41
+fail_count=126
+xpass_count=17
+error_count=9
+
+pass=` seq_ 1 $pass_count | sed 's/.*/pass-&.t/'`
+skip=` seq_ 1 $skip_count | sed 's/.*/skip-&.t/'`
+xfail=`seq_ 1 $xfail_count | sed 's/.*/xfail-&.t/'`
+fail=` seq_ 1 $fail_count | sed 's/.*/fail-&.t/'`
+xpass=`seq_ 1 $xpass_count | sed 's/.*/xpass-&.t/'`
+error=`seq_ 1 $error_count | sed 's/.*/error-&.t/'`
+
+do_check $pass $skip $xfail $fail $xpass $error <<END
+$header
+# TOTAL: $tests_count
+# PASS: $pass_count
+# SKIP: $skip_count
+# XFAIL: $xfail_count
+# FAIL: $fail_count
+# XPASS: $xpass_count
+# ERROR: $error_count
+$failure_footer
+END
+
+:
diff --git a/tests/testsuite-summary-reference-log.test
b/tests/testsuite-summary-reference-log.test
new file mode 100755
index 0000000..7558b5f
--- /dev/null
+++ b/tests/testsuite-summary-reference-log.test
@@ -0,0 +1,88 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that the global testsuite log file referenced in the testsuite
+# summary and in the global testsuite log itself is correct.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+mv configure.in configure.stub
+
+cat > fail << 'END'
+#!/bin/sh
+exit 1
+END
+chmod a+x fail
+
+cat configure.stub - > configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_SUITE_LOG = my_test_suite.log
+TESTS = fail
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+mkdir build
+cd build
+
+../configure
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See \./my_test_suite\.log$' stdout
+
+mkdir bar
+TEST_SUITE_LOG=bar/bar.log $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See \./bar/bar\.log$' stdout
+
+cd ..
+
+echo SUBDIRS = sub > Makefile.am
+mkdir sub
+echo TESTS = fail > sub/Makefile.am
+mv fail sub
+
+cat configure.stub - > configure.in <<'END'
+AC_CONFIG_FILES([sub/Makefile])
+AC_OUTPUT
+END
+
+$ACLOCAL --force
+$AUTOCONF --force
+$AUTOMAKE
+
+./configure
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/test-suite\.log$' stdout
+cd sub
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/test-suite\.log$' stdout
+cd ..
+
+TEST_SUITE_LOG=foo.log $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/foo\.log$' stdout
+
+:
diff --git a/tests/trivial-test-driver b/tests/trivial-test-driver
index f06eb1c..7b3ef10 100644
--- a/tests/trivial-test-driver
+++ b/tests/trivial-test-driver
@@ -20,9 +20,10 @@
# test results per test script.
#
# The exit status of the wrapped script is ignored. Lines in its stdout
-# and stderr beginning with `PASS', `FAIL', `XFAIL', `XPASS' or `SKIP'
-# count as a test case result with the obviously-corresponding outcome.
-# Every other line is ignored for what concerns the testsuite outcome.
+# and stderr beginning with `PASS', `FAIL', `XFAIL', `XPASS', `SKIP' or
+# `ERROR' count as a test case result with the obviously-corresponding
+# outcome. Every other line is ignored for what concerns the testsuite
+# outcome.
#
# This script is used at least by the `driver-custom-multitest*.test'
# tests.
@@ -35,10 +36,12 @@ set -u
test_name=INVALID.NAME
log_file=BAD.LOG
+trs_file=BAD.TRS
while test $# -gt 0; do
case $1 in
--test-name) test_name=$2; shift;;
--log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
# Ignored.
--expect-failure) shift;;
--color-tests) shift;;
@@ -51,50 +54,48 @@ while test $# -gt 0; do
shift
done
-## Run the test script, get test cases results, display them on console.
+## Log file header.
+{
+ echo "RUN: $test_name"
+ echo "RUN: $test_name" | sed 's/./=/g'
+ echo
+} > $log_file
-tmp_out=$log_file-out.tmp
-tmp_res=$log_file-res.tmp
+## Run the test script, get test cases results, display them on console.
-"$@" 2>&1 | tee $tmp_out | (
+"$@" 2>&1 | tee -a $log_file | (
i=0 st=0
- : > $tmp_res
+ exec 5> $trs_file
while read line; do
+ result=
case $line in
- PASS:*|FAIL:*|XPASS:*|XFAIL:*|SKIP:*)
- i=`expr $i + 1`
- result=`LC_ALL=C expr "$line" : '\([A-Z]*\):.*'`
- case $result in FAIL|XPASS) st=1;; esac
- # Output testcase result to console.
- echo "$result: $test_name, testcase $i"
- # Register testcase outcome for the log file.
- echo ":test-result: $line" >> $tmp_res
- echo >> $tmp_res
- ;;
+ PASS:*) result=PASS ;;
+ FAIL:*) result=FAIL ;;
+ XPASS:*) result=XPASS ;;
+ XFAIL:*) result=XFAIL ;;
+ SKIP:*) result=SKIP ;;
+ ERROR:*) result=ERROR ;;
esac
+ if test -n "$result"; then
+ case $result in FAIL|XPASS|ERROR) st=1;; esac
+ # Output testcase result to console.
+ echo "$result: $test_name"
+ # Register testcase outcome for the log file.
+ echo ":test-result: $line" >&5
+ echo >&5
+ fi
done
- exit $st
-)
-
-if test $? -eq 0; then
- global_result=PASS
-else
- global_result=FAIL
-fi
-
-## Write the log file.
-
-{
- echo "$global_result: $test_name"
- echo "$global_result: $test_name" | sed 's/./=/g'
- echo
- cat $tmp_res
- echo
- echo --------------------
- echo
- cat $tmp_out
-} > $log_file
-rm -f $tmp_out $tmp_res
+ if test $st -eq 0; then
+ recheck=no
+ copy_in_global_log=no
+ else
+ recheck=yes
+ copy_in_global_log=yes
+ fi
+ echo ":recheck: $recheck" >&5
+ echo ":copy-in-global-log: $copy_in_global_log" >&5
+ exec 5>&-
+) | awk '{ print $0 ", testcase " NR }'
## And we're done.
diff --git a/tests/txinfo21.test b/tests/txinfo21.test
index eea4890..20bcd00 100755
--- a/tests/txinfo21.test
+++ b/tests/txinfo21.test
@@ -117,8 +117,8 @@ test ! -f rec/main3.html
# Make sure AM_MAKEINFOHTMLFLAGS is supported, and override AM_MAKEINFO.
cat >>Makefile.am <<\EOF
-AM_MAKEINFOHTMLFLAGS=--no-split
-AM_MAKEINFOFLAGS=--unsupported-option
+AM_MAKEINFOHTMLFLAGS = --no-headers --no-split
+AM_MAKEINFOFLAGS = --unsupported-option
EOF
$AUTOMAKE
./configure --prefix "`pwd`"
hooks/post-receive
--
GNU Automake
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Automake-commit] [SCM] GNU Automake branch, test-protocols, updated. v1.11-957-g9a3268ca7,
Stefano Lattarini <=