[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Automake-commit] [SCM] GNU Automake branch, testsuite-work, updated. v1
|
From: |
Stefano Lattarini |
|
Subject: |
[Automake-commit] [SCM] GNU Automake branch, testsuite-work, updated. v1.11-1174-ga956d98 |
|
Date: |
Tue, 09 Aug 2011 22:05:25 +0000 |
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU Automake".
http://git.sv.gnu.org/gitweb/?p=automake.git;a=commitdiff;h=a956d98634008022ff2541b8b9beebae1e025def
The branch, testsuite-work has been updated
via a956d98634008022ff2541b8b9beebae1e025def (commit)
via 02dfe16d6d4e2738f00e168bc7f5c3a664e02530 (commit)
via b3c532b0a3dfb7129bf2119b31e87cc01b49e3d1 (commit)
via 0bb7fe0d70803fde5fbb818f7c801b2c22d553fd (commit)
via 1f9b3058c7bb5166dd9f269f7c334797846a67cf (commit)
via e61b581a99e7a0af8f8dbb793a707f94b0336395 (commit)
via 6b5dccd3a8c79a716b25cbda99f555648e898f60 (commit)
via 1a758e7d0619627aba4f772925154c02e7042f28 (commit)
via 18fb6351e0d7ffe88aac6043e43bcc5163a1dd53 (commit)
via f501499a97946794fe2e5f264e2276ffe826ec07 (commit)
via fe1b11ebe22d45eac5625b237d5dd4d20e0d03c8 (commit)
via 73199a161864ff4f0c692488708c68c22c1c61ea (commit)
via 6a50baf30f92cdcb393f801e8265eddf17ce00ac (commit)
via e171f615247e832582c4806016dc58be7e2f2594 (commit)
via b615fcd597e99a8a16cfa873c6848d3c6204f777 (commit)
via 376293a9ecaf9cc15041260d11e595af867608c8 (commit)
via 07d9b536830dd8369e500c0b989dffb235cf34f0 (commit)
via 389bd6b3d947dab08aa7fbce4540d1bd504e2446 (commit)
via 4c4beef68cf5b0b3ab92e8e41ed322da2d8b21b5 (commit)
via 9880431627fa196f7f5b4fd20aaeac97bee5cbd2 (commit)
via e3b6255fbf58b9120f1df8f91480749def57cec1 (commit)
via c463a964bec391ee9c2735801604e96941a1e33f (commit)
via 53599ad1ee6877116c3ebda34489352cdd15ea24 (commit)
via ff1b47682742db4706a523ef90a6f9daec44058c (commit)
via 0b1c914b122575b00ba3943c57d91b0955581ad5 (commit)
via 8be644b8612ae1ab23bd9de646991c90e03cdea1 (commit)
via 150be8ee36697ce014f3431b961be45c4c6a34b8 (commit)
via f4896832bde810feb7d329f9674082f2d7fc0996 (commit)
via 0e6a7aeb46527a4289b1ccefdc05c10cf1197854 (commit)
via 2ad993189da921058338b2cfdc1897ec7df35354 (commit)
via 2cf22db73f5374bd0f528abfaf65987eab12a665 (commit)
via 1e0238adb8510b959bed87c264581e98afea9a92 (commit)
via bf15d302c88c7da373b3ad7a4d3cea92620baca8 (commit)
via b0a7f1ef5bfbcec5e8942270cdca18b424b5f28f (commit)
via f70e8b593f48b302be43800a5886e790b4280f6a (commit)
via b9dfd77aa5be5a91bfc82ce398d27e8266733554 (commit)
via 232dcc6d2e6c80af70eb72964bfa2c0f08b9de80 (commit)
via cc98b9c624d0a07d703d6cf5eabd06fd0bbc34e8 (commit)
via a0cd0d4b41b7495d2b8b82c7e8d70a1a07664239 (commit)
via fda3de574a6c8398420ad6ae1f456aad368a7193 (commit)
via 152fcd209b21d8a588e787d60abd59d74bbeb69b (commit)
via 0ef3ef52079a967a4ef15968fb2054f1b215e7fb (commit)
via 1072f686df5bb1035620af4a22007feb290d1f12 (commit)
via b7e96983128caec94b256760ae9694c262775988 (commit)
via 42851b794a9bed9fdbb7240b551ebd6bd55f77e7 (commit)
via 30bef0f3dc2e56b1205e3a1d69f5321a9c7a99e6 (commit)
via 4740f56e6fe3360fc21e5de68600a777fd046e99 (commit)
via e0da0d0740c8174270dae2eb2d5fc1f3e8c6c041 (commit)
via 5f5b225a674cde633e42bb3ca3f767ca95ffe6a7 (commit)
via 5c1bb4e240c3538a6bfa7026dfe5f590ac747863 (commit)
via 3f5dcfb065552773b4b0fe295647a9c16363d180 (commit)
via 11aa54925224bb51b85655be1f7d46c6cc0fcf2d (commit)
via 0286308c4923e9b1547811587988fbec73aabf00 (commit)
via 7fb047693f4f820960605a681e6a8f1687dc4a9f (commit)
via 297c8cf72445fd026024b963cd5390f6d7345d3c (commit)
via 45567bd1db489a9745e139dc980a1092624175e7 (commit)
via 273593955a62bbbd7e20b91367c66b6e547d4882 (commit)
via c7fa87264291f611fdc63bd4a49913f98ec5b944 (commit)
via 9f8d51bd31b870b4af9dac5b4030bf3a5c155262 (commit)
via 596ae38451e269e40e3b30ab6f6f016bd9076c04 (commit)
via ff37c9ba221b5b6e2bb365a438617113c89be6c9 (commit)
via 265671e4e2cbd32efaf71595db19b37f97290b69 (commit)
via 0bc42aa61e89dcbde7dc64f932ec795f383f9211 (commit)
via bf630c55d292f7b5b668cfbad31e4d61246f64cb (commit)
via 2a792eb24810033314f675b4e65efbdd9348b706 (commit)
via 3ea0d41765a7f80a1a75e07151f015b793c363fe (commit)
via 8b4dfc2d2923fd5d232b4b20ce147168b3532ab3 (commit)
via 683a7cff745461dbb8851178d37011371ad10ed5 (commit)
via 0cf750d7143939e1118358557a19012a74fdb9d3 (commit)
via 3027d901f4c4698241e074be5063df5b18fa7879 (commit)
via a3345817c8806193742307561787d74117f4b35f (commit)
via 144628d7033cdb098cf77fb94fb5204286adf10e (commit)
via ccf385b973b8eccce6eb38e3b6a35b8b6beded31 (commit)
via cf2e31d3b1f8e7f124c3194417685c0e206fa039 (commit)
via f5cb0635a837eb99591ac0356f242f0496f1ad0f (commit)
via a0d4d97a740412add8673379f7b36b2a40f6418f (commit)
via 4839ca9219b95a6155e4161438318ec0aca3aaef (commit)
via 5e3ae4e14a471a7c61ac75ffd6cd71659e81b07f (commit)
via bc5ba174ea24265b6970e9b747c2cf791f203dc2 (commit)
via 956c58ab490f79be1829ac4fc3a0da58a4b30973 (commit)
via 0e886a6709d920b1f52f6cc80612db0b9d135cf9 (commit)
via 0bc3abcb9f34a00010bd600e5999cb01ad4e75c2 (commit)
via e842d5f294537308fcce7116074291226806b0bc (commit)
via 9b967c28de8fe497c043d8f7eb9c5ea871f2e5f2 (commit)
via 445674ff92cb0929f21627dec0ced4bd05a1c5c8 (commit)
via 0bc7d68a0b1061e9907b420e61fb043eff8850f7 (commit)
via 9a3268ca7a4ec184f014a322ee1ffcbf7055dfd1 (commit)
via f5c022ededc54e53eb3f8a6702c757015f87e322 (commit)
via a5b34b4c6aa1f0d66537c0b4683b6d7dc39a7ffc (commit)
via 05088171d107f4b904e41c308be493efa4e08670 (commit)
via ebb334e7ec6319b36f963143c387c6883e832806 (commit)
via dac4d254d1b8d79bd197d4aece6b58b75ad81247 (commit)
via b9fc781cf7ddeab662e331c12d664d4b6ec2e8ad (commit)
via dffe171f1da9706598d650bf989eb845a9cf275f (commit)
via b1a5d2e3e5255e5ff5abda272c985a5d54073ed9 (commit)
via c349db06721d376cb70be78450e78d30a476910d (commit)
via 25903324b11ea930bf6f6107e119158d494a0603 (commit)
via a2e67e599367ae14bb69fd99202a136db0ec4762 (commit)
via d7fc560de1c1eb43cbdcf68f6a755dbe2ce42adf (commit)
via bd4c823784524b079fc6f0b709d80900e62106af (commit)
via 6d85101e57f1b6a57e317a415b5c46abd35509e4 (commit)
via a017bc080e3c36daa7ee0f3e26820d5f8540fe74 (commit)
via 9fb15f19427bb194feb87c59c52169b2fb9a04d5 (commit)
via 79ddf6800c051f0b75b0b4115568644c156351c5 (commit)
via 6483ac07dfcca5ddd282113f23c3771a8027767c (commit)
via 0c81b43f711fb861f04227ced8dba889596d9c43 (commit)
via 4014b502e2180c1c1ea501ef77e5f97c186d4cbe (commit)
via 24dd0449267486b3268ad812c5fda8b60dad1981 (commit)
via 50c32b61b305cb8c2965470927522ed2f226b917 (commit)
via b9e9d546e8ab45139be6d8e939602cd712c36c8c (commit)
via 3af7ec13fa085bbb532384d6bb7fd01fbd62391d (commit)
via 2fdeb6bb98b502373fbd7af66fd304724ddc94b6 (commit)
via 7858f2bbb985536317fe2b5285ea332db96161c9 (commit)
via ba5fce66ab2e1892ee7f92cc71cf3f1517f6b5bc (commit)
via 58c182ae59f585c5687104e0112d163274c6c732 (commit)
via 1bb5974d3d955590eb771e547a2858430284f477 (commit)
via eaac33fe13811cf104f45a09a6ece911c135fa83 (commit)
via 15400bca303910d059f1a68f94b25cffb1a8ac2f (commit)
via 61b2bb0d647ea5f192e49f5f5885009ea83968be (commit)
via 1402741313cf6c2fe8611eb294f9a8111c994a32 (commit)
via 3605250b571309a41f52ba110c1f14d2a2ca25f7 (commit)
via c96b881f41b180f6176064cc00fe7af620c24c63 (commit)
via 79922343073c66c3f8d4692efb1f31fe47c16fa9 (commit)
via a27c9c4956fd1643cc3a0b3a553ce40039867680 (commit)
from b1289ad8f1071b2e11325a7afcdc6aed632f07f9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
-----------------------------------------------------------------------
Summary of changes:
.gitignore | 16 +-
ChangeLog | 1812 +++++++++++++++++++-
INSTALL | 9 +-
NEWS | 72 +
automake.in | 40 +-
bootstrap | 2 -
doc/.gitignore | 54 +-
doc/amhello/.gitignore | 23 +-
doc/automake.texi | 1144 +++++++++++--
lib/Automake/.gitignore | 2 +-
lib/Automake/tests/.gitignore | 4 +-
lib/Automake/tests/Makefile.in | 327 ++--
lib/Makefile.am | 2 +-
lib/Makefile.in | 2 +-
lib/am/check.am | 391 +++--
lib/am/check2.am | 14 +-
lib/tap-driver | 451 +++++
lib/test-driver | 127 ++
tests/.gitignore | 22 +-
tests/Makefile.am | 286 +++-
tests/Makefile.in | 715 +++++----
tests/README | 50 +-
tests/ac-output-old.tap | 181 ++
tests/acoutbs.test | 42 -
tests/acoutbs2.test | 47 -
tests/acoutnoq.test | 35 -
tests/acoutpt.test | 32 -
tests/acoutpt2.test | 44 -
tests/acoutqnl.test | 32 -
tests/add-missing.tap | 361 ++++
tests/add-missing.test | 315 ----
tests/am-tests-environment.test | 91 +
tests/amopts-variable-expansion.test | 2 +-
tests/autodist.test | 6 +-
tests/automake-cmdline.tap | 63 +
tests/automake.test | 39 -
tests/check-fd-redirect.test | 110 ++
tests/check-no-test-driver.test | 34 +
tests/check.test | 2 +
tests/check10.test | 4 +-
tests/check11.test | 15 +-
tests/check12.test | 28 +-
tests/check2.test | 4 +
tests/check3.test | 2 +-
tests/check4.test | 3 +-
tests/color.test | 108 +-
tests/color2.test | 114 +-
tests/comment9.test | 2 +
tests/cond44.test | 2 +-
tests/cond45.test | 2 +-
tests/configure.test | 2 +-
tests/defs | 440 ++++--
tests/defs-static.in | 20 +-
tests/dejagnu.test | 2 +
tests/depmod-data.test | 32 -
tests/depmod-tests.sh | 226 ---
tests/depmod.tap | 174 ++
tests/exeext4.test | 2 +-
tests/extract-testsuite-summary | 15 +
tests/getopt.test | 4 +-
tests/instspc-data.test | 31 -
tests/instspc-tests.sh | 343 ----
tests/instspc.tap | 278 +++
tests/license2.test | 2 +-
tests/maken3.test | 2 +-
tests/maken4.test | 2 +-
tests/parallel-am.test | 4 +-
tests/parallel-am3.test | 4 +-
tests/parallel-tests-am_tests_environment.test | 95 -
tests/parallel-tests-console-output.test | 104 ++
tests/parallel-tests-driver-install.test | 93 +
tests/parallel-tests-dry-run.test | 112 ++
tests/parallel-tests-empty-testlogs.test | 86 +
tests/parallel-tests-exit-statuses.test | 102 ++
tests/parallel-tests-ext-driver-prog.test | 94 -
tests/parallel-tests-ext-driver.test | 136 --
tests/parallel-tests-extra-programs.test | 174 ++
tests/parallel-tests-fd-redirect.test | 109 ++
tests/parallel-tests-harderror.test | 6 +-
tests/parallel-tests-interrupt.test | 26 +-
tests/parallel-tests-log-compiler-1.test | 126 ++
tests/parallel-tests-log-compiler-2.test | 96 +
tests/parallel-tests-log-override-2.test | 13 +-
tests/parallel-tests-log-override-recheck.test | 43 +-
tests/parallel-tests-no-color-in-log.test | 63 +
tests/parallel-tests-no-spurious-summary.test | 68 +
tests/parallel-tests-once.test | 48 +
tests/parallel-tests-unreadable-log.test | 60 -
tests/parallel-tests-unreadable.test | 75 +
tests/parallel-tests.test | 62 +-
tests/parallel-tests2.test | 36 +-
tests/parallel-tests3.test | 2 +-
tests/parallel-tests6.test | 4 +-
tests/parallel-tests9.test | 17 +-
tests/plain-functions.sh | 40 +
tests/posixsubst-tests.test | 2 +-
tests/repeated-options.test | 2 +-
tests/self-check-cleanup.test | 9 +-
tests/self-check-dir.test | 49 +-
tests/self-check-env-sanitize.tap | 56 +
tests/self-check-env-sanitize.test | 34 -
tests/self-check-exit.tap | 93 +
tests/self-check-exit.test | 83 -
tests/self-check-me.tap | 75 +
tests/self-check-me.test | 45 -
tests/self-check-tap.test | 34 +
tests/suffix-chain.tap | 64 +
tests/suffix10.tap | 68 +
tests/suffix10.test | 66 -
tests/suffix11.tap | 82 +
tests/suffix11.test | 75 -
tests/suffix3.tap | 84 +
tests/suffix3.test | 76 -
tests/suffix8.tap | 91 +
tests/suffix8.test | 86 -
tests/tags2.test | 2 +-
tests/tap-autonumber.test | 75 +
tests/tap-bad-prog.tap | 83 +
tests/tap-bailout.test | 138 ++
tests/tap-basic.test | 177 ++
tests/tap-color.test | 167 ++
tests/tap-common-setup.test | 47 +
tests/tap-deps.test | 91 +
tests/tap-diagnostic-custom.test | 96 +
tests/tap-diagnostic.test | 124 ++
tests/tap-doc.test | 92 +
tests/tap-doc2.test | 135 ++
tests/tap-empty-diagnostic.test | 47 +
tests/tap-empty.test | 55 +
tests/tap-escape-directive.test | 45 +
tests/tap-exit.test | 54 +
tests/tap-fancy.test | 133 ++
tests/tap-fancy2.test | 144 ++
tests/tap-functions.sh | 232 +++
tests/tap-global-log.test | 122 ++
tests/tap-global-result.test | 120 ++
tests/tap-html.test | 80 +
tests/tap-log.test | 164 ++
tests/tap-merge-stdout-stderr.test | 73 +
tests/tap-message-0.test | 66 +
tests/tap-more.test | 164 ++
tests/tap-more2.test | 93 +
tests/tap-no-disable-hard-error.test | 50 +
tests/tap-no-merge-stdout-stderr.test | 53 +
tests/tap-no-spurious-summary.test | 47 +
tests/tap-no-spurious.test | 102 ++
tests/tap-not-ok-skip.test | 39 +
tests/tap-numeric-description.test | 69 +
tests/tap-out-of-order.test | 88 +
tests/tap-passthrough-exit.test | 59 +
tests/tap-passthrough.test | 166 ++
tests/tap-plan-corner.test | 124 ++
tests/tap-plan-errors.test | 93 +
tests/tap-plan-middle.test | 83 +
tests/tap-plan.test | 52 +
tests/tap-realtime.test | 101 ++
tests/tap-recheck-logs.test | 119 ++
tests/tap-recheck.test | 217 +++
tests/tap-setup.sh | 48 +
tests/tap-signal.test | 90 +
tests/tap-skip-whole-badexit.test | 52 +
tests/tap-skip-whole-lastline.test | 59 +
tests/tap-skip-whole-whitespace.test | 48 +
tests/tap-skip-whole.test | 83 +
tests/tap-summary-aux.sh | 361 ++++
tests/tap-summary-color.test | 23 +
tests/tap-summary.test | 23 +
tests/tap-todo-skip-together.test | 39 +
tests/tap-todo-skip-whitespace.test | 100 ++
tests/tap-todo-skip.test | 91 +
tests/tap-unplanned.test | 159 ++
tests/tap-whitespace-normalization.test | 165 ++
tests/tap-with-and-without-number.test | 50 +
tests/tap-xfail-tests.test | 68 +
tests/test-driver-create-log-dir.test | 83 +
tests/test-driver-custom-html.test | 104 ++
tests/test-driver-custom-multitest-recheck.test | 214 +++
tests/test-driver-custom-multitest-recheck2.test | 188 ++
tests/test-driver-custom-multitest.test | 172 ++
tests/test-driver-custom-no-extra-driver.test | 63 +
tests/test-driver-custom-no-html.test | 67 +
tests/test-driver-custom-xfail-tests.test | 166 ++
tests/test-driver-custom.test | 144 ++
tests/test-driver-fail.test | 65 +
tests/test-driver-strip-vpath.test | 98 ++
tests/test-driver-trs-suffix-registered.test | 58 +
tests/test-harness-vpath-rewrite.test | 59 +
tests/test-log.test | 179 ++
tests/test-metadata-global-log.test | 147 ++
tests/test-metadata-global-result.test | 196 +++
tests/test-metadata-recheck.test | 163 ++
tests/test-metadata-results.test | 176 ++
tests/test-missing.test | 72 +
tests/test-missing2.test | 54 +
tests/test-trs-basic.test | 158 ++
tests/test-trs-recover.test | 167 ++
tests/test-trs-recover2.test | 133 ++
tests/tests-environment-and-log-compiler.test | 2 +-
...sts_environment.test => tests-environment.test} | 0
tests/testsuite-summary-checks.sh | 106 ++
tests/testsuite-summary-color.test | 199 +++
tests/testsuite-summary-count-many.test | 105 ++
tests/testsuite-summary-count.test | 161 ++
tests/testsuite-summary-reference-log.test | 88 +
tests/trivial-test-driver | 102 ++
tests/warnings-win-over-strictness.test | 4 +-
tests/werror3.test | 4 +-
tests/werror4.test | 2 +-
208 files changed, 17919 insertions(+), 3452 deletions(-)
create mode 100755 lib/tap-driver
create mode 100755 lib/test-driver
create mode 100755 tests/ac-output-old.tap
delete mode 100755 tests/acoutbs.test
delete mode 100755 tests/acoutbs2.test
delete mode 100755 tests/acoutnoq.test
delete mode 100755 tests/acoutpt.test
delete mode 100755 tests/acoutpt2.test
delete mode 100755 tests/acoutqnl.test
create mode 100755 tests/add-missing.tap
delete mode 100755 tests/add-missing.test
create mode 100755 tests/am-tests-environment.test
create mode 100755 tests/automake-cmdline.tap
delete mode 100755 tests/automake.test
create mode 100755 tests/check-fd-redirect.test
create mode 100755 tests/check-no-test-driver.test
delete mode 100755 tests/depmod-data.test
delete mode 100755 tests/depmod-tests.sh
create mode 100755 tests/depmod.tap
create mode 100644 tests/extract-testsuite-summary
delete mode 100755 tests/instspc-data.test
delete mode 100755 tests/instspc-tests.sh
create mode 100755 tests/instspc.tap
delete mode 100755 tests/parallel-tests-am_tests_environment.test
create mode 100755 tests/parallel-tests-console-output.test
create mode 100755 tests/parallel-tests-driver-install.test
create mode 100755 tests/parallel-tests-dry-run.test
create mode 100755 tests/parallel-tests-empty-testlogs.test
create mode 100755 tests/parallel-tests-exit-statuses.test
delete mode 100755 tests/parallel-tests-ext-driver-prog.test
delete mode 100755 tests/parallel-tests-ext-driver.test
create mode 100755 tests/parallel-tests-extra-programs.test
create mode 100755 tests/parallel-tests-fd-redirect.test
create mode 100755 tests/parallel-tests-log-compiler-1.test
create mode 100755 tests/parallel-tests-log-compiler-2.test
create mode 100755 tests/parallel-tests-no-color-in-log.test
create mode 100755 tests/parallel-tests-no-spurious-summary.test
create mode 100755 tests/parallel-tests-once.test
delete mode 100755 tests/parallel-tests-unreadable-log.test
create mode 100755 tests/parallel-tests-unreadable.test
create mode 100644 tests/plain-functions.sh
create mode 100755 tests/self-check-env-sanitize.tap
delete mode 100755 tests/self-check-env-sanitize.test
create mode 100755 tests/self-check-exit.tap
delete mode 100755 tests/self-check-exit.test
create mode 100755 tests/self-check-me.tap
delete mode 100755 tests/self-check-me.test
create mode 100755 tests/self-check-tap.test
create mode 100755 tests/suffix-chain.tap
create mode 100755 tests/suffix10.tap
delete mode 100755 tests/suffix10.test
create mode 100755 tests/suffix11.tap
delete mode 100755 tests/suffix11.test
create mode 100755 tests/suffix3.tap
delete mode 100755 tests/suffix3.test
create mode 100755 tests/suffix8.tap
delete mode 100755 tests/suffix8.test
create mode 100755 tests/tap-autonumber.test
create mode 100755 tests/tap-bad-prog.tap
create mode 100755 tests/tap-bailout.test
create mode 100755 tests/tap-basic.test
create mode 100755 tests/tap-color.test
create mode 100755 tests/tap-common-setup.test
create mode 100755 tests/tap-deps.test
create mode 100755 tests/tap-diagnostic-custom.test
create mode 100755 tests/tap-diagnostic.test
create mode 100755 tests/tap-doc.test
create mode 100755 tests/tap-doc2.test
create mode 100755 tests/tap-empty-diagnostic.test
create mode 100755 tests/tap-empty.test
create mode 100755 tests/tap-escape-directive.test
create mode 100755 tests/tap-exit.test
create mode 100755 tests/tap-fancy.test
create mode 100755 tests/tap-fancy2.test
create mode 100644 tests/tap-functions.sh
create mode 100755 tests/tap-global-log.test
create mode 100755 tests/tap-global-result.test
create mode 100755 tests/tap-html.test
create mode 100755 tests/tap-log.test
create mode 100755 tests/tap-merge-stdout-stderr.test
create mode 100755 tests/tap-message-0.test
create mode 100755 tests/tap-more.test
create mode 100755 tests/tap-more2.test
create mode 100755 tests/tap-no-disable-hard-error.test
create mode 100755 tests/tap-no-merge-stdout-stderr.test
create mode 100755 tests/tap-no-spurious-summary.test
create mode 100755 tests/tap-no-spurious.test
create mode 100755 tests/tap-not-ok-skip.test
create mode 100755 tests/tap-numeric-description.test
create mode 100755 tests/tap-out-of-order.test
create mode 100755 tests/tap-passthrough-exit.test
create mode 100755 tests/tap-passthrough.test
create mode 100755 tests/tap-plan-corner.test
create mode 100755 tests/tap-plan-errors.test
create mode 100755 tests/tap-plan-middle.test
create mode 100755 tests/tap-plan.test
create mode 100755 tests/tap-realtime.test
create mode 100755 tests/tap-recheck-logs.test
create mode 100755 tests/tap-recheck.test
create mode 100755 tests/tap-setup.sh
create mode 100755 tests/tap-signal.test
create mode 100755 tests/tap-skip-whole-badexit.test
create mode 100755 tests/tap-skip-whole-lastline.test
create mode 100755 tests/tap-skip-whole-whitespace.test
create mode 100755 tests/tap-skip-whole.test
create mode 100755 tests/tap-summary-aux.sh
create mode 100755 tests/tap-summary-color.test
create mode 100755 tests/tap-summary.test
create mode 100755 tests/tap-todo-skip-together.test
create mode 100755 tests/tap-todo-skip-whitespace.test
create mode 100755 tests/tap-todo-skip.test
create mode 100755 tests/tap-unplanned.test
create mode 100755 tests/tap-whitespace-normalization.test
create mode 100755 tests/tap-with-and-without-number.test
create mode 100755 tests/tap-xfail-tests.test
create mode 100755 tests/test-driver-create-log-dir.test
create mode 100755 tests/test-driver-custom-html.test
create mode 100755 tests/test-driver-custom-multitest-recheck.test
create mode 100755 tests/test-driver-custom-multitest-recheck2.test
create mode 100755 tests/test-driver-custom-multitest.test
create mode 100755 tests/test-driver-custom-no-extra-driver.test
create mode 100755 tests/test-driver-custom-no-html.test
create mode 100755 tests/test-driver-custom-xfail-tests.test
create mode 100755 tests/test-driver-custom.test
create mode 100755 tests/test-driver-fail.test
create mode 100755 tests/test-driver-strip-vpath.test
create mode 100755 tests/test-driver-trs-suffix-registered.test
create mode 100755 tests/test-harness-vpath-rewrite.test
create mode 100755 tests/test-log.test
create mode 100755 tests/test-metadata-global-log.test
create mode 100755 tests/test-metadata-global-result.test
create mode 100755 tests/test-metadata-recheck.test
create mode 100755 tests/test-metadata-results.test
create mode 100755 tests/test-missing.test
create mode 100755 tests/test-missing2.test
create mode 100755 tests/test-trs-basic.test
create mode 100755 tests/test-trs-recover.test
create mode 100755 tests/test-trs-recover2.test
rename tests/{check-tests_environment.test => tests-environment.test} (100%)
create mode 100755 tests/testsuite-summary-checks.sh
create mode 100755 tests/testsuite-summary-color.test
create mode 100755 tests/testsuite-summary-count-many.test
create mode 100755 tests/testsuite-summary-count.test
create mode 100755 tests/testsuite-summary-reference-log.test
create mode 100644 tests/trivial-test-driver
diff --git a/.gitignore b/.gitignore
index 716ef8b..68a7fde 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,12 +1,12 @@
-autom4te.cache
+/autom4te.cache
+/aclocal
+/automake
+/config.cache
+/config.log
+/config.status
+/config.status.lineno
+/configure.lineno
Makefile
-aclocal
-automake
-config.cache
-config.log
-config.status
-config.status.lineno
-configure.lineno
cscope.files
cscope.in.out
cscope.out
diff --git a/ChangeLog b/ChangeLog
index dbfc858..d91bc59 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,244 @@
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ test harness: recursive make invocations must use $(AM_MAKEFLAGS)
+ * lib/am/check.am ($(TEST_SUITE_LOG)): Pass $(AM_MAKEFLAGS)
+ to $(MAKE) when issuing a recursive make call.
+ Problem detected by the `sc_tests_make_without_am_makeflags'
+ maintainer check.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ tests: fix spurious failure in 'suffix-chain.tap'
+ * tests/suffix-chain.test: Use more uncommon suffixes that
+ don't trigger built-in recipes with FreeBSD make.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ testsuite: improvements to our shell TAP library and APIs
+ These improvements impact only the Automake's own testsuite.
+ * tests/tap-functions.sh (plan_): Allow new special arguments
+ "unknown" a.k.a. "later" and "lazy" a.k.a. "now". Perform
+ sanity check the given arguments. Adjust incorrect comments.
+ (lazy_plan_): Removed, superseded by the new version of `plan_'.
+ (not): New function, run a command and inverts its exit status.
+ (ok_, not_ok_, skip_): These are just shorthands for common
+ `result_' usages.
+ (xfail_, xpass_, reset_test_count_): Removed, they are not used,
+ nor does a legitimate use for them seem in sight.
+ (not): New function, to run a command inverting its exit status.
+ (command_not_ok_): Removed, superseded by `command_ok_' + `not'.
+ (command_ok_): Extended to accept TAP directives.
+ * tests/defs (trap): Don't use `late_plan_' anymore, use
+ `plan_ "now"' instead.
+ * tests/ac-output-old.tap: Update to take advantage of the new
+ APIs.
+ * tests/self-check-env-sanitize.tap: Likewise.
+ * tests/suffix-chain.tap: Likewise.
+ * tests/suffix10.tap: Likewise.
+ * tests/suffix11.tap: Likewise.
+ * tests/suffix3.tap: Likewise.
+ * tests/suffix8.tap: Likewise.
+ * tests/tap-bad-prog.tap: Likewise.
+ * tests/self-check-exit.tap: Likewise, plus a small improvement
+ in the name of a variable.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ testsuite: convert some `suffix*' tests to TAP
+ This will allow us to explicitly declare some expected failures
+ involving Solaris make as such, instead of partly papering over
+ them with skips and partly letting them fail too loudly.
+ * tests/suffix-chain.test: Rename ...
+ * tests/suffix-chain.tap: ... to this, and convert to the use of
+ TAP. Also, fix some bugs since we are at it, and ...
+ ($required): ... require `cc'.
+ * tests/suffix3.test: Rename ...
+ * tests/suffix3.tap: ... to this, and convert to the use of TAP.
+ Also, since we are at it, throw in some formatting fixes.
+ * tests/suffix8.test: Rename ...
+ * tests/suffix8.tap: ... to this, and convert to the use of TAP.
+ Also, since we are at it, throw in some formatting fixes.
+ * tests/suffix10.test: Rename ...
+ * tests/suffix10.tap: ... to this, and convert to the use of TAP.
+ * tests/suffix11.test: Rename ...
+ * tests/suffix11.tap: ... to this, and convert to the use of TAP.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ tests: don't let a known Solaris make bug poison too many tests
+ See automake bug#7670 and bug#7824.
+ * tests/defs (make_can_chain_suffix_rules): New function, tell
+ whether the make implementation in use can chain suffix rules
+ automatically or not.
+ * tests/suffix3.test: Use it to avoid issuing calls to make that
+ are unportable to make implementations that are not smart enough
+ to chain suffix rules automatically.
+ * tests/suffix8.test: Use it to avoid issuing calls to make that
+ * tests/suffix10.test: Use it to avoid issuing calls to make that
+ * tests/suffix11.test: Use it to avoid issuing calls to make that
+ * tests/suffix-chain.test: New test, exposes the limitation that
+ we have papered over in the tests above.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ gitignore: more use of anchors
+ * .gitignore: Anchor files that are intended to be ignored only
+ if found in the same directory of the `.gitignore' file, not also
+ in its subdirectories.
+ * doc/.gitignore, doc/amhello/.gitignore, lib/Automake/.gitignore,
+ lib/Automake/tests/.gitignore, tests/.gitignore: Likewise. Also,
+ where needed, add new entries that were once implied by the
+ non-anchored entries in the upper-level `.gitignore' files.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ self tests: avoid spurious output on console, and related fixlets
+ * tests/self-check-cleanup.test: Set `$stderr_fileno_' to `2'
+ in the self tests, to avoid spurious diagnostic on console.
+ * tests/self-check-exit.tap: Only unset `am_explicit_skips' and
+ `$stderr_fileno_' in the self tests, not at the global level, it
+ is more correct.
+ ($init): New variable, to reduce code duplication.
+ Some refactoring and adjustments to the name of the tests.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ test defs: few more sanity checks
+ * tests/defs: Catch some more (very unlikely) set-up or internal
+ errors, and diagnose them explicitly. This is intended mostly
+ to be a theoretic-only improvement.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ test defs: really make them "set -e" clean
+ Motivated by a spurious failure of test `instsh3-w.shtst' on
+ Solaris 10.
+ * tests/defs: Make really "set -e" clean, to avoid spurious
+ failures in case this file is sourced by a test script that has
+ already set the `errexit' flag. To be sure we don't regress,
+ move the `set -e' call much earlier, and make the comment telling
+ about the expected "set -e" cleanliness more loud and outstanding.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ self tests: register an expected failures with Solaris /bin/sh
+ Solaris 10 /bin/sh erroneously exit with success right away when
+ the following three conditions are met at the same time:
+ 1. the `errexit' flag is active,
+ 2. an exit trap is installed, and
+ 3. a non-existing command is issued.
+ * tests/self-exit.tap: When that bug is detected, issue an XFAIL
+ rather than a FAIL. Since we are at it, improve by avoiding
+ creation of stray temporary files when testing for non-executable
+ commands.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ gitignore: update and improve for the testsuite files
+ * tests/.gitignore: Don't ignore temporary partial log files,
+ autogenerated tests and makefile fragments that are not used
+ anymore. Anchor all the remaining patterns.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ self tests: some new uses of TAP, few extensions
+ * tests/self-check-me.test: Renamed ...
+ * tests/self-check-me.tap: .. to this, and converted to use TAP.
+ * tests/self-check-exit.test: Renamed ...
+ * tests/self-check-exit.tap: .. to this, and converted to use TAP.
+ Set `$stderr_fileno_' to `2', to avoid spurious messages on the
+ console. Add a new test, and relax another one.
+ * tests/self-check-env-sanitize.test: Renamed ...
+ * tests/self-check-env-sanitize.tap: .. to this, and converted to
+ the use TAP.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-08-09 Stefano Lattarini <address@hidden>
+
+ test defs: test directory creation more configurable
+ After this change, the individual tests will be able to tell
+ `./defs' whether or not to create a temporary test directory
+ for them, and if yes, whether or not to pre-populate it.
+ * tests/defs: Only create the temporary test directory if the
+ variable `$am_create_testdir' is not set to "no". In this
+ case, only pre-populate it if that variable is not set to
+ `$empty'.
+ * tests/defs-static.in: Check that `$am_create_testdir' is not
+ exported in the environment.
+ * tests/self-check-env-sanitize.test: Update.
+ * tests/self-check-dir.test: Extend, and make deeper and more
+ robust.
+ * tests/README (Section "Writing test cases" subsection "Do"):
+ Update.
+ * tests/autodist.test: Set `$am_create_testdir' instead of
+ removing by hand files created by ./defs.
+ * tests/tap-doc2.test: Likewise.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ testsuite: more uses of the `seq_' function
+ * tests/parallel-tests-exit-statuses.test: Use the `seq_' function
+ from `tests/defs', instead of duplicating its semantics.
+ * tests/testsuite-summary-count.test: Likewise.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ testsuite: drop redundant environment sanitization
+ * tests/Makefile.am (TESTS_ENVIRONMENT): Don't remove `planned'
+ from the environment.
+ * tests/defs-static.in: Don't complain if `planned' is set in
+ the environment.
+ * tests/self-check-env-sanitize.test: Update.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ test defs: fix bug in plain version of `skip_all_'
+ * tests/plain-functions.sh (skip_all_): Call `skip_', not `skip'.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ test defs: small cleanups and tweakings
+ * tests/defs: Where possible, prefer `framework_failure_' or
+ `fatal_' to `Exit 99'. Try to catch more possible hard errors
+ and setup failures. Do not hide expected error messages. Avoid
+ a couple of forks when copying files from `$(top_srcdir)/lib'.
+ ($signal): Remove, no more needed.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ testsuite: improve interface for giving early TAP plan
+ Since commit `v1.11-976-g4740f56', an early TAP plan can also be
+ specified after input lines have already been seen, as long as
+ no one of them is a TAP result line. We can now take advantage
+ of this in our own testsuite, allowing the TAP-based scripts to
+ specify their earlier plan after having included `./defs'. In
+ particular, this offers the benefit of allowing those scripts to
+ issue a "skip all" plan (e.g., "1..0 # SKIP reason ...") after
+ the inclusion of `./defs', which wasn't possible with the older
+ interface (introduced in commit `v1.11-1114-g0286308').
+ * tests/defs: Don't try to call `plan_' if `$planned' is defined,
+ the calling scripts can do that directly now.
+ * tests/ac-output-old.tap: Use `plan_' directly instead of
+ setting `$planned' before including `./defs'.
+ * tests/automake-cmdline.tap: Likewise.
+ * tests/depmod.tap: Likewise.
+ * tests/instspc.tap: Likewise.
+ * tests/tap-bad-prog.tap: Likewise.
+ * tests/add-missing.tap: Don't set `$planned'.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ test defs: fix bug in TAP version of `skip_all_'
+ * tests/tap-functions (skip_all_): Set `$have_tap_plan_' to "yes"
+ when called, so that the exit trap won't write another TAP plan.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ test defs: fix skip directives to work also with TAP-based tests
+ * test/defs: Use `skip_all_', not `skip_', to issue a skip of the
+ whole calling test script.
+
2011-08-08 Stefano Lattarini <address@hidden>
test defs: more environment cleanup
@@ -6,12 +247,641 @@
2011-08-08 Stefano Lattarini <address@hidden>
+ test harness: more minor VPATH-rewrite issues
+ * tests/am/check.am (am__set_TESTS_bases, $(TEST_SUITE_LOGS),
+ am--redo-logs, recheck, recheck-html): Cosmetic fixlets to
+ minimize the risk of unwanted VPATH rewrites.
+ (check-TESTS): Likewise, and normalize trailing whitespace
+ since we are at it.
+ Bugs exposed by test cases `check6-p.test' and `check7-p.test'.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ test harness: "make -n check" works correctly with Solaris make
+ To avoid some recipes being executed with "make -n", we used the
+ trick of issuing recursive make calls not directly with $(MAKE),
+ but with a simple indirection instead:
+ ...
+ am__stealth_MAKE = $(MAKE)
+ ...
+ target:
+ ... $(am__stealth_MAKE) ...
+ ...
+ This hack worked with GNU make, but, unfortunately, not with
+ Solaris make. The present change removes the hack, and uses more
+ proper idioms to avoid unwanted recipe executions with "make -n".
+ Bug exposed by test case `parallel-tests-dry-run.test'.
+ * lib/am/check.am (am__stealth_MAKE): Remove, no more used.
+ (.log.trs): Break recipe in multiple shell invocations.
+ ($(TEST_SUITE_LOG)): Likewise, and move some corner-case parts of
+ the recipe to ...
+ (am--redo-logs): ... this new internal helper .PHONY target.
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
tests: fix spurious failure with Solaris make
* tests/distcheck-configure-flags-am.test: Avoid using `+=' too
liberally with AM_DISTCHECK_CONFIGURE_FLAGS, since the line breaks
so introduced, in conjunction with single quotes, might confuse
Solaris make.
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ test harness: fixlet in workaround for VPATH rewrite issue
+ Fixes a minor in previous commit `v1.11-983-gfda3de5'.
+ * lib/am/check.am (am__set_TESTS_bases): Add forgotten end-of-line
+ anchor to sed expression. In comments, add reference to ...
+ * tests/test-trs-basic.test: ... this test, which has been adapted
+ and tightened (and tweaked).
+
+2011-08-08 Stefano Lattarini <address@hidden>
+
+ test harness: work around a VPATH rewrite issue
+ * lib/am/check.am (am__TEST_BASES): Removed, it's role taken
+ over by ...
+ (am__set_TESTS_bases): ... these new variable.
+ ($(TEST_SUITE_LOG): Use it, to avoid VPATH rewrite issues.
+ * automake.in (handle_tests): Update the code for the cleanup
+ of the `.trs' file to use `$(TEST_LOGS)' instead of
+ `$(am__TEST_BASES)'.
+ * tests/test-harness-vpath-rewrite.test: New test, better
+ exposing the problem fixed by this change
+ * tests/Makefile.am (TESTS): Add it.
+ * tests/test-trs-basic.test: Update and extend.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ parallel-tests: work around Solaris XPG4 make segfault
+ Issue exposed by test `posixsubst-tests-p.test', and similar to
+ the problem solved by commit `v1.11-159-ge7aa360'.
+ * lib/am/check.am [%?PARALLEL_TESTS%] (check-TESTS): Trim trailing
+ whitespace from $list, to avoid triggering a nasty bug (potential
+ segfault) on Solaris XPG4 make and Heirloom make.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: fix weird spurious failure with Solaris /bin/sh
+ Solaris /bin/sh, when killed with a SIGTERM or SIGINT signal, can
+ apparently end up exiting with exit status 208, instead of leaving
+ the correct wide exit status to the parent. See:
+ <http://dbaspot.com/shell/396118-bourne-shell-exit-code-term.html>
+ Work around this incompatibility.
+ * tap-signal.test: Look for the above Solaris bug.
+ (signal_caught): Adapt to handle it.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: fix another spurious failure on Solaris make
+ * tests/parallel-tests-log-override-recheck.test: Filter make
+ output before grepping it, for make implementations that, like
+ Solaris' one, print the whole of the failed recipe on failure.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: fix two spurious failures on Solaris make
+ * tests/tests-environment-and-log-compiler.test: Do not fail if
+ "make -n check" fails, as that is issued only for debugging, its
+ semantics being tested in other test scripts.
+ * tests/test-driver-strip-vpath.test: Likewise.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ tap driver: refactor and remove dead code
+ * lib/tap-driver (get_tap_line): Removed, ...
+ (all callers): ... updated to use $parser->next directly instead.
+ (peek_tap_line, unget_tap_line, @tap_lines): Remove, they're not
+ used anymore.
+ (TAP_PEEKING): Block deleted, all its subroutines and variables
+ having been removed.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: fix spurious failure in a test on TAP support
+ * tests/tap-passthrough-exit.test: When the `--ignore-exit' flag
+ of the TAP driver is used, don't look for a message reporting the
+ non-zero exit statuses of tests in the log files; such message is
+ not expected to be there anymore. Related simplifications.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ tap: plan location is more liberal w.r.t. non-TAP lines
+ With this change, only lines that are TAP results will matter
+ w.r.t. the position of the TAP plan in the input; for example,
+ this input:
+ this is a non-TAP line
+ # and this a TAP diagnostic line
+ 1..1
+ ok 1
+ was considered to be an error, diagnosed with a message "test
+ plan in middle of output"; as effect of the current change, such
+ input is now valid. This is more consistent with the behaviour
+ of the `prove' utility.
+ * lib/tap-driver ($lineno): Removed, no more needed.
+ ($tap_stopped): New global variable.
+ (stringify_test_result): Return "ERROR" if a TAP result is found
+ when `$tap_stopped' is set to true.
+ (handle_tap_test): Diagnose TAP results that comes after a late
+ plan. Add a couple of blank lines, for clarity.
+ (handle_tap_plan): Set `$tap_stopped' to true after a late plan
+ is encountered. Do not complain anymore for extra non-TAP lines
+ preceding or following the plan. Adjust comments.
+ (main): Don't increment $lineno anymore.
+ * tests/tap-plan.test: Extend a bit, and remove stale comment.
+ * tests/tap-color.test: Adjust.
+ * tests/tap-passthrough.test: Likewise.
+ * tests/tap-plan-corner.test: Adjust and extend.
+ * tests/tap-plan-errors.test: Likewise.
+ * tests/tap-plan-middle.test: New test.
+ * tests/tap-plan-corner2.test: Delete, it's obsolete now.
+ * tests/Makefile.am (XFAIL_TESTS): Remove it.
+ (tap_with_common_setup_tests): Likewise, and add
+ `tap-plan-corner.test'.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: remove now-passing test from XFAIL_TESTS
+ * tests/Makefile.am (XFAIL_TESTS): Remove `tap-signal.test',
+ which is passing since previous commit `v1.11-974-gc7fa872'.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ test defs: get name of TAP-using tests correctly
+ This change fixes the names of the temporary directories used
+ by the TAP-based tests (e.g., test `foo.tap' now uses `foo.dir'
+ rather than `foo.tap.dir'); more importantly, this make the
+ variable `$destdir' defined in `tests/defs' usable also in the
+ TAP tests.
+ ($me): Also strip the any `.tap' suffix from the name of the
+ test.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: TAP tests properly decide when to remove tempdirs
+ Before this change, the TAP tests in the Automake testsuite were
+ removing the temporary test directory even when they failed or
+ were skipped, thus making debugging more difficult.
+ * tests/tap-functions.sh (incr_tap_count): Removed, superseded
+ by ...
+ (incr_): ... this function, which can increment the value of any
+ variable passed to it.
+ (result_): Updated to use `incr_' instead of the now-removed
+ `incr_tap_count_'. Keep count of failures, xfailures, xpasses,
+ and skips, using ...
+ ($tap_skip_count_, $tap_bad_count, _$tap_xfail_count_): ...
+ these new variables.
+ * tests/defs (trap): Try to use their values to decide whether
+ the temporary directory being used by the test script should be
+ removed or not. Other code reorganizations. And move the code
+ for the removal of the temporary directory out to ...
+ (rm_rf_): ... this new subroutine.
+ (Main code): Use that instead of duplicating the code.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: improve and refactor our custom TAP shell library
+ This change is mostly done in preparation for the next one.
+ * tests/tap-functions.sh (result_with_directive_): Merge into ...
+ (result_): ... this function, which has been greatly extended and
+ improved in various ways.
+ (ok_, not_ok_, skip_, xfail_, xpass_): Adjust.
+ Adjust comments. Remove an extra empty line.
+ * tests/instspc.tap: Use the new improved interface of `result_'.
+ * tests/tap-bad-prog.tap: Likewise.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: update "user interface" description in tests/README
+ * tests/README (Subsection "Interpretation"): Mention the "ERROR"
+ outcome.
+ (Subsection "Getting details from failures"): Don't tell that the
+ test scripts can be run directly, this is not 100% true; anymore;
+ instead ...
+ (Subsection "About the tests"): ... use this new subsection to
+ explain how to run the tests (either simple or TAP-based) by hand.
+ (Subsection "Supported shells"): Extend to refer to TAP tests too.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: TAP tests can have early plan now
+ * tests/defs: When using TAP, allow the user to specify the number
+ of tests through the `$planned' variable.
+ (trap): Do not print the auto-generated plan if an explicit one
+ has already been printed.
+ * tests/defs-static.in: Complain and bail out if the `$planned'
+ variable is exported in the environment. Reformat the code to
+ facilitate future additions.
+ * tests/Makefile.am (TESTS_ENVIRONMENT): Neutralize `$planned'.
+ * tests/self-check-env-sanitize.test: Update.
+ * tests/ac-output-old.tap: Give an explicit TAP plan, by defining
+ `$planned' just before including `./defs'.
+ * tests/depmode.tap: Likewise.
+ * tests/automake-cmdline.tap: Likewise.
+ * tests/tap-bad-prog.tap: Likewise.
+ * tests/instspc.tap: Likewise, and related adjustments.
+ * tests/add-missing.tap: Make it explicit that we have no a-priori
+ plan, and tell why.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: run autogenerated tests with $(LOG_COMPILER) too
+ This change enhances consistency in the testsuite.
+ * tests/Makefile.am: Some cosmetic reordering, to keep the
+ definition of FOO_LOG_COMPILER near to the place where `.foo'
+ is added to $(TEST_EXTENSIONS).
+ (PTEST_LOG_COMPILER): Run `$(srcdir)/parallel-tests.sh' with
+ `$(LOG_COMPILER)', not with `$(SHELL)'.
+ (SHTST_LOG_COMPILER): Likewise.
+
+2011-08-07 Stefano Lattarini <address@hidden>
+
+ testsuite: run TAP and "plain" tests with the same $(LOG_COMPILER)
+ With this change, it will be easier to override, at make time, the
+ shell used to run the tests.
+ * tests/Makefile.am (LOG_COMPILER): Define to `$(SHELL)'.
+ (TEST_LOG_COMPILER): Re-define to `$(LOG_COMPILER)'.
+ (TAP_LOG_COMPILER): Likewise.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ testsuite: run TAP-based tests explicitly with $(SHELL)
+ * tests/Makefile.am (TAP_LOG_COMPILER): Define to `$(SHELL)'.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ tap driver: handle signals received by the tests being run
+ * lib/tap-driver (get_test_exit_message): Also deal with signals,
+ by using the `wait' method of the TAP::Parser object instead of
+ the `exit' method. This required the use of the standard perl
+ module `POSIX'.
+ * doc/automake.texi (Use TAP with the Automake test harness):
+ Document that `--ignore-exit' has effect also on terminating
+ signals. Add a "synchronizing" comment that references the tests
+ 'tap-exit.test' and 'tap-signal.test'.
+ * tests/tap-signal.test: Extend and adjust.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ test driver: a preparatory refactoring (2)
+ * lib/tap-driver (finish): Move code to fetch the message about
+ the exit status of the test being run to ...
+ (get_test_exit_message): ... this new subroutine.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ tap driver: a preparatory refactoring (1)
+ * lib/tap-driver (finish): Do not flush the TAP stream nor
+ fetch the exit status of the TAP producer if running with
+ the `--ignore-exit' option. Obviously, don't try to write
+ such exit status in the test log anymore.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ cosmetics: typofix in the name of a function in the TAP driver
+ * lib/tap-driver (testuite_error): Renamed ...
+ (testsuite_error): ... to this, and minor related reformatting.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ tap: scripts with a SKIP plan but with exit status != 0 must error
+ This change has been motivated by Automake's own testsuite. Some
+ TAP tests there were erroring out (due to unexpected, unhandled
+ failures) before having encountered TAP result, so that the
+ simple-minded shell library implementing TAP generation ended up
+ printing a "1..0" trailing test plan; this caused the script to be
+ reported as a SKIP rather than an ERROR -- a nasty false negative.
+ * lib/tap-driver: Add prototypes for each subroutine, to free up
+ the order in which they can be defined and called.
+ (main): Move the code checking for a bad exit status of the TAP
+ producer ...
+ (finish): ... here, and flush the TAP stream to ensure that the
+ parser always obtains the producer's exit status.
+ * tests/tap-skip-whole-badexit.test: New test.
+ * tests/Makefile.am (tap_with_common_setup_tests): Add it.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ tap: fix whitespace munging of diagnostic messages
+ * lib/tap-driver (extract_tap_comment): Pass the `g' flag to the
+ substitution operator, to strip also trailing whitespaces. Fixes
+ a failure in test `tap-whitespace-normalization.test'.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ testsuite: refactor and cleanup 'instspc.tap'
+ * tests/instspc.tap (is_in_list): New helper subroutine.
+ (expected_to_fail): Re-implement using it.
+ (define_problematic_string): Likewise. Also, rename the special
+ arguments `build-fail' and 'install-fail' to respectively
+ `builddir-fail' and `destdir-fail', and other related changes.
+ (Test data definition): Adapt.
+ ($instspc_xfail_builds_list): Renamed ...
+ ($builddir_xfails): ... to this.
+ ($instspc_xfail_installs_list): Renamed ...
+ ($destdir_xfails): ... to this.
+ ($instspc_names_list): Renamed ...
+ ($test_names_list): ... to this.
+ ($instspc_test_string): Renamed ...
+ ($test_string): ... to this.
+ Add some explicative and "FIXME" comments.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ testsuite: use TAP for `depmod*' tests, related simplifications
+ * tests/depmod-tests.sh: Delete this complex and multifarious
+ script, moving all it checks it used to perform into ...
+ * tests/depmod.tap: ... this TAP-based test script. And extend
+ it quite a bit, since the new form of the test was papering over
+ some pre-existing failures (this was due to the fact that the
+ values used for the VPATH were shorter in the new test script).
+ * tests/depmod-data.test: Deleted.
+ * tests/Makefile.am ($(srcdir)/depmod-tests.am): Don't include
+ nor generate it anymore.
+ (EXTRA_DIST): Don't distribute depmod-tests.sh anymore.
+ (TEST_EXTENSIONS): Remove '.depmod'.
+ (DEPMOD_LOG_COMPILER): Deleted.
+ (depmod-data.log, $(depmod_tests:.depmod=.log)): Delete
+ dependencies for these files, which are not used anymore.
+ ($(depmod_tests)): Delete this dummy dependency declaration.
+ (TESTS): Remove $(depmod_tests).
+ (handwritten_tests): Remove depmod-data.test, add depmod.tap.
+ * bootstrap: Don't generate tests/depmod-tests.am anymore.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ testsuite: use TAP for `instspc*' tests, related simplifications
+ * tests/instspc-tests.sh: Delete this complex and multifarious
+ script, moving all it checks it used to perform into ...
+ * tests/instspc.tap: ... this TAP-based test script.
+ * tests/instspc-data.test: Deleted.
+ * tests/Makefile.am ($(srcdir)/instspc-tests.am): Don't include
+ nor generate it anymore.
+ (EXTRA_DIST): Don't distribute instspc-tests.sh anymore.
+ (TEST_EXTENSIONS): Remove '.instspc'.
+ (INSTSPC_LOG_COMPILER): Deleted.
+ (XFAIL_TESTS): Remove $(instspc_xfail_tests).
+ (instspc-data.log, $(instspc_tests:.instspc=.log)): Delete
+ dependencies for these files, which are not used anymore.
+ ($(instspc_tests)): Delete this dummy dependency declaration.
+ (TESTS): Remove $(instspc_tests).
+ (handwritten_tests): Remove instspc-data.test, add instspc.tap.
+ * bootstrap: Don't generate tests/instspc-tests.am anymore.
+
+2011-08-06 Stefano Lattarini <address@hidden>
+
+ testsuite: do fd redirections with $(AM_TESTS_FD_REDIRECT)
+ * tests/Makefile.am: The redirections of file descriptors needed
+ by our test scripts is now done using the $(AM_TESTS_FD_REDIRECT)
+ variable (the new blessed way since commit `v1.11-906-gb9e9d54'),
+ not using an hack involving $(AM_TESTS_ENVIRONMENT). This has
+ the further benefit of allowing the use of $(TESTS_ENVIRONMENT)
+ again on part of the users.
+ * tests/Makefile.am (AM_TESTS_ENVIRONMENT): Remove redirection of
+ file descriptors.
+ (AM_TESTS_FD_REDIRECT): Redirect file descriptor 9 to original
+ stderr.
+ Comments adjusted.
+ * tests/plain-functions.sh ($stderr_fileno_): Update comment.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ testsuite: use custom TAP diagnostic in our own tests
+ This change allow us to easily and transparently avoid spurious
+ TAP diagnostic in our own testsuite, with the help of the new
+ `--diagnostic-string' option of our TAP test driver.
+ * tests/Makefile.am (TAP_LOG_DRIVER_FLAGS): Also pass option
+ `--diagnostic-string' with value `#%#' (necessarily obtained
+ through some shell indirections).
+ * tests/tap-functions.sh (diag_): Assume that the string denoting
+ TAP diagnostic is stored in the `$diag_string_' variable. Update
+ comments.
+ ($diag_string_): Initialize to "#".
+ * tests/defs ($diag_string_): Re-define to "#%#".
+ (show_): Remove, it's obsolete now.
+ * add-missing.tap: Don't use `show_' anymore to avoid spurious
+ TAP diagnostic; `cat' should suffice now.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ tap: fix typo in TAP driver version message
+ * lib/tap-driver (anonymous subroutine printing the script version
+ message): Add missing trailing newline to the version message.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ tap docs: we don't support pragma or version directives (yet)
+ * doc/automake.texi (Incompatibilities with other TAP parsers
+ and drivers): Document that limitation. Normalize an unrelated
+ "TODO" item.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ tap: new option to change the string designating TAP diagnostic
+ Here we introduce a new option `--diagnostic-string' in our TAP
+ test driver, that allows the user to specify which string should
+ denote the beginning of a TAP diagnostic line. This change is
+ not gratuitous, nor result if over-engineering: it is motivated
+ by real issues that have emerged during the use of TAP in the
+ Automake's own testsuite (see the commit `v1.11-1082-g9b967c2'
+ "testsuite: yet more use of TAP, and related extensions").
+ * doc/automake.texi (Use TAP with Automake test harness): Document
+ the new option.
+ (Incompatibilities with other TAP parsers and drivers): Report it
+ as a potential source of incompatibility.
+ * lib/tap-driver ($diag_string): New global variable, defaulting
+ to "#", and whose value can be changed ...
+ (Getopt::Long::GetOptions): ... by the newly recognized option
+ `--diagnostic-string'.
+ (handle_tap_comment): Subroutine removed, some of its simple logic
+ inlined ...
+ (main): ... in here, where now ...
+ (extract_tap_comment): ... this new subroutine is used.
+ ($USAGE): Adjust.
+ * tests/tap-diagnostic.test: Make one check slightly stricter.
+ * tests/tap-diag-custom.test: New test.
+ * tests/Makefile.am (tap_other_tests): Add it.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ check: add small "synchronization" comment to `test-driver'
+ * lib/test-driver: Add comment to the code initializing ANSI color
+ escapes, telling to keep it in sync with the similar initialization
+ in `lib/am/check.am'.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ maintcheck: fix spurious failure w.r.t. use of bare "exit"
+ * tests/check12.test: Cosmetic changes to avoid triggering a
+ spurious failure of the `sc_tests_Exit_not_exit' maintainer
+ check.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ check: update comments to reflect recent heavy changes
+ * lib/am/check.am [%?PARALLEL_TESTS%]: Update the comments to
+ reflect the recent changes. Remove quite many comments that
+ were merely duplicating excerpts from the Automake manual.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ check: small cleanups and refactorings in test harness and drivers
+ * lib/tap-driver (yn): New subroutine, converts a boolean value to
+ a "yes" or "no" string.
+ (write_test_results): Use it, and related reformatting.
+ * lib/test-driver (trap): Also remove the `.trs' file on signals,
+ for extra safety.
+ (fatal): Remove this function, it's never used.
+ ($logfile, $trsfile): Renamed ...
+ ($log_file, $trs_file): ... to these, for clarity, and in order to
+ be more consistent with the `tap-driver' script.
+ Improve a couple of comments.
+ * automake.in (handle_tests): Don't define anymore the now-obsolete
+ make macro `$(TEST_LOGS_TMP)', nor add it to the list of files to
+ be removed upon "make mostlyclean".
+ * lib/am/check.am ($(TEST_SUITE_LOG)): New shell function `f_ok',
+ tells whether a path refers to an existing, regular, readable file.
+ Use it throughout.
+ (recheck): Be safer w.r.t. make implementation that run recipes
+ with `errexit' shell flag active.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ testsuite: use AM_TESTS_FD_REDIRECT where appropriate
+ * parallel-tests-interrupt.test: Use AM_TESTS_FD_REDIRECT, not
+ AM_TESTS_ENVIRONMENT, to define file descriptors' redirection
+ for "make check". Fix botched comment.
+
+2011-08-05 Stefano Lattarini <address@hidden>
+
+ docs: some fixlets in documentation on testsuites support
+ * doc/automake.texi (Script-based Testsuites): Rename node ...
+ (Scripts-based Testsuites): ... to this. Break overly long lines
+ in an example (were causing problems with PDF output).
+ (Parallel Test Harness): Don't intend paragraphs that come after
+ examples or bulleted list.
+ (Overview of Custom Test Drivers Support): Fix typo s/the the/the/.
+ (Declaring Custom Test Drivers): When giving an aside, prefer comma
+ to parentheses, as it disrupts the reading flow less.
+ (API for Custom Test Drivers): Remove a sentence that was basically
+ duplicated from the previous subsection.
+ (Command-line arguments for test drivers): Don't intend paragraph
+ coming after an itemized list. Clarify wording and fix grammaros.
+ Add a couple of cross-references to earlier explanations of `.log'
+ and `.trs' files.
+ (Links and external resources): Node renamed ...
+ (Links and external resources on TAP): ... to this, for clarity.
+ Slightly extend and clarify introductory sentence.
+
+2011-08-04 Stefano Lattarini <address@hidden>
+
+ test harness: be aware of more metadata, simplify test drivers
+ * lib/am/check.am ($(TEST_SUITE_LOG)): When producing the
+ global test log, take into account the new metadata field
+ `:global-test-result:, to write nicely formatted RST section
+ titles, instead of leaving this chore to the individual test
+ drivers.
+ (am_rst_section): Re-introduce this variable, as removed in
+ commit `v1.11-870-ga27c9c4'.
+ * lib/test-driver, lib/tap-driver: Write the global test result
+ as metadata in the `.trs' file, not as (part of) a formatted RST
+ subsection title in the `.log' file. Related simplifications;
+ in particular, get rid of temporary files usage.
+ * doc/automake.texi (Log files generation and test results
+ recording): Document the new metadata.
+ * tests/check12.test: Update.
+ * tests/parallel-tests-harderror.test: Likewise.
+ * tests/parallel-tests-interrupt.test: Likewise.
+ * tests/parallel-tests-log-override-2.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-unreadable.test: Likewise.
+ * tests/tap-global-result.test: Likewise.
+ * tests/test-metadata-results.test: Likewise.
+ * tests/test-log.test: Likewise, and another minor unrelated fix.
+ * tests/test-metadata-global-result.test: New test.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ tap: check that also a trailing TAP plan can hold a skip directive
+ * tests/tap-skip-whole-lastline.test: New test.
+ * tests/Makefile.am (tap_with_common_setup_tests): Add it.
+
+2011-08-04 Stefano Lattarini <address@hidden>
+
+ testsuite: more correct names for a couple of tests
+ * tests/parallel-tests-ext-driver.test: Rename ...
+ * tests/parallel-tests-log-compiler-1.test: ... to this, and
+ adjust heading comments.
+ * tests/parallel-tests-ext-driver-prog.test: Rename ...
+ * tests/parallel-tests-log-compiler-2.test: ... to this, and
+ adjust heading comments.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-08-04 Stefano Lattarini <address@hidden>
+
+ testsuite: fix another spurious failure
+ * tests/parallel-tests-ext-driver.test: Name out custom/dummy
+ log compilers as `foo-compiler' rather `foo-driver'; not only
+ this is less confusing, it also avoids errors due to the dummy
+ log compiler `test-driver' overriding the `test-driver' helper
+ script installed by Automake.
+ * tests/parallel-tests-ext-driver-prog.test: Adjust heading
+ comments.
+
+2011-08-04 Stefano Lattarini <address@hidden>
+
+ testsuite: fix a spurious failure
+ * tests/parallel-tests-ext-driver.test: Exporting of environment
+ variables directly from $(LOG_COMPILER) variables is not allowed
+ anymore. Adjust to this.
+
+2011-08-04 Stefano Lattarini <address@hidden>
+
+ testsuite: yet more use of TAP, and related extensions
+ * tests/defs (show_): New function, display the contents of one or
+ more files on stdout, with bells & whistles (both for cosmetic and
+ practical reasons, the latter aimed at avoiding spurious TAP
+ diagnostic).
+ (AUTOMAKE_run): Extend and adapt to make it compatible with TAP
+ based tests. Since we are at it, make its implementation more
+ namespace-safe, and improve its command-line interface.
+ (AUTOMAKE_fails): Adapt to the new command-line interface of
+ `AUTOMAKE_run'.
+ * tests/add-missing.test: Renamed ...
+ * tests/add-missing.tap: ... to this, and converted to the use
+ of the TAP protocol, and of the new `show_' function.
+ * tests/automake.test: Renamed ...
+ * tests/automake-cmdline.tap: ... to this, and converted to the
+ use of the TAP protocol.
+ * tests/getopt.test: Adapt to the new command-line interface
+ of `AUTOMAKE_run'.
+ * tests/cond44.test: Likewise.
+ * tests/cond45.test: Likewise.
+ * tests/configure.test: Likewise.
+ * tests/license2.test: Likewise.
+ * tests/parallel-am.test: Likewise.
+ * tests/parallel-am3.test: Likewise.
+ * tests/tags2.test: Likewise.
+ * tests/werror3.test: Likewise.
+ * tests/werror4.test: Likewise.
+ * tests/amopts-variable-expansion.test: Likewise.
+ * tests/warnings-win-over-strictness.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+ (TAP_LOG_DRIVER_FLAGS): Add `--merge', so that TAP diagnostic is
+ reported in the testsuite progress output.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ maintcheck: fix more maintainer-check failures
+ * tests/test-driver-custom-no-html.test (no-rst): Use `EOF',
+ not `EoL', as the here-document delimiter.
+ * tests/test-trs-basic.test: Use `cat + here-doc' rather
+ than `echo' when creating the dummy test scripts, to please
+ maintainer-check.
+ * tests/test-trs-recover.test: Use creative quoting where
+ needed, to please maintainer-check.
+ * tests/parallel-tests-no-color-in-log.test: Likewise.
+ * tests/parallel-tests-dry-run.test: Likewise.
+
2011-08-03 Stefano Lattarini <address@hidden>
maintcheck: fix maintainer-check failures, both real and spurious
@@ -60,6 +930,745 @@
* INSTALL, lib/INSTALL, lib/config.guess, lib/config.sub,
lib/texinfo.tex: Sync from upstream.
+2011-07-08 Stefano Lattarini <address@hidden>
+
+ tests: fix weakness in 'tests-environment-backcompat.test'
+ * tests/tests-environment-backcompat.test: Do not override the
+ content of xfailing test `baz.test' with a "weaker" version that
+ fails unconditionally: the test must fail only when the 'strict'
+ pragma is in use, in order not to reduce coverage.
+
+2011-07-08 Stefano Lattarini <address@hidden>
+
+ docs, tests: synchronize examples on silent-rules from config.site
+ * doc/automake.texi (Automake silent-rules Option): Reference test
+ 'silent-configsite.test' in comments.
+
+2011-07-08 Stefano Lattarini <address@hidden>
+
+ tests: portability fixes in tests on amhello examples
+ * tests/amhello-binpkg.test: Don't use "tar xzf too.tag.gz" to
+ extract a gzip-compressed tarball, that's unportable to some
+ tar implementations; use the "gzip -dc fo.tar.gz | tar xf -"
+ idiom instead.
+ * tests/amhello-cflags.test: Likewise.
+ * tests/amhello-cross-compile.test: Likewise.
+ Suggestion from Ralf Wildenhues.
+
+2011-07-04 Stefano Lattarini <address@hidden>
+
+ remake: add test ensuring that slower remakes don't hang
+ * tests/remake-subdir-long-time.test: New test.
+ * tests/Makefile.am (TESTS): Update.
+ Suggestion by Ralf Wildenhues.
+
+2011-07-04 Stefano Lattarini <address@hidden>
+
+ remake: fix outdated comment in configure.am
+ * lib/am/configure.am: Fix comment falsified by changes in
+ commit `v1.11-366-gbee9871'.
+ Suggestion by Ralf Wildenhues.
+
+2011-07-01 Stefano Lattarini <address@hidden>
+
+ docs, tests: synchronize examples from docs to tests
+ * tests/README (Writing test cases): Give suggestions on how to
+ keep test cases and examples in the documentation synchronized.
+ * doc/automake.texi: Improve or fix existing testcase-referencing
+ comments, and add many new ones.
+ * HACKING (Administrivia): Suggest to test complex examples and
+ idioms from the manual.
+ * tests/specflg8.test: Improve synchronization with the example
+ in the manual.
+ * tests/output11.test:Likewise.
+ * tests/txinfo21.test:Likewise.
+ * tests/interp.test: Likewise.
+ * tests/amhello-cflags.test: New test.
+ * tests/amhello-cross-compile.test: Likewise.
+ * tests/amhello-binpkg.test: Likewise.
+ * tests/tests-environment-backcompat.test: Likewise.
+ * tests/parallel-tests-log-compiler-example.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-06-30 Stefano Lattarini <address@hidden>
+
+ coverage: new test on parallel-tests TESTS runtime overriding
+ * tests/parallel-tests-cmdline-override.test: New test, check that
+ we can use indirections when overriding TESTS and TEST_LOGS from
+ the command line.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ gitignore: ignore `*.trs' files in lib/Automake/tests
+ * lib/Automake/tests/.gitignore: Add `*.trs' pattern.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: list another "forgotten" test script in Makefile.am
+ * tests/Makefile.am (TESTS): Also list test script
+ `test-driver-trs-suffix-registered.test'; it had been introduced
+ in commit `v1.11-910-g0c81b43', but by mistake it wasn't added
+ to the list of tests in the Makefile back then.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: list "forgotten" test script in Makefile.am
+ * tests/Makefile.am (TESTS): Also list the test script
+ `parallel-tests-no-color-in-log.test'; it had been introduced in
+ commit `v1.11-899-geaac33f', but by mistake it wasn't added to
+ the list of tests in the Makefile back then.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: remove duplicated mention of a test in Makefile.am
+ * tests/Makefile.am (tap_other_tests): Remove `tap-empty.test',
+ it is already listed in `$(tap_with_common_setup_tests)'.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: one more use of TAP in our own tests
+ * tests/tap-bad-prog.test, tests/tap-bad-prog2.test: Merged
+ into ...
+ * tests/tap-bad-prog.tap: ... this TAP-generating test.
+ * tests/Makefile.am (tap_other_tests): Remove plan-bad-prog.test
+ and plan-bad-prog2.test.
+ (XFAIL_TESTS): Remove plan-bad-prog2.test.
+ (plan-bad-prog2.log): Remove.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ testsuite: scaffolding to allow use of TAP in our own tests
+ Now that Automake has initial support for the TAP test protocol,
+ we can start "eating our own dog food" and rewrite some tests to
+ use TAP; this should allow for better granularity, especially in
+ the control of skips and expected failures. With this change, we
+ set up the initial scaffolding required by the planned TAP tests,
+ and convert few older tests to use TAP, mostly in order to verify
+ that there are no obvious errors. The tests in our testsuite that
+ use the TAP protocol will be marked by the new `.tap' extension.
+ * tests/plain-functions.sh: New file containing definition of
+ helper shell functions used by the "protocol-less" tests of the
+ Automake testsuite.
+ * tests/tap-functions.sh: New file containing definition of
+ helper shell functions used by the TAP-producing tests of the
+ Automake testsuite.
+ * tests/defs: Remove definitions of some functions that are
+ now defined in `plain-functions.sh'. Instead, source one of
+ `plain-functions.sh' or `tap-functions.sh', depending on whether
+ `$use_tap' is set to "no" or "yes".
+ (exit trap): Call `late_plan_' if `$use_tap' is true. Also,
+ unset shell traces before issuing the latest commands, to avoid
+ confusing the tap driver with spurious output.
+ * tests/defs-static.in ($use_tap): New variable, by default set
+ to "yes" if the calling test script has a `.tap' suffix, and to
+ "no" otherwise. The individual scripts can override it though.
+ In code sanity-checking the environment, verify that `$use_tap'
+ is not exported.
+ * tests/self-check-env-sanitize.test: Update, and small related
+ reformatting.
+ * tests/self-check-tap.test: New very minimal self test.
+ * tests/acoutbs.test, tests/acoutbs2.test, tests/acoutnoq.test,
+ tests/acoutpt.test, tests/acoutpt2.test, tests/acoutqnl.test:
+ Removed, merged into ...
+ * tests/ac-output-old.tap: ... this new test, that uses TAP.
+ * tests/Makefile.am (TAP_LOG_DRIVER): Define to invoke our own
+ `tap-driver' script.
+ (TAP_LOG_DRIVER_FLAGS): Define to `--merge', so that the stdout
+ and stderr of the test scripts remains synced.
+ (TEST_EXTENSIONS): Define, to add the `.tap' suffix; also list
+ the `.test' suffix explicitly.
+ (EXTRA_DIST): Distribute the new files `plain-functions.sh' and
+ `tap-functions.sh'.
+ ($(TEST_LOGS)): Depends on them.
+ (AM_TESTS_ENVIRONMENT): Ensure that `use_tap' is not exported in
+ the environment of the tests.
+ (TESTS): Update.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ * NEWS: Fix typo, and related reformatting.
+
+2011-08-03 Stefano Lattarini <address@hidden>
+
+ * NEWS: add mention of new experimental TAP support
+
+2011-08-02 Stefano Lattarini <address@hidden>
+
+ testsuite: rename some test scripts
+ * tests/parallel-tests-am_tests_environment.test: Renamed ...
+ * tests/am-tests-environment.test: ... to this.
+ * tests/check-tests_environment.test: Renamed ...
+ * tests/tests-environment.test: ... to this.
+ * tests/parallel-test-driver-install.test: Renamed ...
+ * tests/parallel-tests-driver-install.test: ... to this.
+ * tests/parallel-tests-make-n.test: Renamed ...
+ * tests/parallel-tests-dry-run.test: ... to this.
+ * tests/Makefile.am (TESTS): Updated.
+
+2011-08-02 Stefano Lattarini <address@hidden>
+
+ coverage: parallel-tests and lazy dependencies on EXTRA_PROGRAMS
+ * tests/parallel-tests-extra-programs.test: New test.
+ * tests/Makefile.am (TESTS): Update.
+ * doc/automake.texi (Parallel Test Harness): Add a comment
+ pointing to the new test.
+
+2011-08-02 Stefano Lattarini <address@hidden>
+
+ docs: improve, extend and fix documentation on TAP support
+ * doc/automake.texi ("Using the TAP test protocol"): Divide this
+ section into ...
+ ("Introduction to TAP", "Use TAP with the Automake test harness",
+ "Incompatibilities with other TAP parsers and drivers", "Links
+ and external resources"): ... these subsections, extend them by
+ adding more information and examples, and improve them by removing
+ incomplete and/or temporary wordings and TODO items.
+ ("Script-based Testsuites", "Parallel Test Harness"): Add a couple
+ of anchors to improve the granularity of cross-references.
+ * tests/tap-doc2.test: New test, verifying the correctness of the
+ new examples given in the manual.
+ * tests/Makefile.am (tap_other_tests): Add the new test.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ testsuite: separate the only failing check of an xfailing test
+ * tests/plan-bad-prog.test: Move the only failing check of this
+ test (i.e., the one about the total number of "ERROR" outcomes)
+ into ...
+ * tests/plan-bad-prog2.test: ... this new test.
+ * tests/Makefile.am (XFAIL_TESTS): Remove `plan-bad-prog.test',
+ add `plan-bad-prog2.test'.
+ (tap_other_tests): Add `plan-bad-prog2.test'.
+ (plan-bad-prog2.log): Depend on `plan-bad-prog.test'.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ testsuite: fix spurious errors in an xfailing test on TAP support
+ * tests/plan-bad-prog.test: Fix typo in the name of the test
+ being made unreadable. Correct the wording of the potential
+ skip message, and simplify the condition under which the test
+ is to be skipped. Escape literal dots in grep regexps.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ testsuite: better granularity in a couple of tests on TAP support
+ * tests/tap-summary-aux.sh: New auxiliary script, filled with code
+ moved out from ...
+ * tests/tap-summary.test: ... this test, from which the checks on
+ colored testsuite have further been moved into ...
+ * tests/tap-summary-color.test: ... this new test, which in turn
+ * tests/Makefile.am (tap_other_tests): Add `tap-summary-color.test'.
+ (EXTRA_DIST): Distribute `tap-summary-aux.sh'
+ (tap-summary.log): Depend on `tap-summary-aux.sh'.
+ (tap-summary-color.log): Likewise.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ testsuite: optimize tests on TAP for speed and against duplication
+ The creation and configuration of common files and data used by
+ many tests on TAP is, with this change, factored out into a new
+ dedicated auxiliary test, from which those tests will depend upon.
+ This reduces code duplication in tests and, more importantly,
+ offers a noticeable speedup in the involved tests (30-40%).
+ * tests/tap-common-setup.test: New test, setting up the common
+ files and data used by various tests on TAP support.
+ * tests/tap-setup.sh: New file, to be sourced by tests wanting to
+ bring in data generated by `tap-common-setup.test'.
+ * tests/tap-autonumber.test: Update to use the pre-computed data
+ files.
+ * tests/tap-bailout.test: Likewise.
+ * tests/tap-color.test: Likewise.
+ * tests/tap-deps.test: Likewise.
+ * tests/tap-diagnostic.test: Likewise.
+ * tests/tap-empty-diagnostic.test: Likewise.
+ * tests/tap-empty.test: Likewise.
+ * tests/tap-escape-directive.test: Likewise.
+ * tests/tap-exit.test: Likewise.
+ * tests/tap-signal.test: Likewise.
+ * tests/tap-fancy.test: Likewise.
+ * tests/tap-fancy2.test: Likewise.
+ * tests/tap-global-log.test: Likewise.
+ * tests/tap-global-result.test: Likewise.
+ * tests/tap-html.test: Likewise.
+ * tests/tap-log.test: Likewise.
+ * tests/tap-merge-stdout-stderr.test: Likewise.
+ * tests/tap-no-merge-stdout-stderr.test: Likewise.
+ * tests/tap-message-0.test: Likewise.
+ * tests/tap-no-disable-hard-error.test: Likewise.
+ * tests/tap-no-spurious-summary.test: Likewise.
+ * tests/tap-no-spurious.test: Likewise.
+ * tests/tap-not-ok-skip.test: Likewise.
+ * tests/tap-numeric-description.test: Likewise.
+ * tests/tap-out-of-order.test: Likewise.
+ * tests/tap-passthrough.test: Likewise.
+ * tests/tap-passthrough-exit.test: Likewise.
+ * tests/tap-plan.test: Likewise.
+ * tests/tap-plan-corner.test: Likewise.
+ * tests/tap-plan-corner2.test: Likewise.
+ * tests/tap-plan-errors.test: Likewise.
+ * tests/tap-realtime.test: Likewise.
+ * tests/tap-recheck-logs.test: Likewise.
+ * tests/tap-skip-whole.test: Likewise.
+ * tests/tap-todo-skip-together.test: Likewise.
+ * tests/tap-todo-skip-whitespace.test: Likewise.
+ * tests/tap-todo-skip.test: Likewise.
+ * tests/tap-unplanned.test: Likewise.
+ * tests/tap-with-and-without-number.test: Likewise.
+ * tests/tap-xfail-tests.test: Likewise.
+ * tests/tap-skip-whole-whitespace.test: Likewise, and remove
+ redundant definitions of `$sp' and `$tab' (they are already
+ defined in `tests/defs').
+ * tests/tap-whitespace-normalization.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+ (EXTRA_DIST): Distribute `tap-setup.sh'.
+ (tap_with_common_setup_tests, tap_with_common_setup_logs)): New
+ variables, holding respectively the list of tests using the files
+ pre-computed by `tap-common-setup.test', and the list of their
+ corresponding log files.
+ (tap_other_tests): New variable, holding the list of other tests
+ on TAP support.
+ ($(tap_with_common_setup_logs)): Depend on `tap-common-setup.log'
+ and `tap-setup.sh'.
+
+2011-08-01 Stefano Lattarini <address@hidden>
+
+ test harness: use new `.trs' files to hold test metadata
+ With this change, the test harness will keep test metadata in
+ dedicated `.trs' files, instead of having them embedded into the
+ `.log' files. This allows for easier forward-compatibility and
+ extension of test metadata, and for more flexibility in the
+ format of the `.log' files. Note that this change makes the
+ `:end-metadata:' field obsolete.
+ * doc/automake.texi (Parallel Test Harness, Log files generation
+ and test results recording): Document the new APIs and behaviour;
+ some related minor rewordings and fixlets.
+ * NEWS: Update.
+ * automake.in (handle_tests): When bringing in the content of
+ `check2.am', substitute %BASE% with the basename of the `.log'
+ file being created by a rule. Add the generated `.trs' files
+ to the list of files to be cleaned by "make mostlyclean".
+ * lib/am/check.am (am__test_driver_flags): Rename ...
+ (am__common_driver_flags): ... to this, and remove the flags
+ `--test-name' and `--log-file' from it: they are now define in
+ the proper rules in `check2.am'.
+ (am__TEST_BASES): New internal variable, holding the names of
+ the tests, with any registered extension removed.
+ (am__stealth_MAKE): New internal variable, can be used instead of
+ $(MAKE) in recipes requiring a recursive call to make, but which
+ are not intended to be executed by "make -n".
+ (.log.trs): New suffix rule, to recover from deletion of `.trs'
+ files.
+ ($(TEST_SUITE_LOG)): Almost completely rewritten to follow the
+ new API of "test logs in `.log' files, test metadata in `.trs'
+ files". It goes to some length to work correctly in face of
+ unreadable or missing `.log' and `.trs' files, and to error out
+ with proper error messages when this is not possible.
+ [%?PARALLEL_TESTS%] (check-TESTS): Also remove relevant "stale"
+ `.trs' files (in addition to `.log files) before remaking the
+ $(TEST_SUITE_LOG).
+ (recheck, recheck-html): Look for the `:recheck:' field in the
+ `.trs' files, not in the `.log' files.
+ * lib/am/check2.am (?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Adjust
+ the call to the test driver, in particularly passing the new
+ option `--trs-file'.
+ [%am__EXEEXT%] (?GENERIC?%EXT%$(EXEEXT).log): Likewise.
+ * lib/tap-driver ($USAGE): Adjust the help screen.
+ (Getopt::Long::GetOptions): Handle the `--trs-file' option,
+ through the use of ...
+ ($trs_file): ... this new global variable.
+ (finish): Write metadata for the test run to `$trs_file' rather
+ then to `$log_file', through the use of ...
+ (write_test_results): ... this new function.
+ * lib/test-driver (print_usage): Update the help screen.
+ (Option parsing): Handle the `--trs-file' option, through the
+ use of ...
+ ($resfile): ... this new global variable.
+ (Main code): Write metadata for the test run to `$trsfile' rather
+ than to `$logfile'.
+ Minor related adjustments to comments.
+ * tests/.gitignore: Ignore `*.trs' files.
+ * tests/parallel-tests-unreadable-log.test: Moved ...
+ * tests/parallel-tests-unreadable.test: ... to this, and extended
+ to also check the semantics for unreadable `.trs' files.
+ * tests/test-driver-end-metadata.test: Deleted as obsolete.
+ * tests/test-driver-metadata-no-leading-space.test: Likewise.
+ * tests/test-driver-global-log.test: Renamed ...
+ * tests/test-metadata-global-log.test: ... to this, and modified
+ as to verify the new APIs and semantics.
+ * tests/test-driver-recheck: Renamed ...
+ * tests/test-metadata-recheck.test: ... to this, and modified
+ likewise.
+ * tests/parallel-tests-once.test: New test.
+ * tests/parallel-tests-make-n.test: Likewise.
+ * test-metadata-results.test: Likewise.
+ * test-missing.test: Likewise.
+ * test-missing2.test: Likewise.
+ * test-trs-basic.test: Likewise.
+ * test-trs-recover.test: Likewise.
+ * test-trs-recover2.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-27 Stefano Lattarini <address@hidden>
+
+ test harness: allow more metadata in log files
+ This change reworks and improves the parallel test harness to use
+ more specialized reStructuredText fields in the log files (instead
+ of relying on specially-placed of "magic lines" and more indirect
+ semantical formatting); the new fields are the following:
+ - ":recheck:": tell whether the associated test will have to be
+ re-run by "make recheck";
+ - ":copy-in-global-log:": tell whether the content of the log
+ file should be copied in the "global log" `test-suite.log';
+ - ":end-metadata:", which inhibits the scanning of the rest of
+ the log file (for what concerns test metadata).
+ Also, the special `:test-result:' value "END" has been removed,
+ superseded by the new `:end-metadata:' field.
+ * doc/automake.texi (Log files generation and test results
+ recording): Document the new API and semantics. Remove or fix
+ some obsolete comments.
+ * lib/am/check.am ($(TEST_SUITE_LOG), recheck, recheck-html):
+ Adjust comments and code.
+ * lib/tap-driver (finish): Adjust, with the help of ...
+ (must_recheck, copy_in_global_log): ... these new functions.
+ * lib/test-driver (Main code): Adjust, with the help of ...
+ ($recheck, $gcopy): ... these new variables.
+ * tests/trivial-test-driver: Update to obey the new APIs.
+ * tests/test-driver-recheck.test: Likewise.
+ * tests/test-driver-global-log.test: Likewise.
+ * tests/tap-passthrough.test: Relax the test, avoiding to check
+ what is written in `test-suite.log'; such check has been moved ...
+ * tests/tap-global-log.test: ... in this new test, and extended.
+ * tests/test-driver-metadata-no-leading-space.test: New test.
+ * tests/test-driver-end-test-results.test: Removed, it checked
+ the old APIs; superseded by ...
+ * tests/test-driver-end-metadata.test: ... this new test.
+ * tests/tap-log.test: Improve syncing with ...
+ * tests/test-log.test: ... this new test.
+ * tests/parallel-tests.test: Remove some duplication w.r.t. this
+ last new test. Updated heading comments.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-27 Stefano Lattarini <address@hidden>
+
+ testsuite: fix a spurious failure with non-bash shells
+ * tests/tap-fancy2.test: Remove an unportable use of backslashes
+ with the `echo' builtin, which was causing some shells (among them
+ bash and Debian /bin/ksh) to print a `\\' string, while other
+ shells (among them zsh, dash, and Solaris /bin/sh and /bin/ksh)
+ were unexpectedly printing a single `\' character. Since we are
+ at it, add a sanity check to ensure that this issue does not
+ resurface.
+
+2011-07-27 Stefano Lattarini <address@hidden>
+
+ testsuite: fix spurious failures with Solaris /bin/sh
+ * tests/tap-more.test: Use `echo > file', not `: > file', to
+ create empty files in the "for" loops; this is required since,
+ as documented in the autoconf manual, Solaris 10 /bin/sh
+ "optimizes" away the `:' command after the first iteration,
+ even if it is redirected.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/tap-numeric-description.test: Partial rewrite to avoid
+ using positional parameters from the 10th onward, which are
+ unportable to Solaris /bin/sh (using `${10}' causes the shell
+ to die with "bad substitution").
+
+2011-07-31 Stefano Lattarini <address@hidden>
+
+ simple tests: support developer-defined fd redirections
+ Motivated by coreutils bug#8846, and related discussions:
+ <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8846>
+ <http://lists.gnu.org/archive/html/bug-autoconf/2011-06/msg00002.html>
+ In those threads it has been shown how problematic it can be to do
+ portable file descriptor redirections in the testsuite when using
+ the Automake testsuite harness. This change should remedy to that
+ situation.
+ * lib/am/check2.am (?GENERIC?%EXT%$(EXEEXT).log,
+ ?GENERIC?%EXT%.log, ?!GENERIC?%OBJ%): Append redirection defined
+ in $(AM_TESTS_FD_REDIRECT) to the command-line invocations of the
+ test scripts.
+ * lib/am/check.am [!%?PARALLEL_TESTS%] $(check-TESTS): Likewise.
+ * NEWS: Update.
+ * doc/automake.texi (Script-based Testsuites): Document the new
+ feature.
+ * tests/check-fd-redirect.test: New test.
+ * tests/parallel-tests-fd-redirect.test: Likewise.
+ * tests/parallel-tests-am_tests_environment.test: Remove checks
+ about the use of redirections in AM_TESTS_ENVIRONMENT: they would
+ check deprecated (if not undefined) behaviour now. Strengthen a
+ couple of still valid checks, to keep the test more in sync with
+ the documentation. Improve debugging information.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-22 Stefano Lattarini <address@hidden>
+
+ docs: rework and extend documentation on testsuites support
+ * doc/automake.texi (Generalities about Testing): New section.
+ (Simple Tests using parallel-tests): Section removed, merged
+ into ...
+ (Simple Tests): ... this one, which on the other hand has been
+ subdivided into ...
+ (Script-based Testsuites, Serial Test Harness, Parallel Test
+ Harness): ... these new subsection.
+ (DejaGnu Tests): Minor adjustments.
+ Other related typofixes and rewordings throughout the manual;
+ in particular, avoid to use the term "test driver" for three
+ different concepts (and use instead "test harness" and "test
+ runner" where appropriate).
+ * tests/tap-doc.test: New test.
+ * tests/tap-no-disable-hard-error.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-18 Stefano Lattarini <address@hidden>
+
+ tap: support colorization of testsuite progress output
+ * lib/tap-driver (%COLORS): New variable (definition extracted
+ from `lib/am/check.am:$(am__tty_colors)', with some obvious
+ adjustments.
+ (report): Adjust to colorize console output when required,
+ using ...
+ (decorate_result): ... this new function.
+ (colored): New function, used by the one above.
+ * tests/tap-summary.test: Also run the checks when `color-tests'
+ is in use.
+ * tests/Makefile.am (XFAIL_TESTS): Remove `tap-color.test'.
+
+2011-07-18 Stefano Lattarini <address@hidden>
+
+ tap: some preparatory refactoring (2)
+ This is a follow-up simplification.
+ * lib/tap-driver (console_output): Renamed ...
+ (report): ... to this, and extended to appropriately register
+ the test results when needed.
+ (testsuite_error, handle_tap_comment, handle_tap_test,
+ handle_tap_plan): Adjusted accordingly.
+
+2011-07-18 Stefano Lattarini <address@hidden>
+
+ tap: some preparatory refactoring (1)
+ This refactoring is aimed at simplifying the introduction of
+ colored console output for the TAP driver.
+ * lib/tap-driver (console_output): Now accept two arguments, the
+ first one indicating which kind of thing is to be displayed (for
+ now only a test result or a diagnostic comment), and the second
+ one (if present) the message associated to it.
+ (handle_tap_test, handle_tap_comment, handle_tap_plan,
+ testsuite_error): Adapt to the new `console_output' interface.
+
+2011-07-18 Stefano Lattarini <address@hidden>
+
+ tap: add experimental TAP-aware driver
+ * doc/automake.texi (Using the TAP test protocol): New section.
+ (Overview of Custom Test Drivers Support): Minor updates.
+ * lib/tap-driver: New script, TAP-aware test driver for Automake;
+ implemented in perl and based on TAP::Parser.
+ * lib/Makefile.am (dist_script_DATA): Add it.
+ * tests/tap-autonumber.test: New test.
+ * tests/tap-bailout.test: Likewise.
+ * tests/tap-basic.test: Likewise.
+ * tests/tap-deps.test: Likewise.
+ * tests/tap-diagnostic.test: Likewise.
+ * tests/tap-empty.test: Likewise.
+ * tests/tap-empty-diagnostic.test: Likewise.
+ * tests/tap-escape-directive.test: Likewise.
+ * tests/tap-exit.test: Likewise.
+ * tests/tap-fancy.test: Likewise.
+ * tests/tap-fancy2.test: Likewise.
+ * tests/tap-global-result.test: Likewise.
+ * tests/tap-html.test: Likewise.
+ * tests/tap-log.test: Likewise.
+ * tests/tap-merge-stdout-stderr.test: Likewise.
+ * tests/tap-more.test: Likewise.
+ * tests/tap-more2.test: Likewise.
+ * tests/tap-no-merge-stdout-stderr.test: Likewise.
+ * tests/tap-no-spurious-summary.test: Likewise.
+ * tests/tap-no-spurious.test: Likewise.
+ * tests/tap-not-ok-skip.test: Likewise.
+ * tests/tap-numeric-description.test: Likewise.
+ * tests/tap-out-of-order.test: Likewise.
+ * tests/tap-passthrough.test: Likewise.
+ * tests/tap-plan.test: Likewise.
+ * tests/tap-plan-errors.test: Likewise.
+ * tests/tap-plan-corner.test: Likewise.
+ * tests/tap-realtime.test: Likewise.
+ * tests/tap-recheck-logs.test: Likewise.
+ * tests/tap-recheck.test: Likewise.
+ * tests/tap-skip-whole.test: Likewise.
+ * tests/tap-summary.test: Likewise.
+ * tests/tap-todo-skip.test: Likewise.
+ * tests/tap-todo-skip-together.test: Likewise.
+ * tests/tap-todo-skip-whitespace.test: Likewise.
+ * tests/tap-skipall-whitespace.test: Likewise.
+ * tests/tap-unplanned.test: Likewise.
+ * tests/tap-whitespace-normalization.test: Likewise.
+ * tests/tap-with-and-without-number.test: Likewise.
+ * tests/tap-xfail-tests.test: Likewise.
+ * tests/tap-bad-prog.test: New xfailing test.
+ * tests/tap-color.test: Likewise.
+ * tests/tap-plan-corner2.test: Likewise.
+ * tests/tap-message-0.test: Likewise.
+ * tests/tap-signal.test: Likewise.
+ * tests/Makefile.am (TESTS, XFAIL_TESTS): Update.
+
+2011-07-07 Stefano Lattarini <address@hidden>
+
+ test defs: new auxiliary function 'count_test_results'
+ * tests/defs (count_test_results): New function.
+ * tests/check11.test: Use it.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-no-spurious-summary.test: Likewise, and
+ slightly improve debugging output.
+ * tests/parallel-tests.test: Make use of `count_test_results'.
+ Also, make grepping of "make check" output slightly stricter
+ * tests/parallel-tests9.test: Likewise.
+ * tests/parallel-tests-log-override-2.test: Likewise, and throw
+ in a small optimization.
+
+2011-07-07 Stefano Lattarini <address@hidden>
+
+ parallel-tests: simplify testsuite summary
+ Prefer a more deterministic, "tabular" format for the testsuite
+ summary, always listing the numbers of passed, failed, xfailed,
+ xpassed, skipped and errored tests, even when these numbers are
+ zero. This simplify the logic of testsuite summary creation,
+ makes it more easily machine-parseable, and will probably allow
+ for easier addition of new kinds of test results in the future.
+ * lib/am/check.am (am__tty_colors_dummy): New make variable, to
+ reduce code duplication. Extracted from previous versions of
+ $(am__tty_colors), and extended by defining two new variables
+ `$mgn' and `$brg'.
+ [%?COLOR%, %!?COLOR%] (am__tty_colors): Use that new variable.
+ (am__text_box): Delete, is not needed anymore.
+ ($(TEST_SUITE_LOG)): Rewrite associated rules to implement the
+ new testsuite summary format.
+ * NEWS: Update.
+ * tests/check10.test: Don't run with the parallel-tests harness
+ too, that makes no sense anymore.
+ * tests/color.test: Update and adjust.
+ * tests/color2.test: Likewise.
+ * tests/parallel-tests.test: Likewise.
+ * tests/parallel-tests3.test: Likewise.
+ * tests/parallel-tests6.test: Likewise.
+ * tests/parallel-tests9.test: Likewise.
+ * tests/parallel-tests-unreadable-log.test: Likewise.
+ * tests/parallel-tests-empty-testlogs.test: Likewise.
+ * tests/parallel-tests-log-override-recheck.test: Likewise.
+ * tests/parallel-tests-no-spurious-summary.test: Likewise.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-end-test-results.test: Likewise.
+ * tests/parallel-tests-no-color-in-log.test: New test.
+ * tests/testsuite-summary-color.test: Likewise.
+ * tests/testsuite-summary-count.test: Likewise.
+ * tests/testsuite-summary-count-many.test: Likewise.
+ * tests/testsuite-summary-reference-log.test: Likewise.
+ * tests/testsuite-summary-checks.sh: New auxiliary script, used
+ by the new tests above.
+ * tests/extract-testsuite-summary: Likewise.
+ * tests/trivial-test-driver: Optimize for speed when there are
+ lots of of tests.
+ * tests/Makefile.am (EXTRA_DIST): Distribute them.
+ (testsuite-summary-color.log, testsuite-summary-count.log): Depend
+ on them.
+ (testsuite-summary-count-many.log): Depend on the auxiliary scripts
+ 'trivial-test-driver' and 'extract-testsuite-summary'.
+ (TESTS): Update.
+
+2011-07-07 Stefano Lattarini <address@hidden>
+
+ parallel-tests: new recognized test result 'ERROR'
+ * lib/am/check.am ($(TEST_SUITE_LOG)): Recognize a new test result
+ `ERROR'. Use it when encountering unreadable test logs (previously
+ a simple `FAIL' was used in this situations).
+ * lib/test-driver: Set the global test result to `ERROR' when the
+ test exit status is 99. When doing colorized output, color `ERROR'
+ results in magenta.
+ * doc/automake.texi (Log files generation and test results
+ recording): Update by listing `ERROR' too among the list of valid
+ `:test-results:' arguments.
+ * NEWS: Update.
+ * tests/trivial-test-driver: Update.
+ * tests/parallel-tests.test: Likewise.
+ * tests/parallel-tests-harderror.test: Likewise.
+ * tests/parallel-tests-no-spurious-summary.test: Likewise.
+ * tests/test-driver-global-log.test: Likewise.
+ * tests/test-driver-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-custom-no-html.test: Likewise.
+ * tests/test-driver-end-test-results.test: Likewise.
+ * tests/color.test: Likewise. Also, make stricter, and also test
+ from VPATH.
+ * tests/color2.test: Likewise, and improve syncing with color.test.
+ * tests/parallel-tests-exit-statuses.test: New test.
+ * tests/parallel-tests-console-output.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-07-07 Stefano Lattarini <address@hidden>
+
+ parallel-tests: make parsing of test results safer
+ The new code for parsing the testsuite-generated `.log' files,
+ as introduced in commit `v1.11-872-gc96b881', considers each
+ `:test-result:' field anywhere in a `.log' file as a declaration
+ of a test result, and accounts for it as such in the testsuite
+ summary. Unfortunately this could easily cause spurious test
+ failures being reported in the testsuite summary. This happened
+ in practice with the Automake's own testsuite; for example:
+ $ make check TESTS='check12-p.test'; echo exit: $?
+ ...
+ PASS: check12-p.test
+ =====================================
+ 4 of 5 tests failed
+ See tests/test-suite.log
+ Please report to address@hidden
+ =====================================
+ make[2]: *** [test-suite.log] Error 1
+ make: *** [check-am] Error 2
+ exit: 2
+ This change introduces a new special `:test-result:' "END", that,
+ when seen, prevents the rest of the log file from being parsed.
+ For more information, refer to the thread:
+
<http://lists.gnu.org/archive/html/automake-patches/2011-06/msg00199.html>
+ * lib/am/check.am ($(TEST_SUITE_LOG)): Stop the parsing of a log
+ file as soon as the special ":test-result:END" directive is seen.
+ Related changes and enhancements.
+ * lib/test-driver: Protect the rest of the log after the result
+ lined with a ":test-result:END" directive.
+ * doc/automake.texi (Log files generation and test results
+ recording): Update, and related improvements.
+ * tests/parallel-tests-no-spurious-summary.test: New test.
+ * tests/test-driver-end-test-results.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
+2011-06-29 Stefano Lattarini <address@hidden>
+
+ docs: document custom test drivers and protocols
+ * doc/automake.texi (Simple Tests): Note that the TESTS_ENVIRONMENT
+ use suggested here is not portable to 'parallel-tests'.
+ (Simple Tests using parallel-tests): Document new restrictions on
+ the uses of TESTS_ENVIRONMENT and AM_TESTS_ENVIRONMENT.
+ (Custom Test Drivers): New section and node.
+ (Overview of Custom Test Drivers Support): New subsection.
+ (Declaring Custom Test Drivers in @file{Makefile.am}): Likewise.
+ (APIs for Custom Test Drivers): Likewise.
+ (Options): Update description of color-tests.
+ * lib/am/check ($(TEST_SUITE_LOG)): Remove comments that have been
+ moved into the manual.
+ (recheck, recheck-html): Minor adjustments to better conform to the
+ documentation (this should cause no semantic changes w.r.t. the
+ former behaviour); minor improvements and extensions to existing
+ comments.
+ * tests/test-driver-create-log-dir.test: New test.
+ * tests/test-driver-strip-vpath.test: Likewise.
+ * tests/test-driver-global-log.test: Likewise.
+ * tests/test-driver-recheck.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+
2011-07-24 Stefano Lattarini <address@hidden>
tests: small enhancement to 'primary3.test'
@@ -164,20 +1773,6 @@
2011-07-08 Stefano Lattarini <address@hidden>
- tests: fix weakness in 'tests-environment-backcompat.test'
- * tests/tests-environment-backcompat.test: Do not override the
- content of xfailing test `baz.test' with a "weaker" version that
- fails unconditionally: the test must fail only when the 'strict'
- pragma is in use, in order not to reduce coverage.
-
-2011-07-08 Stefano Lattarini <address@hidden>
-
- docs, tests: synchronize examples on silent-rules from config.site
- * doc/automake.texi (Automake silent-rules Option): Reference test
- 'silent-configsite.test' in comments.
-
-2011-07-08 Stefano Lattarini <address@hidden>
-
tests defs: more uses of $top_testsrcdir
* tests/autodist-stamp-vti.test: Use `$top_testsrcdir' instead
of `$testsrcdir/..'.
@@ -222,60 +1817,6 @@
* tests/silent-configsite.test: Likewise.
* tests/tests-environment-backcompat.test: Likewise.
-2011-07-08 Stefano Lattarini <address@hidden>
-
- tests: portability fixes in tests on amhello examples
- * tests/amhello-binpkg.test: Don't use "tar xzf too.tag.gz" to
- extract a gzip-compressed tarball, that's unportable to some
- tar implementations; use the "gzip -dc fo.tar.gz | tar xf -"
- idiom instead.
- * tests/amhello-cflags.test: Likewise.
- * tests/amhello-cross-compile.test: Likewise.
- Suggestion from Ralf Wildenhues.
-
-2011-07-04 Stefano Lattarini <address@hidden>
-
- remake: add test ensuring that slower remakes don't hang
- * tests/remake-subdir-long-time.test: New test.
- * tests/Makefile.am (TESTS): Update.
- Suggestion by Ralf Wildenhues.
-
-2011-07-04 Stefano Lattarini <address@hidden>
-
- remake: fix outdated comment in configure.am
- * lib/am/configure.am: Fix comment falsified by changes in
- commit `v1.11-366-gbee9871'.
- Suggestion by Ralf Wildenhues.
-
-2011-07-01 Stefano Lattarini <address@hidden>
-
- docs, tests: synchronize examples from docs to tests
- * tests/README (Writing test cases): Give suggestions on how to
- keep test cases and examples in the documentation synchronized.
- * doc/automake.texi: Improve or fix existing testcase-referencing
- comments, and add many new ones.
- * HACKING (Administrivia): Suggest to test complex examples and
- idioms from the manual.
- * tests/specflg8.test: Improve synchronization with the example
- in the manual.
- * tests/output11.test:Likewise.
- * tests/txinfo21.test:Likewise.
- * tests/interp.test: Likewise.
- * tests/amhello-cflags.test: New test.
- * tests/amhello-cross-compile.test: Likewise.
- * tests/amhello-binpkg.test: Likewise.
- * tests/tests-environment-backcompat.test: Likewise.
- * tests/parallel-tests-log-compiler-example.test: Likewise.
- * tests/Makefile.am (TESTS): Update.
-
-2011-06-30 Stefano Lattarini <address@hidden>
-
- coverage: new test on parallel-tests TESTS runtime overriding
- * tests/parallel-tests-cmdline-override.test: New test, check that
- we can use indirections when overriding TESTS and TEST_LOGS from
- the command line.
- * tests/Makefile.am (TESTS): Update.
-
2011-06-29 Stefano Lattarini <address@hidden>
docs: explain why AM_TESTS_ENVIRONMENT must be semicolon-terminated
@@ -344,6 +1885,141 @@
2011-06-21 Stefano Lattarini <address@hidden>
+ parallel-tests: allow each test to have multiple results
+ With this change, we improve the code creating the `test-suite.log'
+ global log and the console testsuite summary to make it able to
+ grasp multiple results per test script. This is required in order
+ to introduce the planned support for test protocols, like TAP and
+ SubUnit, which can indeed run multiple tests per test script, each
+ with its individual result.
+ The implementation makes use of a custom reStructuredText field
+ `:test-result:'.
+ Note that no new documentation is added by this change; that is
+ be left for follow-up changes.
+ * lib/check.am ($(TEST_SUITE_LOG)): When processing .log files,
+ recognize a report of a test's result only if it is declared with
+ the custom `:test-result:' reStructuredText field placed at the
+ beginning of a line. Extend and add explanatory comments.
+ (recheck, recheck-html): Add explanatory comments.
+ * lib/test-driver: Write an appropriate reStructuredText field
+ `:test-result:' in the generated log file. Use a reStructuredText
+ transition to better separate the test outcome report from the
+ test script's registered output. Improve comments.
+ * tests/test-driver-custom-xfail-tests.test: Adjust.
+ * tests/parallel-tests7.test: Adjust.
+ * tests/parallel-tests-empty-testlogs.test: New test.
+ * tests/parallel-tests-recheck-override.test: Likewise.
+ * tests/parallel-tests2.test: Extend and keep more in-sync with ...
+ * tests/test-driver-custom-html.test: ... this new related test.
+ * tests/test-driver-custom-no-html.test: New test.
+ * tests/test-driver-custom-multitest.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck.test: Likewise.
+ * tests/test-driver-custom-multitest-recheck2.test: Likewise.
+ * tests/trivial-test-driver: New file, used by the last four tests
+ above.
+ * tests/Makefile.am (TESTS): Update.
+ (EXTRA_DIST): Distribute `trivial-test-driver'.
+ (test-driver-custom-multitest.log): Depend on `trivial-test-driver'.
+ (test-driver-custom-multitest-recheck.log): Likewise.
+ (test-driver-custom-multitest-recheck2.log): Likewise.
+ (test-driver-custom-html.log): Likewise.
+
+2011-06-21 Stefano Lattarini <address@hidden>
+
+ parallel-tests: allow custom driver scripts
+ Allow suffix-based definition of custom "driver script" for the
+ test scripts. These driver scripts will be responsible of
+ launching the tests (or their corresponding $(LOG_COMPILER), if
+ they have an associated one), interpreting and displaying the
+ test results, and writing the `.log' files.
+ This new API should allow easy and flexible use of different
+ test protocols in the future; in particular, we plan to use it
+ to implement TAP and SubUnit harnesses.
+ Note that no new documentation is added by this change; that is
+ be left for follow-up changes.
+ * automake.in (handle_tests): Define default for $(LOG_DRIVER),
+ and, for any registered test extension `<ext>', define defaults
+ for $(<ext>_LOG_DRIVER). Substitute %DRIVER% using these new
+ variables, instead of the old internal $(am__test_driver). When
+ processing check2.am, also substitute %DRIVER_FLAGS%.
+ Require auxiliary script `test-driver' only if no driver has been
+ explicitly defined for the test script kinds.
+ * am/check2.am (?GENERIC?%EXT%$(EXEEXT).log, ?GENERIC?%EXT%.log,
+ ?!GENERIC?%OBJ%): Pass the %DRIVER_FLAGS% to the %DRIVER% call.
+ * tests/parallel-tests-no-extra-driver.test: New test.
+ * tests/test-driver-custom.test: Likewise.
+ * tests/test-driver-custom-xfail-tests.test: Likewise.
+ * tests/test-driver-fail.test: Likewise.
+ * tests/Makefile.am: Update.
+ * NEWS: Update.
+
+2011-06-21 Stefano Lattarini <address@hidden>
+
+ parallel-tests: add auxiliary script 'test-driver', refactor
+ This refactoring should cause no API of functionality change,
+ and is meant only to simplify the future implementation of TAP
+ and SubUnit testsuite drivers. More precisely, our roadmap is
+ to move most of the "testsuite driving" features out of the
+ Automake-generated Makefiles, and into external scripts with
+ well-defined interfaces. This will allow the user to define
+ its own personalized testsuite drivers, and will also offer us
+ a framework upon which to implement our new TAP and SubUnit
+ drivers, all in a very unobtrusive way and retaining an high
+ degree of code reuse and backward-compatibility.
+ * lib/test-driver: New auxiliary script.
+ * lib/Makefile.am (dist_SCRIPT_DATA): Add it.
+ * automake.in (handle_tests): Require the new auxiliary script
+ `test-driver', and define a new internal makefile variable
+ `$(am__test_driver)', used to call it. Perform new substitution
+ on `DRIVER' when processing the `check2.am' file.
+ * lib/check.am (am__tty_colors): Define new shell variable
+ `$am__color_tests'.
+ (am__rst_section): Removed, its role taken over by the new
+ `test-driver' script.
+ (am__test_driver_flags): New variable, contains the command
+ line options passed to `test-driver'.
+ (am__check_pre): Do not deal with temporary files and exit
+ traps anymore, as the `test-driver' script takes care of that
+ now. Define shell variable `$am__enable_hard_errors', used by
+ `$(am__test_driver_flags)'. Reorder so that we don't need to
+ save and restore the value of the `TERM' environment variable
+ anymore.
+ Other related adjustments.
+ (am__check_post): Remove, as its role has been completely taken
+ over by the `test-driver' script.
+ * am/check2.am (?GENERIC?%EXT%$(EXEEXT).log, ?GENERIC?%EXT%.log,
+ ?!GENERIC?%OBJ%): Call the test script through the Automake
+ substituted `%DRIVER%', and honor the command-line options
+ in `$(am__test_driver_flags)'. Do not call the obsoleted
+ `$(am__check_post)' anymore.
+ * doc/automake.texi (Auxiliary Programs): Mention the new
+ `test-driver' script.
+ (Optional): Mention `test-driver' in AC_CONFIG_AUX_DIR.
+ Since we are at it, break the list of auxiliary scripts by
+ placing one per line, to simplify potential future additions
+ of new scripts.
+ * tests/check.test: Adjust.
+ * tests/check2.test : Likewise.
+ * tests/check3.test : Likewise.
+ * tests/check4.test : Likewise.
+ * tests/check10.test: Likewise.
+ * tests/color.test: Likewise.
+ * tests/color2.test: Likewise.
+ * tests/comment9.test: Likewise.
+ * tests/dejagnu.test: Likewise.
+ * tests/exeext4.test: Likewise.
+ * tests/maken3.test: Likewise.
+ * tests/maken4.test: Likewise.
+ * tests/parallel-tests-interrupt.test: Likewise.
+ * tests/posixsubst-tests.test: Likewise.
+ * tests/repeated-options.test: Likewise.
+ * tests/check-no-test-driver.test: New test.
+ * tests/parallel-test-driver-install.test: Likewise.
+ * tests/Makefile.am (TESTS): Update.
+ * NEWS: Update.
+
+2011-06-21 Stefano Lattarini <address@hidden>
+
maintcheck: extend 'sc_tests_plain_*' checks
* Makefile.am (sc_tests_plain_autom4te): New check.
(sc_tests_plain_autoreconf): Likewise.
diff --git a/INSTALL b/INSTALL
index 81fd332..a1e89e1 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,8 +1,8 @@
Installation Instructions
*************************
-Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
-2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
+Copyright (C) 1994-1996, 1999-2002, 2004-2011 Free Software Foundation,
+Inc.
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright
@@ -226,6 +226,11 @@ order to use an ANSI C compiler:
and if that doesn't work, install pre-built binaries of GCC for HP-UX.
+ HP-UX `make' updates targets which have the same time stamps as
+their prerequisites, which makes it generally unusable when shipped
+generated files such as `configure' are involved. Use GNU `make'
+instead.
+
On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot
parse its `<wchar.h>' header file. The option `-nodtk' can be used as
a workaround. If GNU CC is not installed, it is therefore recommended
diff --git a/NEWS b/NEWS
index 35b223a..1c2f102 100644
--- a/NEWS
+++ b/NEWS
@@ -11,6 +11,78 @@ New in 1.11a:
- New `cscope' target to build a cscope database for the source tree.
+* Changes to Automake-generated testsuite harnesses:
+
+ - Test scripts that exit with status 99 to signal an "hard error" (e.g.,
+ and unexpected or internal error, or a failure to set up the test case
+ scenario) have their outcome reported as an 'ERROR' now. Previous
+ versions of automake reported such an outcome as a 'FAIL' (the only
+ difference with normal failures being that hard errors were counted
+ as failures even when the test originating them was listed in
+ XFAIL_TESTS).
+
+ - The testsuite summary displayed by the parallel-test harness has a
+ completely new format, that always list the numbers of passed, failed,
+ xfailed, xpassed, skipped and errored tests, even when these numbers
+ are zero (but using smart coloring when the color-tests option is in
+ effect).
+
+ - The default testsuite driver offered by the 'parallel-tests' option is
+ now implemented (partly at least) with the help of automake-provided
+ auxiliary scripts (e.g., `test-driver'), instead of relying entirely
+ on code in the generated Makefile.in.
+ This has two noteworthy implications. The first one is that projects
+ using the `parallel-tests' option should now either run automake with
+ the `--add-missing' option, or manually copy the `test-driver' script
+ into their tree. The second, and more important, implication is that
+ now, when the `parallel-tests' option is in use, TESTS_ENVIRONMENT can
+ not be used anymore to define a test runner, and the command specified
+ in LOG_COMPILER (and <ext>_LOG_COMPILER) must be a *real* executable
+ program or script. For example, this is still a valid usage (albeit
+ a little contorted):
+
+ TESTS_ENVIRONMENT = \
+ if test -n '$(STRICT_TESTS)'; then \
+ maybe_errexit='-e'; \
+ else \
+ maybe_errexit=''; \
+ fi;
+ LOG_COMPILER = $(SHELL) $$maybe_errexit
+
+ while this is not anymore:
+
+ TESTS_ENVIRONMENT = \
+ $(SHELL) `test -n '$(STRICT_TESTS_CHECKING)' && echo ' -e'`
+
+ neither is this:
+
+ TESTS_ENVIRONMENT = \
+ run_with_perl_or_shell () \
+ { \
+ if grep -q '^#!.*perl' $$1; then
+ $(PERL) $$1; \
+ else \
+ $(SHELL) $$1; \
+ fi; \
+ }
+ LOG_COMPILER = run_with_per_or_shell
+
+ - The package authors can now use customary testsuite drivers within
+ the framework provided by the 'parallel-tests' testsuite harness.
+ Consistently with the existing syntax, this can be done by defining
+ special makefile variables `LOG_DRIVER' and `<ext>_LOG_DRIVER'.
+
+ - A new developer-reserved variable `AM_TESTS_FD_REDIRECT' can be used
+ to redirect/define file descriptors used by the test scripts.
+
+ - The parallel-tests harness generates now, in addition the `.log' files
+ holding the output produced by the test scripts, a new set of `.trs'
+ files, holding "metadata" derived by the execution of the test scripts;
+ among such metadata are the outcomes of the test cases run by a script.
+
+ - Initial and still experimental support for the TAP test protocol is
+ now provided.
+
* WARNING: Future backward-incompatibilities!
- The Automake support for automatic de-ANSI-fication will be removed in
diff --git a/automake.in b/automake.in
index 517eea9..b8fdde3 100644
--- a/automake.in
+++ b/automake.in
@@ -4989,7 +4989,7 @@ sub handle_tests
append_exeext { exists $known_programs{$_[0]} } 'XFAIL_TESTS'
if (var ('XFAIL_TESTS'));
- if (option 'parallel-tests')
+ if (my $parallel_tests = option 'parallel-tests')
{
define_variable ('TEST_SUITE_LOG', 'test-suite.log', INTERNAL);
define_variable ('TEST_SUITE_HTML', '$(TEST_SUITE_LOG:.log=.html)',
INTERNAL);
@@ -5033,14 +5033,32 @@ sub handle_tests
return substr ($obj, 0, length ($obj) - length
($test_suffix)) . '.log'
if substr ($obj, - length ($test_suffix)) eq $test_suffix;
}
+ my $base = $obj;
$obj .= '.log';
+ # The "test driver" program, deputed to handle tests protocol
used by
+ # test scripts. By default, it's assumed that no protocol is
used,
+ # so we fall back to the old "parallel-tests" behaviour,
implemented
+ # by the `test-driver' auxiliary script.
+ if (! var 'LOG_DRIVER')
+ {
+ require_conf_file ($parallel_tests->{position}, FOREIGN,
+ 'test-driver');
+ define_variable ('LOG_DRIVER',
+ "\$(SHELL)
$am_config_aux_dir/test-driver",
+ INTERNAL);
+ }
+ my $driver = '$(LOG_DRIVER)';
+ my $driver_flags = '$(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS)';
my $compile = 'LOG_COMPILE';
define_variable ($compile,
'$(LOG_COMPILER) $(AM_LOG_FLAGS)
$(LOG_FLAGS)', INTERNAL);
$output_rules .= file_contents ('check2', new
Automake::Location,
GENERIC => 0,
OBJ => $obj,
+ BASE => $base,
SOURCE => $val,
+ DRIVER => $driver,
+ DRIVER_FLAGS => $driver_flags,
COMPILE =>'$(' . $compile . ')',
EXT => '',
am__EXEEXT => 'FALSE');
@@ -5071,6 +5089,18 @@ sub handle_tests
{
(my $ext = $test_suffix) =~ s/^\.//;
$ext = uc $ext;
+ # See comments about definition of LOG_DRIVER, above.
+ if (! var "${ext}_LOG_DRIVER")
+ {
+ require_conf_file ($parallel_tests->{position}, FOREIGN,
+ 'test-driver');
+ define_variable ("${ext}_LOG_DRIVER",
+ "\$(SHELL)
$am_config_aux_dir/test-driver",
+ INTERNAL);
+ }
+ my $driver = '$(' . $ext . '_LOG_DRIVER)';
+ my $driver_flags = '$(AM_' . $ext . '_LOG_DRIVER_FLAGS) ' .
+ '$(' . $ext . '_LOG_DRIVER_FLAGS)';
my $compile = $ext . '_LOG_COMPILE';
define_variable ($compile,
'$(' . $ext . '_LOG_COMPILER) $(AM_' . $ext
. '_LOG_FLAGS)'
@@ -5079,17 +5109,17 @@ sub handle_tests
$output_rules .= file_contents ('check2', new
Automake::Location,
GENERIC => 1,
OBJ => '',
+ BASE => '$*',
SOURCE => '$<',
+ DRIVER => $driver,
+ DRIVER_FLAGS => $driver_flags,
COMPILE => '$(' . $compile .
')',
EXT => $test_suffix,
am__EXEEXT => $am_exeext);
}
}
-
- define_variable ('TEST_LOGS_TMP', '$(TEST_LOGS:.log=.log-t)',
INTERNAL);
-
- $clean_files{'$(TEST_LOGS_TMP)'} = MOSTLY_CLEAN;
$clean_files{'$(TEST_LOGS)'} = MOSTLY_CLEAN;
+ $clean_files{'$(TEST_LOGS:.log=.trs)'} = MOSTLY_CLEAN;
$clean_files{'$(TEST_SUITE_LOG)'} = MOSTLY_CLEAN;
$clean_files{'$(TEST_SUITE_HTML)'} = MOSTLY_CLEAN;
}
diff --git a/bootstrap b/bootstrap
index aafc6eb..d289bdf 100755
--- a/bootstrap
+++ b/bootstrap
@@ -107,8 +107,6 @@ dosubst automake.in automake.tmp
cd tests
$BOOTSTRAP_SHELL ./gen-parallel-tests > parallel-tests.am
$BOOTSTRAP_SHELL ./gen-config-shell-tests > config-shell-tests.am
-$BOOTSTRAP_SHELL ./instspc-tests.sh --generate-makefile > instspc-tests.am
-$BOOTSTRAP_SHELL ./depmod-tests.sh --generate-makefile > depmod-tests.am
cd ..
# Run the autotools.
diff --git a/doc/.gitignore b/doc/.gitignore
index 2554ab9..66b9b42 100644
--- a/doc/.gitignore
+++ b/doc/.gitignore
@@ -1,27 +1,27 @@
-amhello-*.tar.gz
-automake.info
-automake.info-[0-9]
-automake*.1
-aclocal*.1
-stamp-vti
-version.texi
-automake.ac
-automake.aux
-automake.cm
-automake.cp
-automake.cps
-automake.dvi
-automake.fn
-automake.fns
-automake.html
-automake.ky
-automake.log
-automake.op
-automake.pdf
-automake.pg
-automake.ps
-automake.toc
-automake.tp
-automake.tr
-automake.vr
-automake.vrs
+/amhello-*.tar.gz
+/automake.info
+/automake.info-[0-9]
+/automake*.1
+/aclocal*.1
+/stamp-vti
+/version.texi
+/automake.ac
+/automake.aux
+/automake.cm
+/automake.cp
+/automake.cps
+/automake.dvi
+/automake.fn
+/automake.fns
+/automake.html
+/automake.ky
+/automake.log
+/automake.op
+/automake.pdf
+/automake.pg
+/automake.ps
+/automake.toc
+/automake.tp
+/automake.tr
+/automake.vr
+/automake.vrs
diff --git a/doc/amhello/.gitignore b/doc/amhello/.gitignore
index 466b7a6..cdd9777 100644
--- a/doc/amhello/.gitignore
+++ b/doc/amhello/.gitignore
@@ -1,8 +1,15 @@
-Makefile.in
-aclocal.m4
-config.h.in
-config.h.in~
-configure
-depcomp
-install-sh
-missing
+/Makefile.in
+/Makefile
+/autom4te.cache
+/aclocal.m4
+/config.h.in
+/config.h.in~
+/configure
+/configure.lineno
+/config.status
+/config.status.lineno
+/config.cache
+/config.log
+/depcomp
+/install-sh
+/missing
diff --git a/doc/automake.texi b/doc/automake.texi
index c982b40..6be5c2d 100644
--- a/doc/automake.texi
+++ b/doc/automake.texi
@@ -317,11 +317,39 @@ What Goes in a Distribution
Support for test suites
-* Simple Tests:: Listing programs and scripts in @code{TESTS}
-* Simple Tests using parallel-tests:: More powerful test driver
-* DejaGnu Tests:: Interfacing with the external testing framework
+* Generalities about Testing:: Generic concepts and terminology about testing
+* Simple Tests:: Listing test scripts in @code{TESTS}
+* Custom Test Drivers:: Writing and using custom test drivers
+* Using the TAP test protocol:: Integrating test scripts that use the TAP
protocol
+* DejaGnu Tests:: Interfacing with the @command{dejagnu} testing
framework
* Install Tests:: Running tests on installed packages
+Simple Tests
+
+* Scripts-based Testsuites:: Automake-specific concepts and terminology
+* Serial Test Harness:: Older (and obsolescent) serial test harness
+* Parallel Test Harness:: Generic concurrent test harness
+
+Using the TAP test protocol
+
+* Introduction to TAP::
+* Use TAP with the Automake test harness::
+* Incompatibilities with other TAP parsers and drivers::
+* Links and external resources on TAP::
+
+Custom Test Drivers
+
+* Overview of Custom Test Drivers Support::
+* Declaring Custom Test Drivers::
+* API for Custom Test Drivers::
+
+API for Custom Test Drivers
+
+* Command-line arguments for test drivers::
+* Log files generation and test results recording::
+* Testsuite progress output::
+* HTML generation from testsuite logs::
+
Changing Automake's Behavior
* Options generalities:: Semantics of Automake option
@@ -2317,6 +2345,10 @@ instead of copying files. Such an operation is
performed when building
multilibs (@pxref{Multilibs}). This file is maintained in the GCC
tree at @url{http://gcc.gnu.org/svn.html}.
address@hidden test-driver
+This implements the default test driver offered by the parallel
+testsuite harness.
+
@item texinfo.tex
Not a program, this file is required for @samp{make dvi}, @samp{make
ps} and @samp{make pdf} to work when Texinfo sources are in the
@@ -2913,12 +2945,24 @@ The Autoconf Manual}.
@item AC_CONFIG_AUX_DIR
Automake will look for various helper scripts, such as
@file{install-sh}, in the directory named in this macro invocation.
address@hidden This list is accurate relative to version 1.8
-(The full list of scripts is: @file{ar-lib}, @file{config.guess},
address@hidden, @file{depcomp}, @file{elisp-comp}, @file{compile},
address@hidden, @file{ltmain.sh}, @file{mdate-sh}, @file{missing},
address@hidden, @file{py-compile}, @file{texinfo.tex}, and
address@hidden) Not all scripts are always searched for; some scripts
address@hidden This list is accurate relative to version 1.11
+(The full list of scripts is:
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden,
address@hidden)
+Not all scripts are always searched for; some scripts
will only be sought if the generated @file{Makefile.in} requires them.
If @code{AC_CONFIG_AUX_DIR} is not given, the scripts are looked for in
@@ -8657,92 +8701,265 @@ default, only the @code{dist-gzip} target is hooked to
@code{dist}.
@cindex @code{make check}
@trindex check
-Automake supports three forms of test suites, the first two of which
-are very similar.
+Automake can generate code to handle two kinds of test suites. One is
+based on integration with the @command{dejagnu} framework. The other
+(and most used) form is based on the use of generic test scripts, and
+its activation is triggered by the definition of the special @code{TESTS}
+variable. This second form allows for various degrees of sophistication
+and customization; in particular, it allows for concurrent execution
+of test scripts, use of established test protocols such as TAP, and
+definition of custom test drivers and test runners.
+
address@hidden
+In either case, the testsuite is invoked via @samp{make check}.
@menu
-* Simple Tests:: Listing programs and scripts in @code{TESTS}
-* Simple Tests using parallel-tests:: More powerful test driver
-* DejaGnu Tests:: Interfacing with the external testing framework
+* Generalities about Testing:: Concepts and terminology about testing
+* Simple Tests:: Listing test scripts in @code{TESTS}
+* Custom Test Drivers:: Writing and using custom test drivers
+* Using the TAP test protocol:: Integrating test scripts that use the TAP
protocol
+* DejaGnu Tests:: Interfacing with the @command{dejagnu} testing
framework
* Install Tests:: Running tests on installed packages
@end menu
address@hidden Generalities about Testing
address@hidden Generalities about Testing
+
+The purpose of testing is to determine whether a program or system behaves
+as expected (e.g., known inputs produce the expected outputs, error
+conditions are correctly handled or reported, and older bugs do not
+resurface).
+
address@hidden test case
+The minimal unit of testing is usually called @emph{test case}, or simply
address@hidden How a test case is defined or delimited, and even what
+exactly @emph{constitutes} a test case, depends heavily on the testing
+paradigm and/or framework in use, so we won't attempt any more precise
+definition. The set of the test cases for a given program or system
+constitutes its @emph{testsuite}.
+
address@hidden test harness
address@hidden testsuite harness
+A @emph{test harness} (also @emph{testsuite harness}) is a program or
+software component that executes all (or part of) the defined test cases,
+analyzes their outcomes, and report or register these outcomes
+appropriately. Again, the details of how this is accomplished (and how
+the developer and user can influence it or interface with it) varies
+wildly, and we'll attempt no precise definition.
+
address@hidden test pass
address@hidden test failure
+A test is said to @emph{pass} when it can determine that the condition or
+behaviour it means to verify holds, and is said to @emph{fail} when it can
+determine that such condition of behaviour does @emph{not} hold.
+
address@hidden test skip
+Sometimes, tests can rely on non-portable tools or prerequisites, or
+simply make no sense on a given system (for example, a test checking a
+Windows-specific feature makes no sense on a GNU/Linux system). In this
+case, accordingly to the definition above, the tests can neither be
+considered passed nor failed; instead, they are @emph{skipped} -- i.e.,
+they are not run, or their result is anyway ignored for what concerns
+the count of failures an successes. Skips are usually explicitly
+reported though, so that the user will be aware that not all of the
+testsuite has really run.
+
address@hidden xfail
address@hidden expected failure
address@hidden expected test failure
address@hidden xpass
address@hidden unexpected pass
address@hidden unexpected test pass
+It's not uncommon, especially during early development stages, that some
+tests fail for known reasons, and that the developer doesn't want to
+tackle these failures immediately (this is especially true when the
+failing tests deal with corner cases). In this situation, the better
+policy is to declare that each of those failures is an @emph{expected
+failure} (or @emph{xfail}). In case a test that is expected to fail ends
+up passing instead, many testing environments will flag the result as a
+special kind of failure called @emph{unexpected pass} (or @emph{xpass}).
+
address@hidden hard error
address@hidden Distinction between errors and failures in testsuites
+Many testing environments and frameworks distinguish between test failures
+and hard errors. As we've seen, a test failure happens when some invariant
+or expected behaviour of the software under test is not met. An @emph{hard
+error} happens when e.g., the set-up of a test case scenario fails, or when
+some other unexpected or highly undesirable condition is encountered (for
+example, the program under test experiences a segmentation fault).
+
address@hidden: Links to other test harnesses (esp. those sharing our
+terminology)?
+
@node Simple Tests
@section Simple Tests
-If the variable @code{TESTS} is defined, its value is taken to be a
-list of programs or scripts to run in order to do the testing.
-Programs needing data files should look for them in @code{srcdir}
-(which is both an environment variable and a make variable) so they
-work when building in a separate directory (@pxref{Build Directories,
-, Build Directories , autoconf, The Autoconf Manual}), and in
-particular for the @code{distcheck} rule (@pxref{Checking the
-Distribution}).
-
-For each of the @code{TESTS}, the result of execution is printed along
-with the test name, where @code{PASS} denotes a successful test,
address@hidden denotes a failed test, @code{XFAIL} an expected failure,
address@hidden an unexpected pass for a test that is supposed to fail,
-and @code{SKIP} denotes a skipped test.
address@hidden
+* Scripts-based Testsuites:: Automake-specific concepts and terminology
+* Serial Test Harness:: Older (and obsolescent) serial test harness
+* Parallel Test Harness:: Generic concurrent test harness
address@hidden menu
+
address@hidden Scripts-based Testsuites
address@hidden Scripts-based Testsuites
+
+If the special variable @code{TESTS} is defined, its value is taken to be
+a list of programs or scripts to run in order to do the testing. Under
+the appropriate circumstances, it's possible for @code{TESTS} to list
+also data files to be passed to one or more test scripts defined by
+different means (the so-called ``log compilers'', @pxref{Parallel Test
+Harness}).
+
+Test scripts can be executed serially or concurrently. Automake
+supports both these kinds of test execution, with the serial test harness
+being the default (for backward-compatibility reasons only, as its use
+is nowadays discouraged). The concurrent test harness relies on the
+concurrence capabilities (if any) offered by the underlying @command{make}
+implementation, and can thus only be as good as those are.
+
+By default, only the exit statuses of the test scripts are considered when
+determining the testsuite outcome. But Automake allows also the use of
+more complex test protocols, either standard (@pxref{Using the TAP test
+protocol}) or custom (@pxref{Custom Test Drivers}). Note that you can
+enable such protocols only when the parallel harness is used: they won't
+work with the serial test harness. In the rest of this section we are
+going to concentrate mostly on protocol-less tests, since we'll have later
+a whole section devoted to the use of test protocols (again, @pxref{Custom
+Test Drivers}).
@cindex Exit status 77, special interpretation
address@hidden Exit status 99, special interpretation
+When no test protocol is in use, an exit status of 0 from a test script will
+denote a success, an exit status of 77 a skipped test, an exit status of 99
+an hard error, and any other exit status will denote a failure.
-The number of failures will be printed at the end of the run. If a
-given test program exits with a status of 77, then its result is ignored
-in the final count. This feature allows non-portable tests to be
-ignored in environments where they don't make sense.
address@hidden Tests, expected failure
address@hidden Expected test failure
address@hidden XFAIL_TESTS
address@hidden DISABLE_HARD_ERRORS
address@hidden Disabling hard errors
+You may define the variable @code{XFAIL_TESTS} to a list of tests
+(usually a subset of @code{TESTS}) that are expected to fail; this will
+effectively reverse the result of those tests (with the provision that
+skips and hard errors remain untouched). You may also instruct the
+testsuite harness to treat hard errors like simple failures, by defining
+the @code{DISABLE_HARD_ERRORS} make variable to a nonempty value.
+
+Note however that, for tests based on more complex test protocols,
+the exact effects of @code{XFAIL_TESTS} and @code{DISABLE_HARD_ERRORS}
+might change, or they might even have no effect at all (for example,
address@hidden Keep this in sync with tap-no-disable-hard-errors.test.
+in tests using TAP, there is not way to disable hard errors, and the
address@hidden variable has no effect on them).
+
address@hidden progress on console}
address@hidden Testsuite progress on console
+The result of each test case run by the scripts in @code{TESTS} will be
+printed on standard output, along with the test name. For test protocols
+that allow more test cases per test script (such as TAP), a number,
+identifier and/or brief description specific for the single test case is
+expected to be printed in addition to the name of the test script. The
+possible results (whose meanings should be clear from the previous
address@hidden about Testing}) are @code{PASS}, @code{FAIL},
address@hidden, @code{XFAIL}, @code{XPASS} and @code{ERROR}. Here is an
+example of output from an hypothetical testsuite that uses both plain
+and TAP tests:
address@hidden Keep in sync with tap-doc.test.
address@hidden
+PASS: foo.sh
+PASS: zardoz.tap 1 - Daemon started
+PASS: zardoz.tap 2 - Daemon responding
+SKIP: zardoz.tap 3 - Daemon uses /proc # SKIP /proc is not mounted
+PASS: zardoz.tap 4 - Daemon stopped
+SKIP: bar.sh
+PASS: mu.tap 1
+XFAIL: mu.tap 2 # TODO frobnication not yet implemented
address@hidden example
+
address@hidden
+A testsuite summary (expected to report at least the number of run,
+skipped and failed tests) will be printed at the end of the testsuite
+run.
address@hidden tests and color-tests}
@vindex AM_COLOR_TESTS
address@hidden Colorized testsuite output
If the Automake option @code{color-tests} is used (@pxref{Options})
and standard output is connected to a capable terminal, then the test
results and the summary are colored appropriately. The user can disable
colored output by setting the @command{make} variable
@samp{AM_COLOR_TESTS=no}, or force colored output even without a connecting
-terminal with @samp{AM_COLOR_TESTS=always}.
-
-Note that the semantics of some @command{make} implementations when used
-in parallel mode (@pxref{Parallel make,,, autoconf, The Autoconf Manual})
-can cause the automatic detection of a connection to a capable terminal
-to fail. In that case, you can still resort to the use of
address@hidden
+terminal with @samp{AM_COLOR_TESTS=always}. It's also worth noting that
+some @command{make} implementations, when used in parallel mode, have
+slightly different semantics (@pxref{Parallel make,,, autoconf,
+The Autoconf Manual}), which can break the automatic detection of a
+connection to a capable terminal. If this is the case, you'll have to
+resort to the use of @samp{AM_COLOR_TESTS=always} in order to have the
+testsuite output colorized.
+
+Test programs that need data files should look for them in @code{srcdir}
+(which is both a make variable and an environment variable made available
+to the tests), so that they work when building in a separate directory
+(@pxref{Build Directories, , Build Directories , autoconf,
+The Autoconf Manual}), and in particular for the @code{distcheck} rule
+(@pxref{Checking the Distribution}).
@vindex TESTS
@vindex TESTS_ENVIRONMENT
-The variable @code{TESTS_ENVIRONMENT} can be used to set environment
-variables for the test run; the environment variable @env{srcdir} is
-set in the rule. If all your test programs are scripts, you can also
-set @code{TESTS_ENVIRONMENT} to an invocation of the shell (e.g.
address@hidden(SHELL) -x} can be useful for debugging the tests), or any other
-interpreter. For instance, the following setup may be used to run tests
-with Perl:
address@hidden AM_TESTS_ENVIRONMENT
+The @code{AM_TESTS_ENVIRONMENT} and @code{TESTS_ENVIRONMENT} variables can
+be used to run initialization code and set environment variables for the
+test scripts. The former variable is developer-reserved, and can be
+defined in the @file{Makefile.am}, while the latter is reserved for the
+user, which can employ it to extend or override the settings in the
+former; for this to work portably, however, the contents of a non-empty
address@hidden @emph{must} be terminated by a semicolon.
+
address@hidden AM_TESTS_FD_REDIRECT
+The @code{AM_TESTS_FD_REDIRECT} variable can be used to define file
+descriptor redirections for the test scripts. One might think that
address@hidden could be used for this purpose, but experience
+has shown that doing so portably is practically impossible. The main
+hurdle is constituted by Korn shells, which usually set the close-on-exec
+flag on file descriptors opened with the @command{exec} builtin, thus
+rendering an idiom like @code{AM_TESTS_ENVIRONMENT = exec 9>&2;}
+ineffectual. This issue also affects some Bourne shells, such as the
+HP-UX's @command{/bin/sh},
address@hidden FIXME: should we offer a link to the relevant discussions on the
address@hidden bug-autoconf list?
@c Keep in sync with tests-environment-backcompat.test.
@example
-TESTS_ENVIRONMENT = $(PERL) -Mstrict -w
-TESTS = foo.pl bar.pl baz.pl
+AM_TESTS_ENVIRONMENT = \
+## Some environment initializations are kept in a separate shell
+## file `tests-env.sh', which can make it easier to also run tests
+## from the command line.
+ . $(srcdir)/tests-env.sh; \
+## On Solaris, prefer more POSIX-compliant versions of the standard
+## tools by default.
+ if test -d /usr/xpg4/bin; then \
+ PATH=/usr/xpg4/bin:$$PATH; export PATH; \
+ fi;
address@hidden $$ restore font-lock
+## With this, the test scripts will be able to print diagnostic
+## messages to the original standard error stream, even if the test
+## driver redirects the stderr of the test scripts to a log file
+## before executing them.
+AM_TESTS_FD_REDIRECT = 9>&2
@end example
-Note that the @option{parallel-tests} driver provides a more elegant
-way to achieve the same effect, freeing the @code{TESTS_ENVIRONMENT}
-variable for the user to override (@pxref{Simple Tests using
-parallel-tests}).
-
-
address@hidden Tests, expected failure
address@hidden Expected test failure
-
address@hidden XFAIL_TESTS
-You may define the variable @code{XFAIL_TESTS} to a list of tests
-(usually a subset of @code{TESTS}) that are expected to fail. This will
-reverse the result of those tests.
address@hidden
+Note however that @code{AM_TESTS_ENVIRONMENT} is, for historical and
+implementation reasons, @emph{not} supported by the serial harness
+(@pxref{Serial Test Harness}).
Automake ensures that each file listed in @code{TESTS} is built before
-any tests are run; you can list both source and derived programs (or
-scripts) in @code{TESTS}; the generated rule will look both in
address@hidden and @file{.}. For instance, you might want to run a C
-program as a test. To do this you would list its name in @code{TESTS}
-and also in @code{check_PROGRAMS}, and then specify it as you would
-any other program.
+it is run; you can list both source and derived programs (or scripts)
+in @code{TESTS}; the generated rule will look both in @code{srcdir} and
address@hidden For instance, you might want to run a C program as a test.
+To do this you would list its name in @code{TESTS} and also in
address@hidden, and then specify it as you would any other
+program.
Programs listed in @code{check_PROGRAMS} (and @code{check_LIBRARIES},
@code{check_LTLIBRARIES}...) are only built during @code{make check},
@@ -8753,54 +8970,100 @@ that @code{check_PROGRAMS} are @emph{not}
automatically added to
by the tests, not the tests themselves. Of course you can set
@code{TESTS = $(check_PROGRAMS)} if all your programs are test cases.
address@hidden Serial Test Harness
address@hidden Serial Test Harness
+
address@hidden:} This harness, while still being the default one, is
+obsolescent, and kept mostly for backward-compatibility reasons.
+The user is advised to use the parallel test harness instead
+(@pxref{Parallel Test Harness}).
address@hidden Simple Tests using parallel-tests
address@hidden Simple Tests using @samp{parallel-tests}
+The serial harness operates by simply running the tests serially, one at
+the time, without any I/O redirection. It's up to the user to implement
+logging of tests' output, if that's requited or desired.
address@hidden TODO: give an example of how this can be done.
+
+For historical and implementation reasons, the @code{AM_TESTS_ENVIRONMENT}
+variable is @emph{not} supported by this harness (it will be silently
+ignored if defined); only @code{TESTS_ENVIRONMENT} is, and it is to be
+considered a developer-reserved variable. This is done so that, when
+using the serial harness, @code{TESTS_ENVIRONMENT} can be defined to an
+invocation of an interpreter through which the tests are to be run.
+For instance, the following setup may be used to run tests with Perl:
+
address@hidden
+TESTS_ENVIRONMENT = $(PERL) -Mstrict -w
+TESTS = foo.pl bar.pl baz.pl
address@hidden example
+
address@hidden
+It's important to note that the use of @code{TESTS_ENVIRONMENT} endorsed
+here would be @emph{invalid} with the parallel harness. That harness
+provides a more elegant way to achieve the same effect, with the further
+benefit of freeing the @code{TESTS_ENVIRONMENT} variable for the user
+(@pxref{Parallel Test Harness}).
+
+Another, less serious limit of the serial harness is that it doesn't
+really distinguish between simple failures and hard errors; this is
+due to historical reasons only, and might be fixed in future Automake
+versions.
+
address@hidden Parallel Test Harness
address@hidden Parallel Test Harness
@cindex @option{parallel-tests}, Using
-The option @option{parallel-tests} (@pxref{Options}) enables a test
-suite driver that is mostly compatible to the simple test driver described
-in the previous section, but provides a few more features and slightly
different
-semantics. It features concurrent execution of tests with @code{make -j},
-allows to specify inter-test dependencies, lazy reruns of tests that
-have not completed in a prior run, summary and verbose output in
address@hidden (reStructuredText) and @samp{HTML} format, and hard errors
-for exceptional failures. Similar to the simple test driver,
address@hidden, @code{XFAIL_TESTS}, and
-the @code{check_*} variables are honored, and the environment variable
address@hidden is set during test execution. Also, @code{TESTS_ENVIRONMENT}
-is still honored, but is complemented by a new developer-reserved variable
address@hidden (described below).
-
-This test driver is still experimental and may undergo changes in order
-to satisfy additional portability requirements.
+The parallel (or concurrent) test harness is enabled by the Automake option
address@hidden . It features concurrent execution of tests with
address@hidden -j}, allows to specify inter-test dependencies, lazy reruns of
+tests that have not completed in a prior run, summary and verbose output in
address@hidden (reStructuredText) and @samp{HTML} format.
+This harness is still somewhat experimental and may undergo changes in
+order to satisfy additional portability requirements.
+
address@hidden of test metadata}
@vindex TEST_SUITE_LOG
@vindex TESTS
-The driver operates by defining a set of @command{make} rules to create
-a summary log file, @code{TEST_SUITE_LOG}, which defaults to
address@hidden and requires a @file{.log} suffix. This file
-depends upon log files created for each single test program listed in
address@hidden, which in turn contain all output produced by the
-corresponding tests.
address@hidden @file{.log} files
address@hidden @file{.trs} files
address@hidden test metadata
+The parallel test harness operates by defining a set of @command{make}
+rules that run the test scripts listed in @code{TESTS}, and, for each
+such script, save its output in a corresponding @file{.log} file and
+its results (and other ``metadata'', @pxref{API for Custom Test Drivers})
+in a corresponding @file{.trs} (as in @b{T}est @address@hidden) file.
address@hidden We choose the `.trs' extension also because, at the time of
writing,
address@hidden it isn't already used for other significant purposes; see e.g.:
address@hidden - http://filext.com/file-extension/trs
address@hidden - http://www.file-extensions.org/search/?searchstring=trs
+The @file{.log} file will contain all the output emitted by the test on
+its standard output and its standard error. The @file{.trs} file will
+contain, among the other things, the results of the test cases run by
+the script.
+
+The parallel test harness will also create a summary log file,
address@hidden, which defaults to @file{test-suite.log} and requires
+a @file{.log} suffix. This file depends upon all the @file{.log} and
address@hidden files created for the test scripts listed in @code{TESTS}.
@vindex VERBOSE
-As with the simple driver above, by default one status line is printed
+As with the serial harness above, by default one status line is printed
per completed test, and a short summary after the suite has completed.
However, standard output and standard error of the test are redirected
to a per-test log file, so that parallel execution does not produce
intermingled output. The output from failed tests is collected in the
@file{test-suite.log} file. If the variable @samp{VERBOSE} is set, this
-file is output after the summary. For best results, the tests should be
-verbose by default now.
+file is output after the summary.
address@hidden FIXME: we should be clearer about what we mean exactly here ...
+For best results, the tests should be verbose by default now.
@vindex TEST_EXTENSIONS
@vindex TEST_LOGS
-Each log file is created when the corresponding test has completed.
-The set of log files is listed in the read-only variable
address@hidden, and defaults to @code{TESTS}, with the executable
-extension if any (@pxref{EXEEXT}), as well as any suffix listed in
address@hidden removed, and @file{.log} appended.
+Each couple of @file{.log} and @file{.trs} files is created when the
+corresponding test has completed. The set of log files is listed in
+the read-only variable @code{TEST_LOGS}, and defaults to @code{TESTS},
+with the executable extension if any (@pxref{EXEEXT}), as well as any
+suffix listed in @code{TEST_EXTENSIONS} removed, and @file{.log} appended.
@code{TEST_EXTENSIONS} defaults to @file{.test}. Results are undefined
if a test file name ends in several concatenated suffixes.
@@ -8816,12 +9079,12 @@ if a test file name ends in several concatenated
suffixes.
@vindex address@hidden
@vindex AM_LOG_FLAGS
For tests that match an extension @address@hidden listed in
address@hidden, you can provide a test driver using the variable
address@hidden@var{ext}_LOG_COMPILER} (note the upper-case extension) and pass
-options in @address@hidden and allow the user to pass
-options in @address@hidden It will cause all tests with
-this extension to be called with this driver. For all tests without a
-registered extension, the variables @code{LOG_COMPILER},
address@hidden, you can provide a custom ``test runner'' using
+the variable @address@hidden (note the upper-case
+extension) and pass options in @address@hidden and allow
+the user to pass options in @address@hidden It will cause
+all tests with this extension to be called with this runner. For all
+tests without a registered extension, the variables @code{LOG_COMPILER},
@code{AM_LOG_FLAGS}, and @code{LOG_FLAGS} may be used. For example,
@c Keep in sync with parallel-tests-log-compiler-example.test.
@@ -8839,37 +9102,34 @@ AM_LOG_FLAGS = -d
@noindent
will invoke @samp{$(PERL) -w foo.pl}, @samp{$(PYTHON) -v bar.py},
and @samp{./wrapper-script -d baz} to produce @file{foo.log},
address@hidden, and @file{baz.log}, respectively. The
address@hidden variable is still expanded before the driver,
-but should be reserved for the user.
address@hidden, and @file{baz.log}, respectively. The @file{foo.trs},
address@hidden and @file{baz.trs} files will be automatically produced
+as a side-effect.
address@hidden AM_TESTS_ENVIRONMENT
-The @code{AM_TESTS_ENVIRONMENT} variable can be used to run initialization
-code and set environment variables for the tests' runs. The user can
-still employ the @code{TESTS_ENVIRONMENT} variable to override settings
-from @code{AM_TESTS_ENVIRONMENT}; for that to work portably, however,
-the contents of a non-empty @code{AM_TESTS_ENVIRONMENT} @emph{must} be
-terminated by a semicolon. Here is an example of a slightly elaborate
-definition:
+It's important to note that, differently from what we've seen for the
+serial test harness (@pxref{Parallel Test Harness}), the
address@hidden and @code{TESTS_ENVIRONMENT} variables
address@hidden be use to define a custom test runner; the
address@hidden and @code{LOG_FLAGS} (or their extension-specific
+counterparts) should be used instead:
@example
-AM_TESTS_ENVIRONMENT = \
-## Some environment initializations are kept in a separate shell file
-## `tests-env.sh', which can make it easier to also run tests from the
-## command line.
- . $(srcdir)/tests-env.sh; \
-## On Solaris, prefer more POSIX-compliant versions of the standard tools
-## by default.
- if test -d /usr/xpg4/bin; then \
- PATH=/usr/xpg4/bin:$$PATH; export PATH; \
- fi;
address@hidden $$ restore font-lock
+## This is WRONG!
+AM_TESTS_ENVIRONMENT = PERL5LIB='$(srcdir)/lib' $(PERL) -Mstrict -w
address@hidden example
+
address@hidden
+## Do this instead.
+AM_TESTS_ENVIRONMENT = PERL5LIB='$(srcdir)/lib'; export PERL5LIB;
+LOG_COMPILER = $(PERL)
+AM_LOG_FLAGS = -Mstrict -w
@end example
@trindex mostlyclean
@trindex check-html
@vindex RST2HTML
@vindex TEST_SUITE_HTML
address@hidden
With @code{make check-html}, the log files may be converted from RST
(reStructuredText, see @uref{http://docutils.sourceforge.net/@/rst.html})
to HTML using @samp{RST2HTML}, which defaults to @command{rst2html} or
@@ -8877,28 +9137,13 @@ to HTML using @samp{RST2HTML}, which defaults to
@command{rst2html} or
set of converted log files. The log and HTML files are removed upon
@code{make mostlyclean}.
address@hidden DISABLE_HARD_ERRORS
address@hidden Exit status 99, special interpretation
address@hidden hard error
-Even in the presence of expected failures (see @code{XFAIL_TESTS}), there
-may be conditions under which a test outcome needs attention. For
-example, with test-driven development, you may write tests for features
-that you have not implemented yet, and thus mark these tests as expected
-to fail. However, you may still be interested in exceptional conditions,
-for example, tests that fail due to a segmentation violation or another
-error that is independent of the feature awaiting implementation.
-Tests can exit with an exit status of 99 to signal such a @emph{hard
-error}. Unless the variable @code{DISABLE_HARD_ERRORS} is set to a
-nonempty value, such tests will be counted as failed.
-
-By default, the test suite driver will run all tests, but there are
+By default, the test suite harness will run all tests, but there are
several ways to limit the set of tests that are run:
@itemize @bullet
@item
-You can set the @code{TESTS} variable, similarly to how you can with
-the simple test driver from the previous section. For example, you can
-use a command like this to run only a subset of the tests:
+You can set the @code{TESTS} variable. For example, you can use a
+command like this to run only a subset of the tests:
@example
env TESTS="foo.test bar.test" make -e check
@@ -8935,14 +9180,15 @@ here too.
@item
@vindex RECHECK_LOGS
@cindex lazy test execution
-By default, the test driver removes all old per-test log files before it
-starts running tests to regenerate them. The variable
address@hidden contains the set of log files which are removed.
address@hidden defaults to @code{TEST_LOGS}, which means all tests
-need to be rechecked. By overriding this variable, you can choose which
-tests need to be reconsidered. For example, you can lazily rerun only
-those tests which are outdated, i.e., older than their prerequisite test
-files, by setting this variable to the empty value:
+By default, the test harness removes all old per-test @file{.log} and
address@hidden files before it starts running tests to regenerate them. The
+variable @code{RECHECK_LOGS} contains the set of @file{.log} (and, by
+implication, @file{.trs}) files which are removed. @code{RECHECK_LOGS}
+defaults to @code{TEST_LOGS}, which means all tests need to be rechecked.
+By overriding this variable, you can choose which tests need to be
+reconsidered. For example, you can lazily rerun only those tests which
+are outdated, i.e., older than their prerequisite test files, by setting
+this variable to the empty value:
@example
env RECHECK_LOGS= make -e check
@@ -8954,15 +9200,16 @@ env RECHECK_LOGS= make -e check
You can ensure that all tests are rerun which have failed or passed
unexpectedly, by running @code{make recheck} in the test directory.
This convenience target will set @code{RECHECK_LOGS} appropriately
-before invoking the main test driver. The @code{recheck-html} target
-does the same as @code{recheck} but again converts the resulting log
-file in HTML format, like the @code{check-html} target.
+before invoking the main test harness. The @code{recheck-html} target
+does the same as @code{recheck} but again converts the resulting
address@hidden file in HTML format, like the @code{check-html} target.
@end itemize
address@hidden
In order to guarantee an ordering between tests even with @code{make
address@hidden, dependencies between the corresponding log files may be
-specified through usual @command{make} dependencies. For example, the
-following snippet lets the test named @file{foo-execute.test} depend
address@hidden, dependencies between the corresponding @file{.log} files
+may be specified through usual @command{make} dependencies. For example,
+the following snippet lets the test named @file{foo-execute.test} depend
upon completion of the test @file{foo-compile.test}:
@example
@@ -8982,6 +9229,7 @@ parallel @command{make address@hidden, so be sure they
are prepared for
concurrent execution.
@cindex Unit tests
address@hidden Keep in sync with 'parallel-tests-extra-programs.test'.
The combination of lazy test execution and correct dependencies between
tests and their sources may be exploited for efficient unit testing
during development. To further speed up the edit-compile-test cycle, it
@@ -9009,6 +9257,573 @@ semantics of FreeBSD and OpenBSD @command{make}
conflict with this).
In case of doubt you may want to require to use GNU @command{make},
or work around the issue with inference rules to generate the tests.
address@hidden Custom Test Drivers
address@hidden Custom Test Drivers
+
address@hidden
+* Overview of Custom Test Drivers Support::
+* Declaring Custom Test Drivers::
+* API for Custom Test Drivers::
address@hidden menu
+
address@hidden Overview of Custom Test Drivers Support
address@hidden Overview of Custom Test Drivers Support
+
+Starting from Automake version 1.12, the parallel test harness allows
+the package authors to use third-party custom test drivers, in case the
+default ones are inadequate for their purposes, or do not support their
+testing protocol of choice.
+
+A custom test driver is expected to properly run the test programs passed
+to it (including the command-line arguments passed to those programs, if
+any), to analyze their execution and outcome, to create the @file{.log}
+and @file{.trs} files associated to these test runs, and to display the test
+results on the console. It is responsibility of the author of the test
+driver to ensure that it implements all the above steps meaningfully and
+correctly; Automake isn't and can't be of any help here. On the other
+hand, the Automake-provided code for testsuite summary generation offers
+support for test drivers allowing several test results per test script,
+if they take care to register such results properly (@pxref{Log files
+generation and test results recording}).
+
+The exact details of how test scripts' results are to be determined and
+analyzed is left to the individual drivers. Some drivers might only
+consider the test script exit status (this is done for example by the
+default test driver used by the parallel test harness, described
+in the previous section). Other drivers might implement more complex and
+advanced test protocols, which might require them to parse and interpreter
+the output emitted by the test script they're running (examples of such
+protocols are TAP and SubUnit).
+
+It's very important to note that, even when using custom test drivers,
+most of the infrastructure described in the previous section about the
+parallel harness remains in place; this includes:
+
address@hidden
address@hidden
+list of test scripts defined in @code{TESTS}, and overridable at
+runtime through the redefinition of @code{TESTS} or @code{TEST_LOGS};
address@hidden
+concurrency through the use of @command{make}'s option @option{-j};
address@hidden
+per-test @file{.log} and @file{.trs} files, and generation of a summary
address@hidden file from them;
address@hidden
address@hidden target, @code{RECHECK_LOGS} variable, and lazy reruns
+of tests;
address@hidden
+inter-test dependencies;
address@hidden
+support for @code{check_*} variables (@code{check_PROGRAMS},
address@hidden, ...);
address@hidden
+use of @code{VERBOSE} environment variable to get verbose output on
+testsuite failures;
address@hidden
+definition and honoring of @code{TESTS_ENVIRONMENT},
address@hidden and @code{AM_TESTS_FD_REDIRECT}
+variables;
address@hidden
+definition of generic and extension-specific @code{LOG_COMPILER} and
address@hidden variables.
address@hidden itemize
+
address@hidden
+On the other hand, the exact semantics of how (and if)
address@hidden, @code{XFAIL_TESTS}, and hard errors are supported
+and handled is left to the individual test drivers.
+
address@hidden TODO: We should really add a working example in the doc/
directory,
address@hidden TODO: and reference if from here.
+
address@hidden Declaring Custom Test Drivers
address@hidden Declaring Custom Test Drivers
+
address@hidden _LOG_DRIVER
address@hidden _LOG_DRIVER_FLAGS
address@hidden LOG_DRIVER
address@hidden LOG_DRIVER_FLAGS
address@hidden @var{ext}_LOG_DRIVER
address@hidden @var{ext}_LOG_DRIVER_FLAGS
address@hidden address@hidden
address@hidden AM_LOG_DRIVER_FLAGS
+Custom testsuite drivers are declared by defining the make variables
address@hidden or @address@hidden (where @var{ext} must
+be declared in @code{TEST_EXTENSIONS}). They must be defined to
+programs or scripts that will be used to drive the execution, logging,
+and outcome report of the tests with corresponding extensions (or of
+those with no registered extension in the case of @code{LOG_DRIVER}).
+Clearly, multiple distinct test drivers can be declared in the same
address@hidden Note moreover that the @code{LOG_DRIVER} variables
+are @emph{not} a substitute for the @code{LOG_COMPILER} variables: the
+two sets of variables can, and often do, usefully and legitimately
+coexist.
+
address@hidden TODO: We should really be able to point to a clarifying example
here!
+
+The developer-reserved variable @code{AM_LOG_DRIVER_FLAGS} and the
+user-reserved variable @code{LOG_DRIVER_FLAGS} can be used to define
+flags that will be passed to each invocation of @code{LOG_DRIVER},
+with the user-defined flags obviously taking precedence over the
+developer-reserved ones. Similarly, for each extension @var{ext}
+declared in @code{TEST_EXTENSIONS}, flags listed in
address@hidden@var{ext}_LOG_DRIVER_FLAGS} and
address@hidden@var{ext}_LOG_DRIVER_FLAGS} will be passed to
+invocations of @address@hidden
+
address@hidden API for Custom Test Drivers
address@hidden API for Custom Test Drivers
+
+Note that @emph{the APIs described here are still somewhat experimental},
+and might undergo changes and tightenings in the future, to accommodate
+for new features or to satisfy additional portability requirements.
+
+The main characteristic of these APIs is that they are designed to share
+as much infrastructure, semantics, and implementation details as possible
+with the parallel test harness and its default driver.
+
address@hidden
+* Command-line arguments for test drivers::
+* Log files generation and test results recording::
+* Testsuite progress output::
+* HTML generation from testsuite logs::
address@hidden menu
+
address@hidden Command-line arguments for test drivers
address@hidden Command-line arguments for test drivers
+
+A custom driver can rely on various command-line options and arguments
+being passed to it automatically by the Automake's @option{parallel-tests}
+harness. It is @emph{mandatory} that it understands all of them (even
+if the exact interpretation of the associated semantics can legitimately
+change between a test driver and another, and even be a no-op in some
+drivers).
+
address@hidden
+Here is the list of options:
+
address@hidden @option
address@hidden address@hidden
+The name of the test, with VPATH prefix (if any) removed. This can have a
+suffix and a directory component (as in e.g., @file{sub/foo.test}), and is
+mostly meant to be used in console reports about testsuite advancements and
+results (@pxref{Testsuite progress output}).
address@hidden address@hidden@var{PATH}.log}
+The @file{.log} file the test driver must create (@pxref{Basics of
+test metadata}). If it has a directory component (as in e.g.,
address@hidden/foo.log}), the test harness will ensure that such directory
+exists @emph{before} the test driver is called.
address@hidden address@hidden@var{PATH}.trs}
+The @file{.trs} file the test driver must create (@pxref{Basics of
+test metadata}). If it has a directory component (as in e.g.,
address@hidden/foo.trs}), the test harness will ensure that such directory
+exists @emph{before} the test driver is called.
address@hidden address@hidden|address@hidden
+Whether the console output should be colorized or not (@pxref{Simple
+tests and color-tests}, to learn when this option gets activated and
+when it doesn't).
address@hidden address@hidden|address@hidden
+Whether the tested program is expected to fail.
address@hidden address@hidden|address@hidden
+Whether ``hard errors'' in the tested program should be treated differently
+from normal failures or not (the default should be @code{yes}). The exact
+meaning of ``hard error'' is highly dependent from the test protocols or
+conventions in use.
address@hidden --
+Explicitly terminate the list of options.
address@hidden table
+
address@hidden
+The first non-option argument passed to the test driver is the program to
+be run, and all the following ones are command-line options and arguments
+for this program.
+
+Note that the exact semantics attached to the @option{--color-tests},
address@hidden and @option{--enable-hard-errors} options are
+left up to the individual test drivers. Still, having a behaviour
+compatible or at least similar to that provided by the default
address@hidden driver is advised, as that would offer a better
+consistency and a more pleasant user experience.
+
address@hidden Log files generation and test results recording
address@hidden Log files generation and test results recording
+
+The test driver must correctly generate the files specified by the
address@hidden and @option{--trs-file} option (even when the tested
+program fails or crashes).
+
+The @file{.log} file should ideally contain all the output produced by the
+tested program, plus optionally other information that might facilitate
+debugging or analysis of bug reports. Apart from that, its format is
+basically free; the only limitation being that it must parse validly as
+reStructuredText if the @file{.log} -> @file{.html} conversion is to be
+supported.
+
+The @file{.trs} file is used to register some metadata through the use
+of custom reStructuredText fields. This metadata is expected to be
+employed in various ways by the parallel test harness; for example, to
+count the test results when printing the testsuite summary, or to decide
+which tests to re-run upon @command{make reheck}. Unrecognized metadata
+in a @file{.trs} file is currently ignored by the harness, but this might
+change in the future. The list of currently recognized metadata follows.
+
address@hidden @code
+
address@hidden :test-result:
address@hidden Register test result
address@hidden Register test case result
address@hidden Test result, registering
address@hidden Test case result, registering
address@hidden @code{:test-result:}
address@hidden reStructuredText field, @code{:test-result:}
+The test driver must use this field to register the results of @emph{each}
+test case run by a test script file. Several @code{:test-result:} fields
+can be present in the same @file{.trs} file; this is done in order to
+support test protocols that allow a single test script to run more test
+cases.
+
address@hidden Keep this in sync with lib/am/check-am:$(TEST_SUITE_LOG).
+The only recognized test results are currently @code{PASS}, @code{XFAIL},
address@hidden, @code{FAIL}, @code{XPASS} and @code{ERROR}. These results,
+when declared with @code{:test-result:}, can be optionally followed by
+text holding the name and/or a brief description of the corresponding
+test; the @option{parallel-tests} harness will ignore such extra text when
+generating @file{test-suite.log} and preparing the testsuite summary.
+
address@hidden Keep in sync with 'test-metadata-recheck.test'.
address@hidden @code{:recheck:}
address@hidden :recheck:
address@hidden reStructuredText field, @code{:recheck:}
+If this field is present and defined to @code{no}, then the corresponding
+test script will @emph{not} be run upon a @command{make recheck}. What
+happens when two or more @code{:recheck:} fields are present in the same
address@hidden file is undefined behaviour.
+
address@hidden Keep in sync with 'test-metadata-global-log.test'.
address@hidden @code{:copy-in-global-log:}
address@hidden :copy-in-global-log:
address@hidden reStructuredText field, @code{:copy-in-global-log:}
+If this field is present and defined to @code{no}, then the content
+of the @file{.log} file will @emph{not} be copied into the global
address@hidden We allow to forsake such copying because, while
+it can be useful in debugging and analysis of bug report, it can also be
+just a waste of space in normal situations, e.g., when a test script is
+successful. What happens when two or more @code{:copy-in-global-log:}
+fields are present in the same @file{.trs} file is undefined behaviour.
+
address@hidden Keep in sync with 'test-metadata-global-result.test'.
address@hidden @code{:test-global-result:}
address@hidden :test-global-result:
address@hidden reStructuredText field, @code{:test-global-result:}
+This is used to declare the "global result" of the script. Currently,
+the value of this field is needed only to be reported (more or less
+verbatim) in the generated global log file @code{$(TEST_SUITE_LOG)},
+so it's quite free-form. For example, a test script which run 10 test
+cases, 6 of which pass and 4 of which are skipped, could reasonably have
+a @code{PASS/SKIP} value for this field, while a test script which run
+19 successful tests and one failed test could have an @code{ALMOST
+PASSED} value. What happens when two or more @code{:test-global-result:}
+fields are present in the same @file{.trs} file is undefined behaviour.
address@hidden table
+
address@hidden
+Let's see a small example. Assume a @file{.trs} file contains the
+following lines:
+
address@hidden
+:test-result: PASS server starts
+:global-log-copy: no
+:test-result: PASS HTTP/1.1 request
+:test-result: FAIL HTTP/1.0 request
+:recheck: yes
+:test-result: SKIP HTTPS request (TLS library wasn't available)
+:test-result: PASS server stops
address@hidden example
+
address@hidden
+Then the corresponding test script will be re-run by @command{make check},
+will contribute with @emph{five} test results to the testsuite summary
+(three of these tests being successful, one failed, and one skipped), and
+the content of the corresponding @file{.log} file will @emph{not} be
+copied in the global log file @file{test-suite.log}.
+
address@hidden Testsuite progress output
address@hidden Testsuite progress output
+
+A custom test driver also has the task of displaying, on the standard
+output, the test results as soon as they become available. Depending on
+the protocol in use, it can also display the reasons for failures and
+skips, and, more generally, any useful diagnostic output (but remember
+that each line on the screen is precious, so that cluttering the screen
+with overly verbose information is bad idea). The exact format of this
+progress output is left up to the test driver; in fact, a custom test
+driver might @emph{theoretically} even decide not to do any such report,
+leaving it all to the testsuite summary (that would be a very lousy idea,
+of course, and serves only to illustrate the flexibility that is
+granted here).
+
+Remember that consistency is good; so, if possible, try to be consistent
+with the output of the built-in Automake test drivers, providing a similar
+``look & feel''. In particular, the testsuite progress output should be
+colorized when the @option{--color-tests} is passed to the driver. On the
+other end, if you are using a known and widespread test protocol with
+well-established implementations, being consistent with those
+implementations' output might be a good idea too.
+
address@hidden TODO: Give an example, maybe inspired to py.test-style output.
address@hidden TODO: That is a good idea because it shows a test driver that
allows
address@hidden TODO: for different levels of verbosity in the progress output
(could
address@hidden TODO: be implemented either using a driver cmdline flag, or an
address@hidden TODO: environment variable, or both).
+
address@hidden HTML generation from testsuite logs
address@hidden HTML generation from testsuite logs
+
+If HTML testsuite output (with @code{check-html}) is to be supported,
+the generated @file{.log} files must contain syntactically valid
+reStructuredText. If this is not the case, the HTML generation will not
+work, although all the other functionalities of the Automake testsuite
+harness should remain untouched, and continue to work correctly.
+
address@hidden Using the TAP test protocol
address@hidden Using the TAP test protocol
+
address@hidden
+* Introduction to TAP::
+* Use TAP with the Automake test harness::
+* Incompatibilities with other TAP parsers and drivers::
+* Links and external resources on TAP::
address@hidden menu
+
address@hidden Introduction to TAP
address@hidden Introduction to TAP
+
+TAP, the Test Anything Protocol, is a simple text-based interface between
+testing modules or programs and a test harness. The tests (also called
+``TAP producers'' in this context) write test results in a simple format
+on standard output; a test harness (also called ``TAP consumer'') will
+parse and interpret these results, and properly present them to the user,
+and/or register them for later analysis. The exact details of how this
+is accomplished can vary among different test harnesses. The Automake
+parallel harness will present the results on the console in the usual
+fashion (@pxref{Testsuite progress on console}), and will use the
address@hidden files (@pxref{Basics of test metadata}) to store the test
+results and related metadata. Apart from that, it will try to remain
+as much compatible as possible with pre-existing and widespread utilities,
+such as the @uref{http://search.cpan.org/~andya/Test-Harness/bin/prove,
address@hidden utility}, at least for the simpler usages.
+
+TAP started its life as part of the test harness for Perl, but today
+it has been (mostly) standardized, and has various independent
+implementations in different languages; among them, C, C++, Perl,
+Python, PHP, and Java. For a semi-official specification of the
+TAP protocol, please refer to the documentation of
address@hidden://search.cpan.org/~petdance/Test-Harness/lib/Test/Harness/TAP.pod,
+ @samp{Test::Harness::TAP}}.
+
+The most relevant real-world usages of TAP are obviously in the testsuites
+of @command{perl} and of many perl modules. Still, also other important
+third-party packages, such as @uref{http://git-scm.com/, @command{git}},
+use TAP in their testsuite.
+
address@hidden Use TAP with the Automake test harness
address@hidden Use TAP with the Automake test harness
+
+Currently, the TAP driver that comes with Automake requires a perl
+interpreter to work, and requires various by-hand steps on the
+developer's part (this should be fixed in future Automake versions).
+You'll have grab the @file{tap-driver} script from the Automake
+distribution by hand, copy it in your source tree, add code to
address@hidden to search a perl interpreter and to define the
address@hidden(PERL)} variable accordingly, and use the Automake support
+for third-party test drivers to instruct the harness to use the
address@hidden to run your TAP-producing tests. See the example
+below for clarification.
+
+Apart from the options common to all the Automake test drivers
+(@pxref{Command-line arguments for test drivers}), the @file{tap-driver}
+supports the following options, whose names are chosen for enhanced
+compatibility with the @command{prove} utility.
+
address@hidden @option
address@hidden Keep in sync with 'tap-exit.test' and 'tap-signal.test'.
address@hidden --ignore-exit
+Causes the test driver to ignore the exit status of the test scripts;
+by default, the driver will report an error if the script exit with a
+non-zero status. This option has effect also
address@hidden --comments
+Instruct the test driver to display TAP diagnostic (i.e., lines beginning
+with the @samp{#} character) in the testsuite progress output too; by
+default, TAP diagnostic is only copied in the @file{.log} file.
address@hidden --no-comments
+Revert the effects of @option{--comments}.
address@hidden --merge
+Instruct the test driver to merge the test scripts' standard error into
+their standard output. This is necessary if you want to ensure that
+diagnostics from the test scripts are displayed in the correct order
+relative to test results; this can be of great help in debugging
+(especially if your test scripts are shell scripts run with shell
+tracing active). As a downside, this option might cause the test
+harness to get confused if anything that appears on standard error
+looks like a test result.
address@hidden --no-merge
+Revert the effects of @option{--merge}.
address@hidden address@hidden
+Change the string that introduces TAP diagnostic from the default value
+of address@hidden'' to @address@hidden This can be useful if your
+TAP-based test scripts produce verbose output on which they have limited
+control (because, say, the output comes by other tools invoked in the
+scripts), and it might contain text that gets spuriously interpreted as
+TAP diagnostic: such an issue can be solved by redefining the string that
+activates TAP diagnostic to a value you know won't appear by chance in
+the tests' output. Note however that this feature is non-standard, as
+the ``official'' TAP protocol does not allow for such a customization; so
+don't use it if you can avoid it.
address@hidden table
+
address@hidden
+Here is an example of how the TAP driver can be set up and used.
+
address@hidden Keep in sync with tap-doc2.test.
address@hidden
+% @kbd{cat configure.ac}
+AC_INIT([GNU Try Tap], [1.0], [bug-automake@@gnu.org])
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE([foreign parallel-tests -Wall -Werror])
+AC_CONFIG_FILES([Makefile])
+AC_REQUIRE_AUX_FILE([tap-driver])
+AC_PATH_PROG([PERL], [perl])
+test -n "$PERL" || AC_MSG_ERROR([perl not found])
+$PERL -MTAP::Parser -e 1 || AC_MSG_ERROR([TAP::Parser not found])
+AC_OUTPUT
+
+% @kbd{cat Makefile.am}
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/build-aux/tap-driver
+TESTS = foo.test bar.test baz.test
+EXTRA_DIST = $(TESTS)
+
+% @kbd{cat foo.test}
+#!/bin/sh
+echo 1..4 # Number of tests to be executed.
+echo 'ok 1 - Swallows fly'
+echo 'not ok 2 - Caterpillars fly # TODO metamorphosis in progress'
+echo 'ok 3 - Pigs fly # SKIP not enough acid'
+echo '# I just love word plays ...'
+echo 'ok 4 - Flies fly too :-)'
+
+% @kbd{cat bar.test}
+#!/bin/sh
+echo 1..3
+echo 'not ok 1 - Bummer, this test has failed.'
+echo 'ok 2 - This passed though.'
+echo 'Bail out! Ennui kicking in, sorry...'
+echo 'ok 3 - This will not be seen.'
+
+% @kbd{cat baz.test}
+#!/bin/sh
+echo 1..1
+echo ok 1
+# Exit with error, even if all the test case has been successful.
+exit 7
+
+% @kbd{cp @var{PREFIX}/share/address@hidden/tap-driver .}
+% @kbd{autoreconf -vi && ./configure && make check}
+...
+PASS: foo.test 1 - Swallows fly
+XFAIL: foo.test 2 - Caterpillars fly # TODO metamorphosis in progress
+SKIP: foo.test 3 - Pigs fly # SKIP not enough acid
+PASS: foo.test 4 - Flies fly too :-)
+FAIL: bar.test 1 - Bummer, this test has failed.
+PASS: bar.test 2 - This passed though.
+ERROR: bar.test - Bail out! Ennui kicking in, sorry...
+PASS: baz.test 1
+ERROR: baz.test - exited with status 7
+...
+Please report to bug-automake@@gnu.org
+...
+% @kbd{echo exit status: $?}
+exit status: 1
+
address@hidden Keep the "skewed" indentation below, it produces pretty PDF
output.
+% @kbd{env TEST_LOG_DRIVER_FLAGS='--comments --ignore-exit' \
+ TESTS='foo.test baz.test' make -e check}
+...
+PASS: foo.test 1 - Swallows fly
+XFAIL: foo.test 2 - Caterpillars fly # TODO metamorphosis in progress
+SKIP: foo.test 3 - Pigs fly # SKIP not enough acid
+# foo.test: I just love word plays...
+PASS: foo.test 4 - Flies fly too :-)
+PASS: baz.test 1
+...
+% @kbd{echo exit status: $?}
+exit status: 0
address@hidden example
+
address@hidden Incompatibilities with other TAP parsers and drivers
address@hidden Incompatibilities with other TAP parsers and drivers
+
+For implementation or historical reasons, the TAP driver and harness as
+implemented by Automake have some minors incompatibilities with the
+mainstream versions, which you should be aware of.
+
address@hidden @bullet
address@hidden
+A @code{Bail out!} directive doesn't stop the whole testsuite, but only
+the test script it occurs into. This doesn't follows TAP specifications,
+but on the other hand it maximizes compatibility (and code sharing) with
+the ``hard error'' concept of the default @option{parallel-tests} driver.
address@hidden
+The @code{version} and @code{pragma} directives are not supported.
address@hidden
+The @option{--diagnostic-string} option of out driver allows to modify
+the string that introduces TAP diagnostic from the default value
+of address@hidden''. The standard TAP protocol has currently no way to
+allow this, so if you use it your diagnostic will be lost to more
+compliant tools like @command{prove} and @code{Test::Harness}
address@hidden
+And there are probably some other small and yet undiscovered
+incompatibilities, especially in corner cases or with rare usages.
address@hidden itemize
+
address@hidden Links and external resources on TAP
address@hidden Links and external resources on TAP
+
address@hidden
+Here are some links to more extensive official or third-party
+documentation and resources about the TAP protocol and related
+tools and libraries.
address@hidden @bullet
address@hidden
address@hidden://search.cpan.org/~petdance/Test-Harness/lib/Test/Harness/TAP.pod,
+ @samp{Test::Harness::TAP}},
+the (mostly) official documentation about the TAP format and protocol.
address@hidden
address@hidden://search.cpan.org/~andya/Test-Harness/bin/prove,
+ @command{prove}},
+the most famous command-line TAP test driver, included in the distribution
+of @command{perl} and
address@hidden://search.cpan.org/~andya/Test-Harness/lib/Test/Harness.pm,
+ @samp{Test::Harness}}.
address@hidden
+The @uref{http://testanything.org/wiki/index.php/Main_Page,TAP wiki}.
address@hidden
+A ``gentle introduction'' to testing for perl coders:
address@hidden://search.cpan.org/dist/Test-Simple/lib/Test/Tutorial.pod,
+ @samp{Test::Tutorial}}.
address@hidden
address@hidden://search.cpan.org/~mschwern/Test-Simple/lib/Test/Simple.pm,
+ @samp{Test::Simple}}
+and
address@hidden://search.cpan.org/~mschwern/Test-Simple/lib/Test/More.pm,
+ @samp{Test::More}},
+the standard perl testing libraries, which are based on TAP.
address@hidden
address@hidden://www.eyrie.org/~eagle/software/c-tap-harness/,C TAP Harness},
+a C-based project implementing both a TAP producer and a TAP consumer.
address@hidden
address@hidden://www.tap4j.org/,tap4j},
+a Java-based project implementing both a TAP producer and a TAP consumer.
address@hidden itemize
@node DejaGnu Tests
@section DejaGnu Tests
@@ -9053,8 +9868,6 @@ Especially, @file{site.exp} should not be distributed.
For more information regarding DejaGnu test suites, see @ref{Top, , ,
dejagnu, The DejaGnu Manual}.
-In either case, the testing is done via @samp{make check}.
-
@node Install Tests
@section Install Tests
@@ -9265,8 +10078,9 @@ in the first few lines of the @file{NEWS} file.
@item @option{color-tests}
@cindex Option, @option{color-tests}
@opindex color-tests
-Cause output of the simple test suite (@pxref{Simple Tests}) to be
-colorized on capable terminals.
+Cause output of the serial and parallel test harnesses (see @ref{Simple
+Tests}) and of properly-written custom test drivers (@pxref{Custom Test
+Drivers}) to be colorized on capable terminals.
@item @option{dejagnu}
@cindex Option, @option{dejagnu}
@@ -9394,8 +10208,8 @@ this directory.
@item @option{parallel-tests}
@cindex Option, @option{parallel-tests}
@opindex parallel-tests
-Enable test suite driver for @code{TESTS} that can run tests in parallel
-(@pxref{Simple Tests using parallel-tests}, for more information).
+Enable test suite harness for @code{TESTS} that can run tests in parallel
+(@pxref{Parallel Test Harness}, for more information).
@item @option{readme-alpha}
@cindex Option, @option{readme-alpha}
diff --git a/lib/Automake/.gitignore b/lib/Automake/.gitignore
index d539f93..f99f508 100644
--- a/lib/Automake/.gitignore
+++ b/lib/Automake/.gitignore
@@ -1 +1 @@
-Config.pm
+/Config.pm
diff --git a/lib/Automake/tests/.gitignore b/lib/Automake/tests/.gitignore
index ffbdfaf..56e8f36 100644
--- a/lib/Automake/tests/.gitignore
+++ b/lib/Automake/tests/.gitignore
@@ -1,2 +1,2 @@
-*.log
-*.log-t
+/*.trs
+/*.log
diff --git a/lib/Automake/tests/Makefile.in b/lib/Automake/tests/Makefile.in
index 18308b0..24bc878 100644
--- a/lib/Automake/tests/Makefile.in
+++ b/lib/Automake/tests/Makefile.in
@@ -74,19 +74,25 @@ am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY))
am__v_at_0 = @
SOURCES =
DIST_SOURCES =
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
&& { \
+ am__color_tests=yes; \
red='[0;31m'; \
grn='[0;32m'; \
lgn='[1;32m'; \
blu='[1;34m'; \
+ mgn='[0;35m'; \
+ brg='[1m'; \
std='[m'; \
}
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
@@ -113,87 +119,64 @@ am__base_list = \
# Restructured Text title and section.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
am__rst_section = sed 'p;s/./=/g;p;g'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
am__sh_e_setup = case $$- in *e*) set +e;; esac
+# Default flags passed to test drivers.
+am__common_driver_flags = \
+ --color-tests "$$am__color_tests" \
+ --enable-hard-errors "$$am__enable_hard_errors" \
+ --expect-failure "$$am__expect_failure"
# To be inserted before the command running the test. Creates the
# directory for the log if needed. Stores in $dir the directory
# containing $f, in $tst the test, in $log the log. Executes the
# developer- defined test setup AM_TESTS_ENVIRONMENT (if any), and
-# passes TESTS_ENVIRONMENT. Saves and restores TERM around uses of
-# TESTS_ENVIRONMENT and AM_TESTS_ENVIRONMENT, in case any of them
-# unsets it.
+# passes TESTS_ENVIRONMENT. Set up options for the wrapper that
+# will run the test scripts (or their associated LOG_COMPILER, if
+# thy have one).
am__check_pre = \
$(am__sh_e_setup); \
$(am__vpath_adj_setup) $(am__vpath_adj) \
+$(am__tty_colors); \
srcdir=$(srcdir); export srcdir; \
-rm -f address@hidden; \
-am__trap='rm -f '\''$(abs_builddir)/address@hidden'\''; (exit $$st); exit
$$st'; \
-trap "st=129; $$am__trap" 1; trap "st=130; $$am__trap" 2; \
-trap "st=141; $$am__trap" 13; trap "st=143; $$am__trap" 15; \
am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`; \
test "x$$am__odir" = x. || $(MKDIR_P) "$$am__odir" || exit $$?; \
if test -f "./$$f"; then dir=./; \
elif test -f "$$f"; then dir=; \
else dir="$(srcdir)/"; fi; \
-tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \
-$(AM_TESTS_ENVIRONMENT) \
-$(TESTS_ENVIRONMENT)
-# To be appended to the command running the test. Handle the stdout
-# and stderr redirection, and catch the exit status.
-am__check_post = \
->address@hidden 2>&1; \
-estatus=$$?; \
-if test -n '$(DISABLE_HARD_ERRORS)' \
- && test $$estatus -eq 99; then \
- estatus=1; \
-fi; \
-TERM=$$__SAVED_TERM; export TERM; \
-$(am__tty_colors); \
-xfailed=PASS; \
+tst=$$dir$$f; log='$@'; \
+if test -n '$(DISABLE_HARD_ERRORS)'; then \
+ am__enable_hard_errors=no; \
+else \
+ am__enable_hard_errors=yes; \
+fi; \
case " $(XFAIL_TESTS) " in \
*[\ \ ]$$f[\ \ ]* | *[\ \ ]$$dir$$f[\ \ ]*) \
- xfailed=XFAIL;; \
-esac; \
-case $$estatus.$$xfailed in \
- 0.XFAIL) col=$$red; res=XPASS;; \
- 0.*) col=$$grn; res=PASS ;; \
- 77.*) col=$$blu; res=SKIP ;; \
- 99.*) col=$$red; res=FAIL ;; \
- *.XFAIL) col=$$lgn; res=XFAIL;; \
- *.*) col=$$red; res=FAIL ;; \
-esac; \
-echo "$${col}$$res$${std}: $$f"; \
-echo "$$res: $$f (exit: $$estatus)" | \
- $(am__rst_section) >$@; \
-cat address@hidden >>$@; \
-rm -f address@hidden
+ am__expect_failure=yes;; \
+ *) \
+ am__expect_failure=no;; \
+esac; \
+$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT)
+# A shell command to get the names of the tests scripts with any registered
+# extension removed (i.e., equivalently, the names of the test logs, with
+# the `.log' extension removed). The result is saved in the shell variable
+# `$bases'. This honors runtime overriding of TESTS and TEST_LOGS. Sadly,
+# we cannot use something simpler, involving e.g., "$(TEST_LOGS:.log=)",
+# since that might cause problem with VPATH rewrites for suffix-less tests.
+# See also 'test-harness-vpath-rewrite.test' and 'test-trs-basic.test'.
+am__set_TESTS_bases = \
+ bases='$(TEST_LOGS)'; \
+ bases=`for i in $$bases; do echo $$i; done | sed 's/\.log$$//'`; \
+ bases=`echo $$bases`
RECHECK_LOGS = $(TEST_LOGS)
AM_RECURSIVE_TARGETS = check check-html recheck recheck-html
TEST_SUITE_LOG = test-suite.log
TEST_SUITE_HTML = $(TEST_SUITE_LOG:.log=.html)
am__test_logs1 = $(TESTS:=.log)
TEST_LOGS = $(am__test_logs1:.pl.log=.log)
+PL_LOG_DRIVER = $(SHELL) $(top_srcdir)/lib/test-driver
PL_LOG_COMPILE = $(PL_LOG_COMPILER) $(AM_PL_LOG_FLAGS) $(PL_LOG_FLAGS)
-TEST_LOGS_TMP = $(TEST_LOGS:.log=.log-t)
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
AMTAR = @AMTAR@
@@ -325,7 +308,7 @@ am__tdf = diff-in-tests-lists.tmp
all: all-am
.SUFFIXES:
-.SUFFIXES: .html .log .pl
+.SUFFIXES: .html .log .pl .trs
$(srcdir)/Makefile.in: $(srcdir)/Makefile.am
$(top_srcdir)/CheckListOfTests.am $(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
@@ -365,92 +348,144 @@ CTAGS:
cscope cscopelist:
+# Recover from deleted `.trs' file; this should ensure that
+# "rm -f foo.log; make foo.trs" re-run `foo.test', and re-create
+# both `foo.log' and `foo.trs'. Break the recipe in two subshells
+# to avoid problems with "make -n".
+.log.trs:
+ rm -f $< $@
+ $(MAKE) $(AM_MAKEFLAGS) $<
+
+# Helper recipe used by $(TEST_SUITE_LOG) below, to avoid problems with
+# "make -n". Break this recipe in multiple shell invocations too, to
+# really work as expected with "make -n".
+am--redo-logs:
+ @rm -f $$redo_logs
+ @rm -f $$redo_results
+ @$(MAKE) $(AM_MAKEFLAGS) $$redo_logs
+ @st=0; \
+ errmsg="fatal: making $(TEST_SUITE_LOG): failed to create"; \
+ for i in $$redo_bases; do \
+ test -f $$i.trs && test -r $$i.trs \
+ || { echo "$$errmsg $$i.trs"; st=1; }; \
+ test -f $$i.log && test -r $$i.log \
+ || { echo "$$errmsg $$i.log"; st=1; }; \
+ done; \
+ test $$st -eq 0
+
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
- list='$(TEST_LOGS)'; \
- results=`for f in $$list; do \
- test -r $$f && read line < $$f && echo "$$line" \
- || echo FAIL; \
- done`; \
- all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[
]*//'`; \
- fail=`echo "$$results" | grep -c '^FAIL'`; \
- pass=`echo "$$results" | grep -c '^PASS'`; \
- skip=`echo "$$results" | grep -c '^SKIP'`; \
- xfail=`echo "$$results" | grep -c '^XFAIL'`; \
- xpass=`echo "$$results" | grep -c '^XPASS'`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+ @$(am__set_TESTS_bases); \
+ am__f_ok () { test -f "$$1" && test -r "$$1"; }; \
+ redo_bases=`for i in $$bases; do \
+ am__f_ok $$i.trs && am__f_ok $$i.log || echo $$i; \
+ done`; \
+ if test -n "$$redo_bases"; then \
+ redo_logs=`for i in $$redo_bases; do echo $$i.log; done`; \
+ redo_results=`for i in $$redo_bases; do echo $$i.trs; done`; \
+ redo_bases="$$redo_bases" \
+ redo_logs="$$redo_logs" \
+ redo_results="$$redo_results" \
+ $(MAKE) $(AM_MAKEFLAGS) am--redo-logs || exit 1; \
+ else :; fi;
+ @$(am__sh_e_setup); $(am__tty_colors); $(am__set_TESTS_bases); \
+ ws='[ ]'; \
+ results=`for b in $$bases; do echo $$b.trs; done`; \
+ test -n "$$results" || results=/dev/null; \
+ all=` grep "^$$ws*:test-result:" $$results | wc -l`; \
+ pass=` grep "^$$ws*:test-result:$$ws*PASS" $$results | wc -l`; \
+ fail=` grep "^$$ws*:test-result:$$ws*FAIL" $$results | wc -l`; \
+ skip=` grep "^$$ws*:test-result:$$ws*SKIP" $$results | wc -l`; \
+ xfail=`grep "^$$ws*:test-result:$$ws*XFAIL" $$results | wc -l`; \
+ xpass=`grep "^$$ws*:test-result:$$ws*XPASS" $$results | wc -l`; \
+ error=`grep "^$$ws*:test-result:$$ws*ERROR" $$results | wc -l`; \
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+ br='==================='; br=$$br$$br$$br$$br; \
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
- for f in $$list; do \
- test -r $$f && read line < $$f || line=; \
- case $$line in \
- PASS:*|XFAIL:*);; \
- *) echo; cat $$f;; \
- esac; \
- done; \
+ for i in $$bases; do \
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$i.trs \
+ >/dev/null; then continue; \
+ fi; \
+ glob_res=`sed -n -e "s/$$ws*$$//" \
+ -e "s/^$$ws*:global-test-result:$$ws*//p" \
+ $$i.trs`; \
+ test -n "$$glob_res" || glob_res=RUN; \
+ echo "$$glob_res: $$i" | $(am__rst_section); \
+ if test ! -r $$i.log; then \
+ echo "fatal: making $@: $$i.log is unreadable" >&2; \
+ exit 1; \
+ fi; \
+ cat $$i.log; echo; \
+ done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
- $(am__tty_colors); \
- if $$exit; then \
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
+ fi; \
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+ $$success
# Run all the tests.
check-TESTS:
@list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
+ @list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list
@test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
- @list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test .log = $$f || echo $$f; \
- done | tr '\012\015' ' '`; \
+ @list='$(TEST_LOGS)'; \
+ list=`for i in $$list; do \
+ test .log = $$i || echo $$i; \
+ done | tr '\012\015' ' '`; \
+ list=`echo "$$list" | sed 's/ *$$//'`; \
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) TEST_LOGS="$$list"
.log.html:
@@ -478,18 +513,21 @@ check-html:
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML) || exit 4; \
exit $$rv
recheck recheck-html:
- @target=`echo $@ | sed 's,^re,,'`; \
- list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test -f $$f || continue; \
- if test -r $$f && read line < $$f; then \
- case $$line in FAIL*|XPASS*) echo $$f;; esac; \
- else echo $$f; fi; \
+ @ws='[ ]'; \
+ target=`echo $@ | sed 's,^re,,'`; \
+ $(am__set_TESTS_bases); \
+ list=`for i in $$bases; do \
+ test -f $$i.trs || test -f $$i.log || continue; \
+ grep "^$$ws*:recheck:$$ws*no$$ws*$$" $$i.trs \
+ >/dev/null 2>&1 || echo $$i.log; \
done | tr '\012\015' ' '`; \
list=`echo "$$list" | sed 's/ *$$//'`; \
$(MAKE) $(AM_MAKEFLAGS) $$target AM_MAKEFLAGS='$(AM_MAKEFLAGS)
TEST_LOGS="'"$$list"'"'
.pl.log:
- @p='$<'; $(am__check_pre) $(PL_LOG_COMPILE) "$$tst" $(am__check_post)
+ @p='$<'; $(am__check_pre) $(PL_LOG_DRIVER) --test-name "$$f" \
+ --log-file '$*.log' --trs-file '$*.trs' \
+ $(am__common_driver_flags) $(AM_PL_LOG_DRIVER_FLAGS)
$(PL_LOG_DRIVER_FLAGS) -- $(PL_LOG_COMPILE) "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
@@ -547,7 +585,7 @@ install-strip:
fi
mostlyclean-generic:
-test -z "$(TEST_LOGS)" || rm -f $(TEST_LOGS)
- -test -z "$(TEST_LOGS_TMP)" || rm -f $(TEST_LOGS_TMP)
+ -test -z "$(TEST_LOGS:.log=.trs)" || rm -f $(TEST_LOGS:.log=.trs)
-test -z "$(TEST_SUITE_HTML)" || rm -f $(TEST_SUITE_HTML)
-test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
@@ -629,16 +667,17 @@ uninstall-am:
.MAKE: check-am check-html install-am install-strip recheck \
recheck-html
-.PHONY: all all-am check check-TESTS check-am check-html clean \
- clean-generic clean-local distclean distclean-generic distdir \
- dvi dvi-am html html-am info info-am install install-am \
- install-data install-data-am install-dvi install-dvi-am \
- install-exec install-exec-am install-html install-html-am \
- install-info install-info-am install-man install-pdf \
- install-pdf-am install-ps install-ps-am install-strip \
- installcheck installcheck-am installdirs maintainer-clean \
- maintainer-clean-generic mostlyclean mostlyclean-generic pdf \
- pdf-am ps ps-am recheck recheck-html uninstall uninstall-am
+.PHONY: all all-am am--redo-logs check check-TESTS check-am check-html \
+ clean clean-generic clean-local distclean distclean-generic \
+ distdir dvi dvi-am html html-am info info-am install \
+ install-am install-data install-data-am install-dvi \
+ install-dvi-am install-exec install-exec-am install-html \
+ install-html-am install-info install-info-am install-man \
+ install-pdf install-pdf-am install-ps install-ps-am \
+ install-strip installcheck installcheck-am installdirs \
+ maintainer-clean maintainer-clean-generic mostlyclean \
+ mostlyclean-generic pdf pdf-am ps ps-am recheck recheck-html \
+ uninstall uninstall-am
.PHONY: maintainer-check-list-of-tests
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 5bdc02e..a9467f1 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -29,7 +29,7 @@ dist_pkgvdata_DATA = COPYING INSTALL texinfo.tex ansi2knr.c
ansi2knr.1 \
scriptdir = $(pkgvdatadir)
dist_script_DATA = config.guess config.sub install-sh mdate-sh missing \
mkinstalldirs elisp-comp ylwrap acinstall depcomp compile py-compile \
- symlink-tree ar-lib
+ symlink-tree ar-lib test-driver tap-driver
EXTRA_DIST = gnupload
diff --git a/lib/Makefile.in b/lib/Makefile.in
index 737be4a..5bf597e 100644
--- a/lib/Makefile.in
+++ b/lib/Makefile.in
@@ -247,7 +247,7 @@ dist_pkgvdata_DATA = COPYING INSTALL texinfo.tex ansi2knr.c
ansi2knr.1 \
scriptdir = $(pkgvdatadir)
dist_script_DATA = config.guess config.sub install-sh mdate-sh missing \
mkinstalldirs elisp-comp ylwrap acinstall depcomp compile py-compile \
- symlink-tree ar-lib
+ symlink-tree ar-lib test-driver tap-driver
EXTRA_DIST = gnupload
all: all-recursive
diff --git a/lib/am/check.am b/lib/am/check.am
index 82ba7ce..76c5273 100644
--- a/lib/am/check.am
+++ b/lib/am/check.am
@@ -15,25 +15,30 @@
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
if %?COLOR%
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
&& { \
+ am__color_tests=yes; \
red='[0;31m'; \
grn='[0;32m'; \
lgn='[1;32m'; \
blu='[1;34m'; \
+ mgn='[0;35m'; \
+ brg='[1m'; \
std='[m'; \
}
else !%?COLOR%
-am__tty_colors = \
-red=; grn=; lgn=; blu=; std=
+am__tty_colors = $(am__tty_colors_dummy)
endif !%?COLOR%
.PHONY: check-TESTS
@@ -44,190 +49,245 @@ include inst-vars.am
## New parallel test driver.
##
-## This code is adapted from check.mk which was originally
-## written at EPITA/LRDE, further developed at Gostai, then made
-## its way from GNU coreutils to end up, largely rewritten, in
-## Automake.
-##
-## It provides special support for "unit tests", that is to say,
-## tests that (once run) no longer need to be re-compiled and
-## re-run at each "make check", unless their sources changed. To
-## enable unit-test supports, set RECHECK_LOGS to empty. In such a
-## setting, that heavily relies on correct dependencies, its users may
-## prefer to define EXTRA_PROGRAMS instead of check_PROGRAMS, because
-## it allows intertwined compilation and execution of the tests.
-## Sometimes this helps catching errors earlier (you don't have to
-## wait for all the tests to be compiled).
-##
-## Define TEST_SUITE_LOG to be the name of the global log to create.
-## Define TEST_LOGS to the set of logs to include in it. It defaults
-## to $(TESTS), with `.test' and address@hidden@' removed, and `'.log'
-## appended.
-##
-## In addition to the magic "exit 77 means SKIP" feature (which was
-## imported from automake), there is a magic "exit 99 means FAIL" feature
-## which is useful if you need to issue a hard error no matter whether the
-## test is XFAIL or not. You can disable this feature by setting the
-## variable DISABLE_HARD_ERRORS to a nonempty value.
+## The first version of the code here was adapted from check.mk, which was
+## originally written at EPITA/LRDE, further developed at Gostai, then made
+## its way from GNU coreutils to end up, largely rewritten, in Automake.
+## The current version is an heavy rewrite of that, to allow for support
+## of more test metadata, and the use of custom test derivers and protocols
+## (among them, TAP).
# Restructured Text title and section.
-am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
+am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
am__rst_section = sed 'p;s/./=/g;p;g'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
-
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
am__sh_e_setup = case $$- in *e*) set +e;; esac
+# Default flags passed to test drivers.
+am__common_driver_flags = \
+ --color-tests "$$am__color_tests" \
+ --enable-hard-errors "$$am__enable_hard_errors" \
+ --expect-failure "$$am__expect_failure"
+
# To be inserted before the command running the test. Creates the
# directory for the log if needed. Stores in $dir the directory
# containing $f, in $tst the test, in $log the log. Executes the
# developer- defined test setup AM_TESTS_ENVIRONMENT (if any), and
-# passes TESTS_ENVIRONMENT. Saves and restores TERM around uses of
-# TESTS_ENVIRONMENT and AM_TESTS_ENVIRONMENT, in case any of them
-# unsets it.
+# passes TESTS_ENVIRONMENT. Set up options for the wrapper that
+# will run the test scripts (or their associated LOG_COMPILER, if
+# thy have one).
am__check_pre = \
$(am__sh_e_setup); \
$(am__vpath_adj_setup) $(am__vpath_adj) \
+$(am__tty_colors); \
srcdir=$(srcdir); export srcdir; \
-rm -f address@hidden; \
-am__trap='rm -f '\''$(abs_builddir)/address@hidden'\''; (exit $$st); exit
$$st'; \
-trap "st=129; $$am__trap" 1; trap "st=130; $$am__trap" 2; \
-trap "st=141; $$am__trap" 13; trap "st=143; $$am__trap" 15; \
am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`; \
test "x$$am__odir" = x. || $(MKDIR_P) "$$am__odir" || exit $$?; \
if test -f "./$$f"; then dir=./; \
elif test -f "$$f"; then dir=; \
else dir="$(srcdir)/"; fi; \
-tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \
-$(AM_TESTS_ENVIRONMENT) \
-$(TESTS_ENVIRONMENT)
-
-# To be appended to the command running the test. Handle the stdout
-# and stderr redirection, and catch the exit status.
-am__check_post = \
->address@hidden 2>&1; \
-estatus=$$?; \
-if test -n '$(DISABLE_HARD_ERRORS)' \
- && test $$estatus -eq 99; then \
- estatus=1; \
-fi; \
-TERM=$$__SAVED_TERM; export TERM; \
-$(am__tty_colors); \
-xfailed=PASS; \
+tst=$$dir$$f; log='$@'; \
+if test -n '$(DISABLE_HARD_ERRORS)'; then \
+ am__enable_hard_errors=no; \
+else \
+ am__enable_hard_errors=yes; \
+fi; \
+## The use of $dir below is required to account for VPATH
+## rewriting done by Sun make.
case " $(XFAIL_TESTS) " in \
*[\ \ ]$$f[\ \ ]* | *[\ \ ]$$dir$$f[\ \ ]*) \
- xfailed=XFAIL;; \
-esac; \
-case $$estatus.$$xfailed in \
- 0.XFAIL) col=$$red; res=XPASS;; \
- 0.*) col=$$grn; res=PASS ;; \
- 77.*) col=$$blu; res=SKIP ;; \
- 99.*) col=$$red; res=FAIL ;; \
- *.XFAIL) col=$$lgn; res=XFAIL;; \
- *.*) col=$$red; res=FAIL ;; \
-esac; \
-echo "$${col}$$res$${std}: $$f"; \
-echo "$$res: $$f (exit: $$estatus)" | \
- $(am__rst_section) >$@; \
-cat address@hidden >>$@; \
-rm -f address@hidden
+ am__expect_failure=yes;; \
+ *) \
+ am__expect_failure=no;; \
+esac; \
+$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT)
+
+# A shell command to get the names of the tests scripts with any registered
+# extension removed (i.e., equivalently, the names of the test logs, with
+# the `.log' extension removed). The result is saved in the shell variable
+# `$bases'. This honors runtime overriding of TESTS and TEST_LOGS. Sadly,
+# we cannot use something simpler, involving e.g., "$(TEST_LOGS:.log=)",
+# since that might cause problem with VPATH rewrites for suffix-less tests.
+# See also 'test-harness-vpath-rewrite.test' and 'test-trs-basic.test'.
+am__set_TESTS_bases = \
+ bases='$(TEST_LOGS)'; \
+ bases=`for i in $$bases; do echo $$i; done | sed 's/\.log$$//'`; \
+## Trim away any extra whitespace. This has already proved useful in
+## avoiding weird bug on lesser make implementations.
+ bases=`echo $$bases`
+
+# Recover from deleted `.trs' file; this should ensure that
+# "rm -f foo.log; make foo.trs" re-run `foo.test', and re-create
+# both `foo.log' and `foo.trs'. Break the recipe in two subshells
+# to avoid problems with "make -n".
+.log.trs:
+ rm -f $< $@
+ $(MAKE) $(AM_MAKEFLAGS) $<
+
+# Helper recipe used by $(TEST_SUITE_LOG) below, to avoid problems with
+# "make -n". Break this recipe in multiple shell invocations too, to
+# really work as expected with "make -n".
+am--redo-logs:
+ @rm -f $$redo_logs
+ @rm -f $$redo_results
+ @$(MAKE) $(AM_MAKEFLAGS) $$redo_logs
+## Sanity check: each unreadable or non-existent test result file should
+## has been properly remade at this point, as should the corresponding log
+## file.
+ @st=0; \
+ errmsg="fatal: making $(TEST_SUITE_LOG): failed to create"; \
+ for i in $$redo_bases; do \
+ test -f $$i.trs && test -r $$i.trs \
+ || { echo "$$errmsg $$i.trs"; st=1; }; \
+ test -f $$i.log && test -r $$i.log \
+ || { echo "$$errmsg $$i.log"; st=1; }; \
+ done; \
+ test $$st -eq 0
+.PHONY: am--redo-logs
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
- list='$(TEST_LOGS)'; \
- results=`for f in $$list; do \
- test -r $$f && read line < $$f && echo "$$line" \
- || echo FAIL; \
- done`; \
- all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[
]*//'`; \
- fail=`echo "$$results" | grep -c '^FAIL'`; \
- pass=`echo "$$results" | grep -c '^PASS'`; \
- skip=`echo "$$results" | grep -c '^SKIP'`; \
- xfail=`echo "$$results" | grep -c '^XFAIL'`; \
- xpass=`echo "$$results" | grep -c '^XPASS'`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+ @$(am__set_TESTS_bases); \
+## Helper shell function, tells whether a path refers to an existing,
+## regular, readable file.
+ am__f_ok () { test -f "$$1" && test -r "$$1"; }; \
+## We need to ensures that all the required `.trs' and `.log' files will
+## be present and readable. The direct dependencies of $(TEST_SUITE_LOG)
+## only ensure that all the `.log' files exists; they don't ensure that
+## the `.log' files are readable, and worse, they don't ensure that the
+## `.trs' files even exist.
+ redo_bases=`for i in $$bases; do \
+ am__f_ok $$i.trs && am__f_ok $$i.log || echo $$i; \
+ done`; \
+ if test -n "$$redo_bases"; then \
+## Uh-oh, either some `.log' files were unreadable, or some `.trs' files
+## were missing (or unreadable). We need to re-run the corresponding
+## tests in order to re-create them.
+ redo_logs=`for i in $$redo_bases; do echo $$i.log; done`; \
+ redo_results=`for i in $$redo_bases; do echo $$i.trs; done`; \
+## The exported variables are needed by the helper hook.
+ redo_bases="$$redo_bases" \
+ redo_logs="$$redo_logs" \
+ redo_results="$$redo_results" \
+ $(MAKE) $(AM_MAKEFLAGS) am--redo-logs || exit 1; \
+ else :; fi;
+## We need a new subshell to work portably with "make -n", since the
+## previous part of the recipe contained a $(MAKE) invocation.
+ @$(am__sh_e_setup); $(am__tty_colors); $(am__set_TESTS_bases); \
+ ws='[ ]'; \
+## List of test result files.
+ results=`for b in $$bases; do echo $$b.trs; done`; \
+ test -n "$$results" || results=/dev/null; \
+## Prepare data for the test suite summary. These do not take into account
+## unreadable test results, but they'll be appropriately updated later if
+## needed.
+ all=` grep "^$$ws*:test-result:" $$results | wc -l`; \
+ pass=` grep "^$$ws*:test-result:$$ws*PASS" $$results | wc -l`; \
+ fail=` grep "^$$ws*:test-result:$$ws*FAIL" $$results | wc -l`; \
+ skip=` grep "^$$ws*:test-result:$$ws*SKIP" $$results | wc -l`; \
+ xfail=`grep "^$$ws*:test-result:$$ws*XFAIL" $$results | wc -l`; \
+ xpass=`grep "^$$ws*:test-result:$$ws*XPASS" $$results | wc -l`; \
+ error=`grep "^$$ws*:test-result:$$ws*ERROR" $$results | wc -l`; \
+## Whether the testsuite was successful or not.
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+## Make $br a line of exactly 76 `=' characters, that will be used to
+## enclose the testsuite summary report when displayed on the console.
+ br='==================='; br=$$br$$br$$br$$br; \
+## When writing the test summary to the console, we want to color a line
+## reporting the count of some result *only* if at least one test
+## experienced such a result. This function is handy in this regard.
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+## A shell function that creates the testsuite summary. We need it
+## because we have to create *two* summaries, one for test-suite.log,
+## and a possibly-colorized one for console output.
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
+## Write "global" testsuite log.
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
- for f in $$list; do \
- test -r $$f && read line < $$f || line=; \
- case $$line in \
- PASS:*|XFAIL:*);; \
- *) echo; cat $$f;; \
- esac; \
- done; \
+ for i in $$bases; do \
+## FIXME: one fork per test -- this is horrendously inefficient!
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$i.trs \
+ >/dev/null; then continue; \
+ fi; \
+## Get the declared "global result" of the test.
+## FIXME: yet another one fork per test here!
+ glob_res=`sed -n -e "s/$$ws*$$//" \
+ -e "s/^$$ws*:global-test-result:$$ws*//p" \
+ $$i.trs`; \
+## If no global result is explicitly declared, we'll merely mark the
+## test as "RUN" in the global test log.
+ test -n "$$glob_res" || glob_res=RUN; \
+## Write the name and result of the test as an RST section title.
+ echo "$$glob_res: $$i" | $(am__rst_section); \
+## If we should have remade any unreadable `.log', above.
+ if test ! -r $$i.log; then \
+ echo "fatal: making $@: $$i.log is unreadable" >&2; \
+ exit 1; \
+ fi; \
+ cat $$i.log; echo; \
+ done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
- $(am__tty_colors); \
- if $$exit; then \
+## Emit the test summary on the console.
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+## Multi line coloring is problematic with "less -R", so we really need
+## to color each line individually.
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+## This is expected to go to the console, so it might have to be colorized.
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
+ fi; \
+## Be sure to exit with the proper exit status.
+ $$success
RECHECK_LOGS = $(TEST_LOGS)
@@ -235,6 +295,7 @@ RECHECK_LOGS = $(TEST_LOGS)
check-TESTS:
## Expand $(RECHECK_LOGS) only once, to avoid exceeding line length limits.
@list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
+ @list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list
## We always have to remove TEST_SUITE_LOG, to ensure its rule is run
## in any case even in lazy mode: otherwise, if no test needs rerunning,
## or a prior run plus reruns all happen within the same timestamp
@@ -248,10 +309,13 @@ check-TESTS:
## 3.80 to erroneously expand $(TESTS_LOGS) to `foo.log .log'.
## Work around this bug.
@test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
- @list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test .log = $$f || echo $$f; \
- done | tr '\012\015' ' '`; \
+ @list='$(TEST_LOGS)'; \
+ list=`for i in $$list; do \
+ test .log = $$i || echo $$i; \
+ done | tr '\012\015' ' '`; \
+## This apparently useless munging helps to avoid a nasty bug (a
+## segmentation fault!) on Solaris XPG4 make.
+ list=`echo "$$list" | sed 's/ *$$//'`; \
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) TEST_LOGS="$$list"
AM_RECURSIVE_TARGETS += check
@@ -294,15 +358,18 @@ AM_RECURSIVE_TARGETS += check-html
## Rechecking failures. ##
## -------------------- ##
-## Rerun all FAILed or XPASSed tests.
+## Rerun all tests that experienced an error or an unexpected failure.
recheck recheck-html:
- @target=`echo $@ | sed 's,^re,,'`; \
- list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test -f $$f || continue; \
- if test -r $$f && read line < $$f; then \
- case $$line in FAIL*|XPASS*) echo $$f;; esac; \
- else echo $$f; fi; \
+ @ws='[ ]'; \
+ target=`echo $@ | sed 's,^re,,'`; \
+ $(am__set_TESTS_bases); \
+ list=`for i in $$bases; do \
+## Skip tests that haven't been run, but recover gracefully from deleted
+## `.trs' files.
+ test -f $$i.trs || test -f $$i.log || continue; \
+## FIXME: one fork per test -- this is horrendously inefficient!
+ grep "^$$ws*:recheck:$$ws*no$$ws*$$" $$i.trs \
+ >/dev/null 2>&1 || echo $$i.log; \
done | tr '\012\015' ' '`; \
## This apparently useless munging helps to avoid a nasty bug (a
## segmentation fault!) on Solaris XPG4 make.
@@ -330,7 +397,7 @@ check-TESTS: $(TESTS)
## why we also try `dir='
elif test -f $$tst; then dir=; \
else dir="$(srcdir)/"; fi; \
- if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
+ if $(TESTS_ENVIRONMENT) $${dir}$$tst $(AM_TESTS_FD_REDIRECT); then \
## Success
all=`expr $$all + 1`; \
case " $(XFAIL_TESTS) " in \
diff --git a/lib/am/check2.am b/lib/am/check2.am
index 054c62d..ad0a4aa 100644
--- a/lib/am/check2.am
+++ b/lib/am/check2.am
@@ -1,5 +1,5 @@
## automake - create Makefile.in from Makefile.am
-## Copyright (C) 2008, 2009 Free Software Foundation, Inc.
+## Copyright (C) 2008, 2009, 2011 Free Software Foundation, Inc.
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
@@ -14,10 +14,13 @@
## You should have received a copy of the GNU General Public License
## along with this program. If not, see <http://www.gnu.org/licenses/>.
-## From a test file to a log file.
+## From a test file to a .log and .trs file.
?GENERIC?%EXT%.log:
?!GENERIC?%OBJ%: %SOURCE%
- @p='%SOURCE%'; $(am__check_pre) %COMPILE% "$$tst" $(am__check_post)
+ @p='%SOURCE%'; $(am__check_pre) %DRIVER% --test-name "$$f" \
+ --log-file '%BASE%.log' --trs-file '%BASE%.trs' \
+ $(am__common_driver_flags) %DRIVER_FLAGS% -- %COMPILE% "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
## If no programs are built in this package, then this rule is removed
## at automake time. Otherwise, %am__EXEEXT% expands to a configure time
@@ -25,5 +28,8 @@
## conflict with the previous one.
if %am__EXEEXT%
?GENERIC?%EXT%$(EXEEXT).log:
- @p='%SOURCE%'; $(am__check_pre) %COMPILE% "$$tst" $(am__check_post)
+ @p='%SOURCE%'; $(am__check_pre) %DRIVER% --test-name "$$f" \
+ --log-file '%BASE%.log' --trs-file '%BASE%.trs' \
+ $(am__common_driver_flags) %DRIVER_FLAGS% -- %COMPILE% "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
endif %am__EXEEXT%
diff --git a/lib/tap-driver b/lib/tap-driver
new file mode 100755
index 0000000..2393346
--- /dev/null
+++ b/lib/tap-driver
@@ -0,0 +1,451 @@
+#! /usr/bin/env perl
+# Temporary/experimental TAP test driver for Automake.
+# TODO: should be rewritten portably (e.g., in awk or shell).
+
+# ---------------------------------- #
+# Imports, static data, and setup. #
+# ---------------------------------- #
+
+use warnings FATAL => 'all';
+use strict;
+use Getopt::Long ();
+use TAP::Parser;
+
+my $ME = "tap-driver";
+
+my $USAGE = <<'END';
+Usage:
+ tap-driver --test-name=NAME --log-file=PATH --trs-file=PATH
+ [--expect-failure={yes|no}] [--color-tests={yes|no}]
+ [--enable-hard-errors={yes|no}] [--ignore-exit]
+ [--diagnostic-string=STRING] [--merge|--no-merge]
+ [--comments|--no-comments] [--] TEST-COMMAND
+The `--test-name' and `--log-file' options are mandatory.
+END
+
+my $HELP = "$ME: TAP-aware test driver for Automake testsuite harness." .
+ "\n" . $USAGE;
+
+my $VERSION = '(experimental version)';
+
+# Keep this in sync with `lib/am/check.am:$(am__tty_colors)'.
+my %COLOR = (
+ red => "\e[0;31m",
+ grn => "\e[0;32m",
+ lgn => "\e[1;32m",
+ blu => "\e[1;34m",
+ mgn => "\e[0;35m",
+ brg => "\e[1m",
+ std => "\e[m",
+);
+
+# ------------------- #
+# Global variables. #
+# ------------------- #
+
+my $testno = 0; # Number of test results seen so far.
+my $plan_seen = 0; # Whether the TAP plan has been seen or not.
+my $parser; # TAP parser object (will be initialized later).
+
+# When true, it means that the rest of the input stream cannot
+# contain any further TAP results.
+my $tap_stopped = 0;
+
+# ----------------- #
+# Option parsing. #
+# ----------------- #
+
+my %cfg = (
+ "color-tests" => 0,
+ "expect-failure" => 0,
+ "enable-hard-errors" => 1,
+ "merge" => 0,
+ "comments" => 0,
+ "ignore-exit" => 0,
+);
+
+my $test_script_name = undef;
+my $log_file = undef;
+my $trs_file = undef;
+my $diag_string = "#";
+
+Getopt::Long::GetOptions (
+ 'help' => sub { print $HELP; exit 0; },
+ 'version' => sub { print "$ME $VERSION\n"; exit 0; },
+ 'test-name=s' => \$test_script_name,
+ 'log-file=s' => \$log_file,
+ 'trs-file=s' => \$trs_file,
+ 'color-tests=s' => \&bool_opt,
+ 'expect-failure=s' => \&bool_opt,
+ 'enable-hard-errors=s' => \&bool_opt,
+ 'diagnostic-string=s' => \$diag_string,
+ 'comments' => sub { $cfg{"comments"} = 1; },
+ 'no-comments' => sub { $cfg{"comments"} = 0; },
+ 'merge' => sub { $cfg{"merge"} = 1; },
+ 'no-merge' => sub { $cfg{"merge"} = 0; },
+ 'ignore-exit' => sub { $cfg{"ignore-exit"} = 1; },
+ ) or exit 1;
+
+# ------------- #
+# Prototypes. #
+# ------------- #
+
+sub add_test_result ($);
+sub bool_opt ($$);
+sub colored ($$);
+sub copy_in_global_log ();
+sub decorate_result ($);
+sub extract_tap_comment ($);
+sub finish ();
+sub get_global_test_result ();
+sub get_test_exit_message ();
+sub get_test_results ();
+sub handle_tap_bailout ($);
+sub handle_tap_plan ($);
+sub handle_tap_test ($);
+sub main (@);
+sub must_recheck ();
+sub report ($;$);
+sub start (@);
+sub stringify_test_result ($);
+sub testsuite_error ($);
+sub write_test_results ();
+sub yn ($);
+
+# -------------- #
+# Subroutines. #
+# -------------- #
+
+sub bool_opt ($$)
+{
+ my ($opt, $val) = @_;
+ if ($val =~ /^(?:y|yes)\z/i)
+ {
+ $cfg{$opt} = 1;
+ }
+ elsif ($val =~ /^(?:n|no)\z/i)
+ {
+ $cfg{$opt} = 0;
+ }
+ else
+ {
+ die "invalid argument '$val' for option '$opt'\n";
+ }
+}
+
+# Convert a boolean to a "yes"/"no" string.
+sub yn ($)
+{
+ my $bool = shift;
+ return $bool ? "yes" : "no";
+}
+
+TEST_RESULTS :
+{
+ my (@test_results, %test_results);
+
+ sub add_test_result ($)
+ {
+ my $res = shift;
+ push @test_results, $res;
+ $test_results{$res} = 1;
+ }
+
+ sub get_test_results ()
+ {
+ return @test_results;
+ }
+
+ # Whether the test script should be re-run by "make recheck".
+ sub must_recheck ()
+ {
+ return grep { !/^(?:XFAIL|PASS|SKIP)$/ } (keys %test_results);
+ }
+
+ # Whether the content of the log file associated to this test should
+ # be copied into the "global" test-suite.log.
+ sub copy_in_global_log ()
+ {
+ return grep { not $_ eq "PASS" } (keys %test_results);
+ }
+
+ # FIXME: this can certainly be improved ...
+ sub get_global_test_result ()
+ {
+ my @results = keys %test_results;
+ return "ERROR" if exists $test_results{"ERROR"};
+ return "SKIP" if @results == 1 && $results[0] eq "SKIP";
+ return "FAIL" if exists $test_results{"FAIL"};
+ return "FAIL" if exists $test_results{"XPASS"};
+ return "PASS";
+ }
+
+}
+
+sub write_test_results ()
+{
+ open RES, ">", $trs_file or die "opening $trs_file: $!\n";
+ print RES ":global-test-result: " . get_global_test_result . "\n";
+ print RES ":recheck: " . yn (must_recheck) . "\n";
+ print RES ":copy-in-global-log: " . yn (copy_in_global_log) . "\n";
+ foreach my $result (get_test_results)
+ {
+ print RES ":test-result: $result\n";
+ }
+ close RES or die "closing $trs_file: $!\n";
+}
+
+sub start (@)
+{
+ # Redirect stderr and stdout to a temporary log file. Save the
+ # original stdout stream, since we need it to print testsuite
+ # progress output.
+ open LOG, ">", $log_file or die "opening $log_file: $!\n";
+ open OLDOUT, ">&STDOUT" or die "duplicating stdout: $!\n";
+ open STDOUT, ">&LOG" or die "redirecting stdout: $!\n";
+ open STDERR, ">&LOG" or die "redirecting stderr: $!\n";
+ $parser = TAP::Parser->new ({ exec => address@hidden, merge => $cfg{merge}
});
+ $parser->ignore_exit(1) if $cfg{"ignore-exit"};
+}
+
+sub get_test_exit_message ()
+{
+ # Flush all the remaining TAP stream, so that we can obtain the
+ # exit status of the TAP producer.
+ do {} while defined $parser->next;
+ my $wstatus = $parser->wait;
+ # Return an undefined value if the producer exited with success.
+ return unless $wstatus;
+ # Otherwise, determine whether it exited with error or was terminated
+ # by a signal.
+ use POSIX qw (WIFEXITED WEXITSTATUS WIFSIGNALED WTERMSIG);
+ if (WIFEXITED ($wstatus))
+ {
+ return sprintf "exited with status %d", WEXITSTATUS ($wstatus);
+ }
+ elsif (WIFSIGNALED ($wstatus))
+ {
+ return sprintf "terminated by signal %d", WTERMSIG ($wstatus);
+ }
+ else
+ {
+ return "terminated abnormally";
+ }
+}
+
+sub finish ()
+{
+ if (!$cfg{"ignore-exit"} and my $msg = get_test_exit_message)
+ {
+ testsuite_error $msg;
+ }
+ write_test_results;
+ close LOG or die "closing $log_file: $!\n";
+ exit 0;
+}
+
+sub stringify_test_result ($)
+{
+ my $result = shift;
+ my $PASS = $cfg{"expect-failure"} ? "XPASS": "PASS";
+ my $FAIL = $cfg{"expect-failure"} ? "XFAIL": "FAIL";
+ if ($result->is_unplanned || $result->number != $testno || $tap_stopped)
+ {
+ return "ERROR";
+ }
+ elsif (!$result->directive)
+ {
+ return $result->is_ok ? $PASS: $FAIL;
+ }
+ elsif ($result->has_todo)
+ {
+ return $result->is_actual_ok ? "XPASS" : "XFAIL";
+ }
+ elsif ($result->has_skip)
+ {
+ return $result->is_ok ? "SKIP" : $FAIL;
+ }
+ die "INTERNAL ERROR"; # NOTREACHED
+}
+
+sub colored ($$)
+{
+ my ($color_name, $text) = @_;
+ return $COLOR{$color_name} . $text . $COLOR{'std'};
+}
+
+sub decorate_result ($)
+{
+ my $result = shift;
+ return $result unless $cfg{"color-tests"};
+ my %color_for_result =
+ (
+ "ERROR" => 'mgn',
+ "PASS" => 'grn',
+ "XPASS" => 'red',
+ "FAIL" => 'red',
+ "XFAIL" => 'lgn',
+ "SKIP" => 'blu',
+ );
+ if (my $color = $color_for_result{$result})
+ {
+ return colored ($color, $result);
+ }
+ else
+ {
+ return $result; # Don't colorize unknown stuff.
+ }
+}
+
+sub report ($;$)
+{
+ my ($msg, $result, $explanation) = (undef, @_);
+ if ($result =~ /^(?:X?(?:PASS|FAIL)|SKIP|ERROR)/)
+ {
+ $msg = ": $test_script_name";
+ add_test_result $result;
+ }
+ elsif ($result eq "#")
+ {
+ $msg = " $test_script_name:";
+ }
+ else
+ {
+ die "INTERNAL ERROR"; # NOTREACHED
+ }
+ $msg .= " $explanation" if defined $explanation;
+ $msg .= "\n";
+ # Output on console might be colorized.
+ print OLDOUT decorate_result ($result) . $msg;
+ # Log the result in the log file too, to help debugging (this is
+ # especially true when said result is a TAP error or "Bail out!").
+ print $result . $msg;
+}
+
+sub testsuite_error ($)
+{
+ report "ERROR", "- $_[0]";
+}
+
+sub handle_tap_test ($)
+{
+ $testno++;
+ my $test = shift;
+
+ my $test_result = stringify_test_result $test;
+ my $string = $test->number;
+
+ if (my $description = $test->description)
+ {
+ $string .= " $description";
+ }
+
+ if ($tap_stopped)
+ {
+ $string .= " # AFTER LATE PLAN";
+ }
+ elsif ($test->is_unplanned)
+ {
+ $string .= " # UNPLANNED";
+ }
+ elsif ($test->number != $testno)
+ {
+ $string .= " # OUT-OF-ORDER (expecting $testno)";
+ }
+ elsif (my $directive = $test->directive)
+ {
+ $string .= " # $directive";
+ if (my $explanation = $test->explanation)
+ {
+ $string .= " $explanation";
+ }
+ }
+
+ report $test_result, $string;
+}
+
+sub handle_tap_plan ($)
+{
+ my $plan = shift;
+ # Only one plan per stream is acceptable.
+ testsuite_error "multiple test plans" if $plan_seen;
+ $plan_seen = 1;
+ # TAP plan must come either before or after *all* the TAP results.
+ # So, if we find it after having already seen at least one TAP result,
+ # set a flag signaling that no more TAP results are acceptable.
+ $tap_stopped = 1 if $testno >= 1;
+ # Nothing more to do, unless the plan contains a SKIP directive.
+ return
+ if not defined $plan->directive && length ($plan->directive) > 0;
+ my $explanation = $plan->explanation ?
+ "- " . $plan->explanation : undef;
+ report "SKIP", $explanation;
+ finish;
+}
+
+sub handle_tap_bailout ($)
+{
+ my ($bailout, $msg) = ($_[0], "Bail out!");
+ $msg .= " " . $bailout->explanation if $bailout->explanation;
+ testsuite_error $msg;
+ finish;
+}
+
+sub extract_tap_comment ($)
+{
+ local $_ = shift;
+ if (/^\Q$diag_string\E(.*)$/o)
+ {
+ (my $comment = $1) =~ s/(?:^\s*|\s*$)//g;
+ return $comment;
+ }
+ return "";
+}
+
+sub main (@)
+{
+ start @_;
+
+ while (defined (my $cur = $parser->next))
+ {
+ # Verbatim copy any input line into the log file.
+ print $cur->raw . "\n";
+ if ($cur->is_plan)
+ {
+ handle_tap_plan ($cur);
+ }
+ elsif ($cur->is_test)
+ {
+ handle_tap_test ($cur);
+ }
+ elsif ($cur->is_bailout)
+ {
+ handle_tap_bailout ($cur);
+ }
+ elsif ($cfg{comments})
+ {
+ my $comment = extract_tap_comment ($cur->raw);
+ report "#", "$comment" if length $comment;
+ }
+ }
+ if (!$plan_seen)
+ {
+ testsuite_error "missing test plan";
+ }
+ elsif ($parser->tests_planned != $parser->tests_run)
+ {
+ my ($planned, $run) = ($parser->tests_planned, $parser->tests_run);
+ my $bad_amount = $run > $planned ? "many" : "few";
+ testsuite_error (sprintf "too %s tests run (expected %d, got %d)",
+ $bad_amount, $planned, $run);
+ }
+ finish;
+}
+
+# ----------- #
+# Main code. #
+# ----------- #
+
+main @ARGV;
+
+# vim: ft=perl ts=4 sw=4 et
diff --git a/lib/test-driver b/lib/test-driver
new file mode 100755
index 0000000..8522a49
--- /dev/null
+++ b/lib/test-driver
@@ -0,0 +1,127 @@
+#! /bin/sh
+# test-driver - basic driver script for the `parallel-tests' mode.
+
+scriptversion=2011-08-04.22; # UTC
+
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# This file is maintained in Automake, please report
+# bugs to <address@hidden> or send patches to
+# <address@hidden>.
+
+# Make unconditional expansion of undefined variables an error. This
+# helps a lot in preventing typo-related bugs.
+set -u
+
+usage_error ()
+{
+ echo "$0: $*" >&2
+ print_usage >&2
+ exit 2
+}
+
+print_usage ()
+{
+ cat <<END
+Usage:
+ test-driver --test-name=NAME --log-file=PATH --trs-file=PATH
+ [--expect-failure={yes|no}] [--color-tests={yes|no}]
+ [--enable-hard-errors={yes|no}] [--] TEST-SCRIPT
+The \`--test-name' and \`--log-file' options are mandatory.
+END
+}
+
+# TODO: better error handling in option parsing (in particular, ensure
+# TODO: $log_file, $trs_file and $test_name are defined).
+test_name= # Used for reporting.
+log_file= # Where to save the output of the test script.
+trs_file= # Where to save the metadata of the test run.
+expect_failure=no
+color_tests=no
+enable_hard_errors=yes
+while test $# -gt 0; do
+ case $1 in
+ --help) print_usage; exit $?;;
+ --version) echo "test-driver $scriptversion"; exit $?;;
+ --test-name) test_name=$2; shift;;
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --color-tests) color_tests=$2; shift;;
+ --expect-failure) expect_failure=$2; shift;;
+ --enable-hard-errors) enable_hard_errors=$2; shift;;
+ --) shift; break;;
+ -*) usage_error "invalid option: '$1'";;
+ esac
+ shift
+done
+
+if test $color_tests = yes; then
+ # Keep this in sync with `lib/am/check.am:$(am__tty_colors)'.
+ red='[0;31m' # Red.
+ grn='[0;32m' # Green.
+ lgn='[1;32m' # Light green.
+ blu='[1;34m' # Blue.
+ mgn='[0;35m' # Magenta.
+ std='[m' # No color.
+else
+ red= grn= lgn= blu= mgn= std=
+fi
+
+do_exit='rm -f $log_file $trs_file; (exit $st); exit $st'
+trap "st=129; $do_exit" 1
+trap "st=130; $do_exit" 2
+trap "st=141; $do_exit" 13
+trap "st=143; $do_exit" 15
+
+# Test script is run here.
+"$@" >$log_file 2>&1
+estatus=$?
+if test $enable_hard_errors = no && test $estatus -eq 99; then
+ estatus=1
+fi
+
+case $estatus:$expect_failure in
+ 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;;
+ 0:*) col=$grn res=PASS recheck=no gcopy=no;;
+ 77:*) col=$blu res=SKIP recheck=no gcopy=yes;;
+ 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;;
+ *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;;
+ *:*) col=$red res=FAIL recheck=yes gcopy=yes;;
+esac
+
+# Report outcome to console.
+echo "${col}${res}${std}: $test_name"
+
+# Register the test result, and other relevant metadata.
+echo ":test-result: $res" > $trs_file
+echo ":global-test-result: $res" >> $trs_file
+echo ":recheck: $recheck" >> $trs_file
+echo ":copy-in-global-log: $gcopy" >> $trs_file
+
+# Local Variables:
+# mode: shell-script
+# sh-indentation: 2
+# eval: (add-hook 'write-file-hooks 'time-stamp)
+# time-stamp-start: "scriptversion="
+# time-stamp-format: "%:y-%02m-%02d.%02H"
+# time-stamp-time-zone: "UTC"
+# time-stamp-end: "; # UTC"
+# End:
diff --git a/tests/.gitignore b/tests/.gitignore
index a6d0cce..4ec56b1 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -1,14 +1,8 @@
-aclocal-*
-automake-*
-defs-static
-instspc-tests.am
-parallel-tests.am
-config-shell-tests.am
-depmod-tests.am
-*.dir
-*.log
-*.log-t
-*-p.test
-instspc-*-build.test
-instspc-*-install.test
-*.test-t
+/aclocal-*
+/automake-*
+/defs-static
+/parallel-tests.am
+/config-shell-tests.am
+/*.dir
+/*.log
+/*.trs
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 925c2ca..f2e9c2d 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -16,13 +16,28 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# Run the tests with the shell detected at configure time.
+LOG_COMPILER = $(SHELL)
+
+TEST_EXTENSIONS = .test .tap
+TEST_LOG_COMPILER = $(LOG_COMPILER)
+TAP_LOG_COMPILER = $(LOG_COMPILER)
+
+TAP_LOG_DRIVER = $(PERL) $(top_srcdir)/lib/tap-driver
+TAP_LOG_DRIVER_FLAGS = --merge --comments
+## Avoid spurious TAP diagnostic. We use an awful indirection with a
+## shell command substitution here, since the string needs to contain
+## `#' characters, and these can't be escaped portably in a make macro
+## definition.
+## FIXME: this is quite inefficient though, as it adds one extra fork
+## FIXME: per TAP test script; this is OK for the moment, since we have
+## FIXME: few such scripts, but might become an issue in the future.
+## Keep this in sync with the definition of $diag_string_ in tests/defs.
+TAP_LOG_DRIVER_FLAGS += --diagnostic-string `printf '\043%%\043\n'`
+
MAINTAINERCLEANFILES =
EXTRA_DIST = ChangeLog-old
-TEST_EXTENSIONS = .test
-# Run the tests with the shell detected at configure time.
-TEST_LOG_COMPILER = $(SHELL)
-
XFAIL_TESTS = \
all.test \
auxdir2.test \
@@ -31,6 +46,7 @@ gcj6.test \
override-conditional-2.test \
pr8365-remake-timing.test \
yacc-dist-nobuild-subdir.test \
+tap-message-0.test \
txinfo5.test
@@ -41,7 +57,7 @@ $(srcdir)/parallel-tests.am: gen-parallel-tests Makefile.am
EXTRA_DIST += gen-parallel-tests
TEST_EXTENSIONS += .ptest
-PTEST_LOG_COMPILER = $(SHELL) $(srcdir)/parallel-tests.sh
+PTEST_LOG_COMPILER = $(LOG_COMPILER) $(srcdir)/parallel-tests.sh
EXTRA_DIST += parallel-tests.sh
# All `*-p.ptest' tests work by sourcing the script `parallel-tests.sh'.
@@ -51,48 +67,6 @@ $(parallel_tests:.ptest=.log): parallel-tests.sh
$(parallel_tests):
-include $(srcdir)/instspc-tests.am
-
-TEST_EXTENSIONS += .instspc
-INSTSPC_LOG_COMPILER = $(SHELL) $(srcdir)/instspc-tests.sh
-EXTRA_DIST += instspc-tests.sh
-
-$(srcdir)/instspc-tests.am: instspc-tests.sh Makefile.am
- $(AM_V_GEN)($(am__cd) $(srcdir) \
- && $(SHELL) ./instspc-tests.sh --generate-makefile) >$@
-
-# All `*.instspc' tests work by calling the script `instspc-tests.sh'.
-# Also, they all use shared data generated by the `instspc-data.test'
-# helper test (for reasons of speed).
-instspc-data.log: instspc-tests.sh
-$(instspc_tests:.instspc=.log): instspc-tests.sh instspc-data.log
-# This dummy declaration is required to have make actually produce
-# expected log files from the `.instspc.log' suffix rule.
-$(instspc_tests):
-
-XFAIL_TESTS += $(instspc_xfail_tests)
-
-
-include $(srcdir)/depmod-tests.am
-
-$(srcdir)/depmod-tests.am: depmod-tests.sh Makefile.am
- $(AM_V_GEN)($(am__cd) $(srcdir) \
- && $(SHELL) ./depmod-tests.sh --generate-makefile) >$@
-
-TEST_EXTENSIONS += .depmod
-DEPMOD_LOG_COMPILER = $(SHELL) $(srcdir)/depmod-tests.sh
-EXTRA_DIST += depmod-tests.sh
-
-# All `*.depmod' tests work by calling the script `depmod-tests.sh'.
-# Also, they all use shared data generated by the `depmod-data.test'
-# helper test (for reasons of speed).
-depmod-data.log: depmod-tests.sh
-$(depmod_tests:.depmod=.log): depmod-tests.sh depmod-data.log
-# This dummy declaration is required to have make actually produce
-# expected log files from the `.depmod.log' suffix rule.
-$(depmod_tests):
-
-
include $(srcdir)/config-shell-tests.am
$(srcdir)/config-shell-tests.am: gen-config-shell-tests Makefile.am
@@ -100,7 +74,7 @@ $(srcdir)/config-shell-tests.am: gen-config-shell-tests
Makefile.am
EXTRA_DIST += gen-config-shell-tests
TEST_EXTENSIONS += .shtst
-SHTST_LOG_COMPILER = $(SHELL) $(srcdir)/config-shell-tests.sh
+SHTST_LOG_COMPILER = $(LOG_COMPILER) $(srcdir)/config-shell-tests.sh
EXTRA_DIST += config-shell-tests.sh
# All `*-w.shtst' tests work by sourcing the script `config-shell-tests.sh'.
@@ -112,39 +86,37 @@ $(config_shell_tests):
# Some testsuite-influential variables should be overridable from the
# test scripts, but not from the environment.
-# The `AM_TESTS_REEXEC=no' setting tells the tests not to needlessly
-# re-execute themselves with the shell detected at configure time, since
-# we are already running them under it explicitly in our setup (see e.g.
-# the definition of TEST_LOG_COMPILER above).
-# We want warning messages and explanations for skipped tests to go to
-# the console if possible, so set up `stderr_fileno_' properly.
-# The `9>&2' redirection *must* be placed at the end, and without a
-# following semicolon, because some shells (e.g., various Korn Shells
-# or HP-HX /bin/sh) close any file descriptor > 2 upon `exec' sycall.
-# For more references, with lots of discussion, see:
-# - http://lists.gnu.org/archive/html/bug-autoconf/2011-06/msg00002.html
-# - http://thread.gmane.org/gmane.comp.gnu.coreutils.bugs/22488
-# - http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8846
AM_TESTS_ENVIRONMENT = \
test x"$$me" = x || unset me; \
test x"$$required" = x || unset required; \
+ test x"$$use_tap" = x || unset use_tap; \
test x"$$parallel_tests" = x || unset parallel_tests; \
test x"$$test_prefer_config_shell" || unset test_prefer_config_shell; \
test x"$$original_AUTOMAKE" = x || unset original_AUTOMAKE; \
- test x"$$original_ACLOCAL" = x || unset original_ACLOCAL; \
- AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC; \
- stderr_fileno_=9; export stderr_fileno_; 9>&2
+ test x"$$original_ACLOCAL" = x || unset original_ACLOCAL;
+# The `AM_TESTS_REEXEC=no' setting tells the tests not to needlessly
+# re-execute themselves with the shell detected at configure time, since
+# we are already running them under it explicitly in our setup (see e.g.
+# the definition of TEST_LOG_COMPILER above).
+AM_TESTS_ENVIRONMENT += AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC;
+# We want warning messages and explanations for skipped tests to go to
+# the console if possible, so set up `stderr_fileno_' properly.
+AM_TESTS_FD_REDIRECT = 9>&2
+AM_TESTS_ENVIRONMENT += stderr_fileno_=9; export stderr_fileno_;
TESTS = \
$(handwritten_tests) \
$(config_shell_tests) \
- $(parallel_tests) \
- $(depmod_tests) \
- $(instspc_tests)
+ $(parallel_tests)
EXTRA_DIST += $(handwritten_tests)
handwritten_tests = \
+## This tests might quite a long time, since they run various checks
+## sequentially; so place them early first to improve performance on
+## concurrent testsuite runs.
+depmod.tap \
+instspc.tap \
aclocal.test \
aclocal3.test \
aclocal4.test \
@@ -167,16 +139,11 @@ acloca20.test \
acloca21.test \
acloca22.test \
acloca23.test \
-acoutnoq.test \
-acoutpt.test \
-acoutpt2.test \
-acoutqnl.test \
-acoutbs.test \
-acoutbs2.test \
+ac-output-old.tap \
acsilent.test \
acsubst.test \
acsubst2.test \
-add-missing.test \
+add-missing.tap \
all.test \
all2.test \
alloca.test \
@@ -224,7 +191,7 @@ autohdr2.test \
autohdr3.test \
autohdr4.test \
autohdrdry.test \
-automake.test \
+automake-cmdline.tap \
auxdir.test \
auxdir2.test \
auxdir3.test \
@@ -276,9 +243,9 @@ check12.test \
check-subst.test \
check-subst-prog.test \
check-exported-srcdir.test \
+check-fd-redirect.test \
check-tests-in-builddir.test \
-check-tests_environment.test \
-tests-environment-backcompat.test \
+check-no-test-driver.test \
checkall.test \
clean.test \
clean2.test \
@@ -425,7 +392,6 @@ depcomp6.test \
depcomp7.test \
depcomp8a.test \
depcomp8b.test \
-depmod-data.test \
depdist.test \
depend.test \
depend2.test \
@@ -572,7 +538,6 @@ instman2.test \
instmany.test \
instmany-mans.test \
instmany-python.test \
-instspc-data.test \
interp.test \
interp2.test \
java.test \
@@ -762,10 +727,12 @@ parallel-tests9.test \
parallel-tests10.test \
parallel-tests-suffix.test \
parallel-tests-suffix-prog.test \
-parallel-tests-ext-driver.test \
-parallel-tests-ext-driver-prog.test \
-parallel-tests-am_tests_environment.test \
-parallel-tests-unreadable-log.test \
+parallel-tests-log-compiler-1.test \
+parallel-tests-log-compiler-2.test \
+parallel-tests-dry-run.test \
+parallel-tests-fd-redirect.test \
+parallel-tests-extra-programs.test \
+parallel-tests-unreadable.test \
parallel-tests-subdir.test \
parallel-tests-interrupt.test \
parallel-tests-reset-term.test \
@@ -773,8 +740,45 @@ parallel-tests-harderror.test \
parallel-tests-log-override-1.test \
parallel-tests-log-override-2.test \
parallel-tests-log-override-recheck.test \
-parallel-tests-cmdline-override.test \
parallel-tests-log-compiler-example.test \
+parallel-tests-cmdline-override.test \
+parallel-tests-empty-testlogs.test \
+parallel-tests-driver-install.test \
+parallel-tests-no-color-in-log.test \
+parallel-tests-no-spurious-summary.test \
+parallel-tests-exit-statuses.test \
+parallel-tests-console-output.test \
+parallel-tests-once.test \
+tests-environment.test \
+am-tests-environment.test \
+tests-environment-backcompat.test \
+testsuite-summary-color.test \
+testsuite-summary-count.test \
+testsuite-summary-count-many.test \
+testsuite-summary-reference-log.test \
+test-driver-custom-no-extra-driver.test \
+test-driver-custom.test \
+test-driver-custom-xfail-tests.test \
+test-driver-custom-multitest.test \
+test-driver-custom-multitest-recheck.test \
+test-driver-custom-multitest-recheck2.test \
+test-driver-custom-html.test \
+test-driver-custom-no-html.test \
+test-driver-create-log-dir.test \
+test-driver-strip-vpath.test \
+test-driver-trs-suffix-registered.test \
+test-driver-fail.test \
+test-harness-vpath-rewrite.test \
+test-log.test \
+test-metadata-global-log.test \
+test-metadata-global-result.test \
+test-metadata-recheck.test \
+test-metadata-results.test \
+test-missing.test \
+test-missing2.test \
+test-trs-basic.test \
+test-trs-recover.test \
+test-trs-recover2.test \
parse.test \
percent.test \
percent2.test \
@@ -895,15 +899,16 @@ repeated-options.test \
rulepat.test \
self-check-cleanup.test \
self-check-dir.test \
-self-check-env-sanitize.test \
-self-check-exit.test \
+self-check-env-sanitize.tap \
+self-check-exit.tap \
self-check-explicit-skips.test \
self-check-is_newest.test \
-self-check-me.test \
+self-check-me.tap \
self-check-reexec.test \
self-check-report.test \
self-check-sanity.test \
self-check-seq.test \
+self-check-tap.test \
self-check-unindent.test \
sanity.test \
scripts.test \
@@ -996,22 +1001,25 @@ substre2.test \
substtarg.test \
suffix.test \
suffix2.test \
-suffix3.test \
+suffix3.tap \
suffix4.test \
suffix5.test \
suffix6.test \
suffix6b.test \
suffix6c.test \
suffix7.test \
-suffix8.test \
+suffix8.tap \
suffix9.test \
-suffix10.test \
-suffix11.test \
+suffix10.tap \
+suffix11.tap \
suffix12.test \
suffix13.test \
+suffix-chain.tap \
symlink.test \
symlink2.test \
syntax.test \
+$(tap_with_common_setup_tests) \
+$(tap_other_tests) \
tags.test \
tags2.test \
tagsub.test \
@@ -1122,14 +1130,106 @@ yflags-force-override.test \
yflags-force-conditional.test \
yflags-var-expand.test
+## Files containing auxiliary functions used by our test cases.
+EXTRA_DIST += tap-functions.sh plain-functions.sh
+
+test-driver-custom-multitest.log: trivial-test-driver
+test-driver-custom-multitest-recheck.log: trivial-test-driver
+test-driver-custom-multitest-recheck2.log: trivial-test-driver
+test-driver-custom-html.log: trivial-test-driver
+EXTRA_DIST += trivial-test-driver
+
+testsuite-summary-color.log testsuite-summary-count.log: \
+ testsuite-summary-checks.sh extract-testsuite-summary
+EXTRA_DIST += testsuite-summary-checks.sh
+EXTRA_DIST += extract-testsuite-summary
+
+testsuite-summary-count-many.log: trivial-test-driver
+testsuite-summary-count-many.log: extract-testsuite-summary
+
+# List of tests on TAP support that use the files pre-computed by
+# `tap-common-setup.test', and sources the `tap-setup.sh' helper
+# script.
+tap_with_common_setup_tests = \
+tap-autonumber.test \
+tap-bailout.test \
+tap-color.test \
+tap-deps.test \
+tap-diagnostic.test \
+tap-empty-diagnostic.test \
+tap-empty.test \
+tap-escape-directive.test \
+tap-exit.test \
+tap-signal.test \
+tap-fancy.test \
+tap-fancy2.test \
+tap-global-log.test \
+tap-global-result.test \
+tap-html.test \
+tap-log.test \
+tap-merge-stdout-stderr.test \
+tap-no-merge-stdout-stderr.test \
+tap-message-0.test \
+tap-no-disable-hard-error.test \
+tap-no-spurious-summary.test \
+tap-no-spurious.test \
+tap-not-ok-skip.test \
+tap-numeric-description.test \
+tap-out-of-order.test \
+tap-passthrough.test \
+tap-passthrough-exit.test \
+tap-plan.test \
+tap-plan-corner.test \
+tap-plan-errors.test \
+tap-plan-middle.test \
+tap-realtime.test \
+tap-recheck-logs.test \
+tap-skip-whole-whitespace.test \
+tap-skip-whole.test \
+tap-skip-whole-lastline.test \
+tap-skip-whole-badexit.test \
+tap-todo-skip-together.test \
+tap-todo-skip-whitespace.test \
+tap-todo-skip.test \
+tap-unplanned.test \
+tap-whitespace-normalization.test \
+tap-with-and-without-number.test \
+tap-xfail-tests.test
+
+# Their log files.
+tap_with_common_setup_logs = $(tap_with_common_setup_tests:.test=.log)
+
+# Their dependencies.
+$(tap_with_common_setup_logs): tap-common-setup.log tap-setup.sh
+EXTRA_DIST += tap-setup.sh
+
+# Other tests on TAP support.
+tap_other_tests = \
+tap-common-setup.test \
+tap-bad-prog.tap \
+tap-basic.test \
+tap-diagnostic-custom.test \
+tap-doc.test \
+tap-doc2.test \
+tap-more.test \
+tap-more2.test \
+tap-recheck.test \
+tap-summary.test \
+tap-summary-color.test
+
+tap-summary.log tap-summary-color.log: tap-summary-aux.sh
+EXTRA_DIST += tap-summary-aux.sh
-## Checking the test of tests.
-checked_test_extensions = .test
+## Checking the list of tests.
+checked_test_extensions = .test .tap
expected_list_of_tests = $(handwritten_tests)
include $(top_srcdir)/CheckListOfTests.am
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
+# FIXME: this should be made more granular once we have a cleaner
+# subdivision of the tests.
+$(TEST_LOGS): plain-functions.sh tap-functions.sh
clean-local: clean-local-check
.PHONY: clean-local-check
diff --git a/tests/Makefile.in b/tests/Makefile.in
index f5aea3b..2821858 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -52,8 +52,7 @@ host_triplet = @host@
DIST_COMMON = README $(srcdir)/Makefile.am $(srcdir)/Makefile.in \
$(srcdir)/aclocal.in $(srcdir)/automake.in \
$(srcdir)/config-shell-tests.am $(srcdir)/defs \
- $(srcdir)/defs-static.in $(srcdir)/depmod-tests.am \
- $(srcdir)/instspc-tests.am $(srcdir)/parallel-tests.am \
+ $(srcdir)/defs-static.in $(srcdir)/parallel-tests.am \
$(top_srcdir)/CheckListOfTests.am
subdir = tests
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
@@ -79,19 +78,25 @@ am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY))
am__v_at_0 = @
SOURCES =
DIST_SOURCES =
+am__tty_colors_dummy = \
+ mgn= red= grn= lgn= blu= brg= std=; \
+ am__color_tests=no
# If stdout is a non-dumb tty, use colors. If test -t is not supported,
# then this fails; a conservative approach. Of course do not redirect
# stdout here, just stderr.
am__tty_colors = \
-red=; grn=; lgn=; blu=; std=; \
+$(am__tty_colors_dummy); \
test "X$(AM_COLOR_TESTS)" != Xno \
&& test "X$$TERM" != Xdumb \
&& { test "X$(AM_COLOR_TESTS)" = Xalways || test -t 1 2>/dev/null; } \
&& { \
+ am__color_tests=yes; \
red='[0;31m'; \
grn='[0;32m'; \
lgn='[1;32m'; \
blu='[1;34m'; \
+ mgn='[0;35m'; \
+ brg='[1m'; \
std='[m'; \
}
am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
@@ -118,100 +123,76 @@ am__base_list = \
# Restructured Text title and section.
am__rst_title = sed 's/.*/ & /;h;s/./=/g;p;x;p;g;p;s/.*//'
am__rst_section = sed 'p;s/./=/g;p;g'
-# Put stdin (possibly several lines separated by ". ") in a box.
-# Prefix each line by 'col' and terminate each with 'std', for coloring.
-# Multi line coloring is problematic with "less -R", so we really need
-# to color each line individually.
-am__text_box = $(AWK) '{ \
- n = split($$0, lines, "\\. "); max = 0; \
- for (i = 1; i <= n; ++i) \
- if (max < length(lines[i])) \
- max = length(lines[i]); \
- for (i = 0; i < max; ++i) \
- line = line "="; \
- print col line std; \
- for (i = 1; i <= n; ++i) \
- if (lines[i]) \
- print col lines[i] std; \
- print col line std; \
-}'
# Solaris 10 'make', and several other traditional 'make' implementations,
# pass "-e" to $(SHELL), and POSIX 2008 even requires this. Work around it
# by disabling -e (using the XSI extension "set +e") if it's set.
am__sh_e_setup = case $$- in *e*) set +e;; esac
+# Default flags passed to test drivers.
+am__common_driver_flags = \
+ --color-tests "$$am__color_tests" \
+ --enable-hard-errors "$$am__enable_hard_errors" \
+ --expect-failure "$$am__expect_failure"
# To be inserted before the command running the test. Creates the
# directory for the log if needed. Stores in $dir the directory
# containing $f, in $tst the test, in $log the log. Executes the
# developer- defined test setup AM_TESTS_ENVIRONMENT (if any), and
-# passes TESTS_ENVIRONMENT. Saves and restores TERM around uses of
-# TESTS_ENVIRONMENT and AM_TESTS_ENVIRONMENT, in case any of them
-# unsets it.
+# passes TESTS_ENVIRONMENT. Set up options for the wrapper that
+# will run the test scripts (or their associated LOG_COMPILER, if
+# thy have one).
am__check_pre = \
$(am__sh_e_setup); \
$(am__vpath_adj_setup) $(am__vpath_adj) \
+$(am__tty_colors); \
srcdir=$(srcdir); export srcdir; \
-rm -f address@hidden; \
-am__trap='rm -f '\''$(abs_builddir)/address@hidden'\''; (exit $$st); exit
$$st'; \
-trap "st=129; $$am__trap" 1; trap "st=130; $$am__trap" 2; \
-trap "st=141; $$am__trap" 13; trap "st=143; $$am__trap" 15; \
am__odir=`echo "./$@" | sed 's|/[^/]*$$||'`; \
test "x$$am__odir" = x. || $(MKDIR_P) "$$am__odir" || exit $$?; \
if test -f "./$$f"; then dir=./; \
elif test -f "$$f"; then dir=; \
else dir="$(srcdir)/"; fi; \
-tst=$$dir$$f; log='$@'; __SAVED_TERM=$$TERM; \
-$(AM_TESTS_ENVIRONMENT) \
-$(TESTS_ENVIRONMENT)
-# To be appended to the command running the test. Handle the stdout
-# and stderr redirection, and catch the exit status.
-am__check_post = \
->address@hidden 2>&1; \
-estatus=$$?; \
-if test -n '$(DISABLE_HARD_ERRORS)' \
- && test $$estatus -eq 99; then \
- estatus=1; \
-fi; \
-TERM=$$__SAVED_TERM; export TERM; \
-$(am__tty_colors); \
-xfailed=PASS; \
+tst=$$dir$$f; log='$@'; \
+if test -n '$(DISABLE_HARD_ERRORS)'; then \
+ am__enable_hard_errors=no; \
+else \
+ am__enable_hard_errors=yes; \
+fi; \
case " $(XFAIL_TESTS) " in \
*[\ \ ]$$f[\ \ ]* | *[\ \ ]$$dir$$f[\ \ ]*) \
- xfailed=XFAIL;; \
-esac; \
-case $$estatus.$$xfailed in \
- 0.XFAIL) col=$$red; res=XPASS;; \
- 0.*) col=$$grn; res=PASS ;; \
- 77.*) col=$$blu; res=SKIP ;; \
- 99.*) col=$$red; res=FAIL ;; \
- *.XFAIL) col=$$lgn; res=XFAIL;; \
- *.*) col=$$red; res=FAIL ;; \
-esac; \
-echo "$${col}$$res$${std}: $$f"; \
-echo "$$res: $$f (exit: $$estatus)" | \
- $(am__rst_section) >$@; \
-cat address@hidden >>$@; \
-rm -f address@hidden
+ am__expect_failure=yes;; \
+ *) \
+ am__expect_failure=no;; \
+esac; \
+$(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT)
+# A shell command to get the names of the tests scripts with any registered
+# extension removed (i.e., equivalently, the names of the test logs, with
+# the `.log' extension removed). The result is saved in the shell variable
+# `$bases'. This honors runtime overriding of TESTS and TEST_LOGS. Sadly,
+# we cannot use something simpler, involving e.g., "$(TEST_LOGS:.log=)",
+# since that might cause problem with VPATH rewrites for suffix-less tests.
+# See also 'test-harness-vpath-rewrite.test' and 'test-trs-basic.test'.
+am__set_TESTS_bases = \
+ bases='$(TEST_LOGS)'; \
+ bases=`for i in $$bases; do echo $$i; done | sed 's/\.log$$//'`; \
+ bases=`echo $$bases`
RECHECK_LOGS = $(TEST_LOGS)
AM_RECURSIVE_TARGETS = check check-html recheck recheck-html
TEST_SUITE_LOG = test-suite.log
TEST_SUITE_HTML = $(TEST_SUITE_LOG:.log=.html)
am__test_logs1 = $(TESTS:=.log)
am__test_logs2 = $(am__test_logs1:.test.log=.log)
+TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/lib/test-driver
TEST_LOG_COMPILE = $(TEST_LOG_COMPILER) $(AM_TEST_LOG_FLAGS) \
$(TEST_LOG_FLAGS)
-am__test_logs3 = $(am__test_logs2:.ptest.log=.log)
+am__test_logs3 = $(am__test_logs2:.tap.log=.log)
+TAP_LOG_COMPILE = $(TAP_LOG_COMPILER) $(AM_TAP_LOG_FLAGS) \
+ $(TAP_LOG_FLAGS)
+am__test_logs4 = $(am__test_logs3:.ptest.log=.log)
+PTEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/lib/test-driver
PTEST_LOG_COMPILE = $(PTEST_LOG_COMPILER) $(AM_PTEST_LOG_FLAGS) \
$(PTEST_LOG_FLAGS)
-am__test_logs4 = $(am__test_logs3:.instspc.log=.log)
-INSTSPC_LOG_COMPILE = $(INSTSPC_LOG_COMPILER) $(AM_INSTSPC_LOG_FLAGS) \
- $(INSTSPC_LOG_FLAGS)
-am__test_logs5 = $(am__test_logs4:.depmod.log=.log)
-DEPMOD_LOG_COMPILE = $(DEPMOD_LOG_COMPILER) $(AM_DEPMOD_LOG_FLAGS) \
- $(DEPMOD_LOG_FLAGS)
-TEST_LOGS = $(am__test_logs5:.shtst.log=.log)
+TEST_LOGS = $(am__test_logs4:.shtst.log=.log)
+SHTST_LOG_DRIVER = $(SHELL) $(top_srcdir)/lib/test-driver
SHTST_LOG_COMPILE = $(SHTST_LOG_COMPILER) $(AM_SHTST_LOG_FLAGS) \
$(SHTST_LOG_FLAGS)
-TEST_LOGS_TMP = $(TEST_LOGS:.log=.log-t)
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
ACLOCAL = @ACLOCAL@
AMTAR = @AMTAR@
@@ -311,94 +292,48 @@ target_alias = @target_alias@
top_build_prefix = @top_build_prefix@
top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
+
+# Run the tests with the shell detected at configure time.
+LOG_COMPILER = $(SHELL)
+TEST_EXTENSIONS = .test .tap .ptest .shtst
+TEST_LOG_COMPILER = $(LOG_COMPILER)
+TAP_LOG_COMPILER = $(LOG_COMPILER)
+TAP_LOG_DRIVER = $(PERL) $(top_srcdir)/lib/tap-driver
+TAP_LOG_DRIVER_FLAGS = --merge --comments --diagnostic-string `printf \
+ '\043%%\043\n'`
MAINTAINERCLEANFILES =
EXTRA_DIST = ChangeLog-old gen-parallel-tests parallel-tests.sh \
- instspc-tests.sh depmod-tests.sh gen-config-shell-tests \
- config-shell-tests.sh $(handwritten_tests)
-TEST_EXTENSIONS = .test .ptest .instspc .depmod .shtst
-# Run the tests with the shell detected at configure time.
-TEST_LOG_COMPILER = $(SHELL)
-XFAIL_TESTS = all.test auxdir2.test cond17.test gcj6.test \
- override-conditional-2.test pr8365-remake-timing.test \
- yacc-dist-nobuild-subdir.test txinfo5.test \
- $(instspc_xfail_tests)
+ gen-config-shell-tests config-shell-tests.sh \
+ $(handwritten_tests) tap-functions.sh plain-functions.sh \
+ trivial-test-driver testsuite-summary-checks.sh \
+ extract-testsuite-summary tap-setup.sh tap-summary-aux.sh
+XFAIL_TESTS = \
+all.test \
+auxdir2.test \
+cond17.test \
+gcj6.test \
+override-conditional-2.test \
+pr8365-remake-timing.test \
+yacc-dist-nobuild-subdir.test \
+tap-message-0.test \
+txinfo5.test
+
parallel_tests = backcompat5-p.ptest check-exported-srcdir-p.ptest \
- check-subst-prog-p.ptest check-subst-p.ptest \
- check-tests-in-builddir-p.ptest \
- check-tests_environment-p.ptest check-p.ptest check10-p.ptest \
- check11-p.ptest check12-p.ptest check2-p.ptest check3-p.ptest \
- check4-p.ptest check5-p.ptest check6-p.ptest check7-p.ptest \
- check8-p.ptest color-p.ptest color2-p.ptest comment9-p.ptest \
- dejagnu-p.ptest exeext4-p.ptest maken3-p.ptest maken4-p.ptest \
- posixsubst-tests-p.ptest
-PTEST_LOG_COMPILER = $(SHELL) $(srcdir)/parallel-tests.sh
-instspc_tests = build-squote.instspc install-squote.instspc \
- build-dquote.instspc install-dquote.instspc \
- build-bquote.instspc install-bquote.instspc \
- build-sharp.instspc install-sharp.instspc build-dollar.instspc \
- install-dollar.instspc build-bang.instspc install-bang.instspc \
- build-bslash.instspc install-bslash.instspc \
- build-ampersand.instspc install-ampersand.instspc \
- build-percent.instspc install-percent.instspc \
- build-leftpar.instspc install-leftpar.instspc \
- build-rightpar.instspc install-rightpar.instspc \
- build-pipe.instspc install-pipe.instspc build-caret.instspc \
- install-caret.instspc build-tilde.instspc \
- install-tilde.instspc build-qmark.instspc \
- install-qmark.instspc build-star.instspc install-star.instspc \
- build-plus.instspc install-plus.instspc build-minus.instspc \
- install-minus.instspc build-comma.instspc \
- install-comma.instspc build-colon.instspc \
- install-colon.instspc build-semicol.instspc \
- install-semicol.instspc build-equal.instspc \
- install-equal.instspc build-less.instspc install-less.instspc \
- build-more.instspc install-more.instspc build-at.instspc \
- install-at.instspc build-lqbrack.instspc \
- install-lqbrack.instspc build-rqbrack.instspc \
- install-rqbrack.instspc build-lcbrack.instspc \
- install-lcbrack.instspc build-rcbrack.instspc \
- install-rcbrack.instspc build-space.instspc \
- install-space.instspc build-tab.instspc install-tab.instspc \
- build-linefeed.instspc install-linefeed.instspc \
- build-backspace.instspc install-backspace.instspc \
- build-formfeed.instspc install-formfeed.instspc \
- build-carriageret.instspc install-carriageret.instspc \
- build-quadrigraph0.instspc install-quadrigraph0.instspc \
- build-quadrigraph1.instspc install-quadrigraph1.instspc \
- build-quadrigraph2.instspc install-quadrigraph2.instspc \
- build-quadrigraph3.instspc install-quadrigraph3.instspc \
- build-quadrigraph4.instspc install-quadrigraph4.instspc \
- build-a_b.instspc install-a_b.instspc build-a__b.instspc \
- install-a__b.instspc build-a_lf_b.instspc \
- install-a_lf_b.instspc build-dotdotdot.instspc \
- install-dotdotdot.instspc build-dosdrive.instspc \
- install-dosdrive.instspc build-miscglob1.instspc \
- install-miscglob1.instspc build-miscglob2.instspc \
- install-miscglob2.instspc
-instspc_xfail_tests = build-squote.instspc build-dquote.instspc \
- build-bquote.instspc build-sharp.instspc build-dollar.instspc \
- build-bslash.instspc build-ampersand.instspc \
- build-linefeed.instspc build-quadrigraph0.instspc \
- build-a_lf_b.instspc install-squote.instspc \
- install-dquote.instspc install-bquote.instspc \
- install-sharp.instspc install-dollar.instspc \
- install-linefeed.instspc install-a_lf_b.instspc
-INSTSPC_LOG_COMPILER = $(SHELL) $(srcdir)/instspc-tests.sh
-depmod_tests = depcomp-gcc3.depmod depcomp-gcc.depmod \
- depcomp-hp.depmod depcomp-sgi.depmod depcomp-aix.depmod \
- depcomp-icc.depmod depcomp-hp2.depmod depcomp-tru64.depmod \
- depcomp-msvc7.depmod depcomp-msvc7msys.depmod \
- depcomp-nosideeffect.depmod depcomp-dashmstdout.depmod \
- depcomp-dashXmstdout.depmod depcomp-makedepend.depmod \
- depcomp-cpp.depmod depcomp-msvisualcpp.depmod \
- depcomp-msvcmsys.depmod
-DEPMOD_LOG_COMPILER = $(SHELL) $(srcdir)/depmod-tests.sh
+ check-fd-redirect-p.ptest check-subst-prog-p.ptest \
+ check-subst-p.ptest check-tests-in-builddir-p.ptest \
+ check-p.ptest check11-p.ptest check12-p.ptest check2-p.ptest \
+ check3-p.ptest check4-p.ptest check5-p.ptest check6-p.ptest \
+ check7-p.ptest check8-p.ptest color-p.ptest color2-p.ptest \
+ comment9-p.ptest dejagnu-p.ptest exeext4-p.ptest \
+ maken3-p.ptest maken4-p.ptest posixsubst-tests-p.ptest \
+ tests-environment-p.ptest
+PTEST_LOG_COMPILER = $(LOG_COMPILER) $(srcdir)/parallel-tests.sh
config_shell_tests = ar-lib-w.shtst compile-w.shtst compile2-w.shtst \
compile3-w.shtst compile4-w.shtst compile5-w.shtst \
compile6-w.shtst instsh2-w.shtst instsh3-w.shtst \
mdate5-w.shtst mdate6-w.shtst missing-w.shtst missing2-w.shtst \
missing3-w.shtst missing5-w.shtst mkinst3-w.shtst
-SHTST_LOG_COMPILER = $(SHELL) $(srcdir)/config-shell-tests.sh
+SHTST_LOG_COMPILER = $(LOG_COMPILER) $(srcdir)/config-shell-tests.sh
# Some testsuite-influential variables should be overridable from the
# test scripts, but not from the environment.
@@ -406,33 +341,25 @@ SHTST_LOG_COMPILER = $(SHELL)
$(srcdir)/config-shell-tests.sh
# re-execute themselves with the shell detected at configure time, since
# we are already running them under it explicitly in our setup (see e.g.
# the definition of TEST_LOG_COMPILER above).
+AM_TESTS_ENVIRONMENT = test x"$$me" = x || unset me; test \
+ x"$$required" = x || unset required; test x"$$use_tap" = x || \
+ unset use_tap; test x"$$parallel_tests" = x || unset \
+ parallel_tests; test x"$$test_prefer_config_shell" || unset \
+ test_prefer_config_shell; test x"$$original_AUTOMAKE" = x || \
+ unset original_AUTOMAKE; test x"$$original_ACLOCAL" = x || \
+ unset original_ACLOCAL; AM_TESTS_REEXEC=no; export \
+ AM_TESTS_REEXEC; stderr_fileno_=9; export stderr_fileno_;
# We want warning messages and explanations for skipped tests to go to
# the console if possible, so set up `stderr_fileno_' properly.
-# The `9>&2' redirection *must* be placed at the end, and without a
-# following semicolon, because some shells (e.g., various Korn Shells
-# or HP-HX /bin/sh) close any file descriptor > 2 upon `exec' sycall.
-# For more references, with lots of discussion, see:
-# - http://lists.gnu.org/archive/html/bug-autoconf/2011-06/msg00002.html
-# - http://thread.gmane.org/gmane.comp.gnu.coreutils.bugs/22488
-# - http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8846
-AM_TESTS_ENVIRONMENT = \
- test x"$$me" = x || unset me; \
- test x"$$required" = x || unset required; \
- test x"$$parallel_tests" = x || unset parallel_tests; \
- test x"$$test_prefer_config_shell" || unset test_prefer_config_shell; \
- test x"$$original_AUTOMAKE" = x || unset original_AUTOMAKE; \
- test x"$$original_ACLOCAL" = x || unset original_ACLOCAL; \
- AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC; \
- stderr_fileno_=9; export stderr_fileno_; 9>&2
-
+AM_TESTS_FD_REDIRECT = 9>&2
TESTS = \
$(handwritten_tests) \
$(config_shell_tests) \
- $(parallel_tests) \
- $(depmod_tests) \
- $(instspc_tests)
+ $(parallel_tests)
handwritten_tests = \
+depmod.tap \
+instspc.tap \
aclocal.test \
aclocal3.test \
aclocal4.test \
@@ -455,16 +382,11 @@ acloca20.test \
acloca21.test \
acloca22.test \
acloca23.test \
-acoutnoq.test \
-acoutpt.test \
-acoutpt2.test \
-acoutqnl.test \
-acoutbs.test \
-acoutbs2.test \
+ac-output-old.tap \
acsilent.test \
acsubst.test \
acsubst2.test \
-add-missing.test \
+add-missing.tap \
all.test \
all2.test \
alloca.test \
@@ -512,7 +434,7 @@ autohdr2.test \
autohdr3.test \
autohdr4.test \
autohdrdry.test \
-automake.test \
+automake-cmdline.tap \
auxdir.test \
auxdir2.test \
auxdir3.test \
@@ -564,9 +486,9 @@ check12.test \
check-subst.test \
check-subst-prog.test \
check-exported-srcdir.test \
+check-fd-redirect.test \
check-tests-in-builddir.test \
-check-tests_environment.test \
-tests-environment-backcompat.test \
+check-no-test-driver.test \
checkall.test \
clean.test \
clean2.test \
@@ -713,7 +635,6 @@ depcomp6.test \
depcomp7.test \
depcomp8a.test \
depcomp8b.test \
-depmod-data.test \
depdist.test \
depend.test \
depend2.test \
@@ -860,7 +781,6 @@ instman2.test \
instmany.test \
instmany-mans.test \
instmany-python.test \
-instspc-data.test \
interp.test \
interp2.test \
java.test \
@@ -1050,10 +970,12 @@ parallel-tests9.test \
parallel-tests10.test \
parallel-tests-suffix.test \
parallel-tests-suffix-prog.test \
-parallel-tests-ext-driver.test \
-parallel-tests-ext-driver-prog.test \
-parallel-tests-am_tests_environment.test \
-parallel-tests-unreadable-log.test \
+parallel-tests-log-compiler-1.test \
+parallel-tests-log-compiler-2.test \
+parallel-tests-dry-run.test \
+parallel-tests-fd-redirect.test \
+parallel-tests-extra-programs.test \
+parallel-tests-unreadable.test \
parallel-tests-subdir.test \
parallel-tests-interrupt.test \
parallel-tests-reset-term.test \
@@ -1061,8 +983,45 @@ parallel-tests-harderror.test \
parallel-tests-log-override-1.test \
parallel-tests-log-override-2.test \
parallel-tests-log-override-recheck.test \
-parallel-tests-cmdline-override.test \
parallel-tests-log-compiler-example.test \
+parallel-tests-cmdline-override.test \
+parallel-tests-empty-testlogs.test \
+parallel-tests-driver-install.test \
+parallel-tests-no-color-in-log.test \
+parallel-tests-no-spurious-summary.test \
+parallel-tests-exit-statuses.test \
+parallel-tests-console-output.test \
+parallel-tests-once.test \
+tests-environment.test \
+am-tests-environment.test \
+tests-environment-backcompat.test \
+testsuite-summary-color.test \
+testsuite-summary-count.test \
+testsuite-summary-count-many.test \
+testsuite-summary-reference-log.test \
+test-driver-custom-no-extra-driver.test \
+test-driver-custom.test \
+test-driver-custom-xfail-tests.test \
+test-driver-custom-multitest.test \
+test-driver-custom-multitest-recheck.test \
+test-driver-custom-multitest-recheck2.test \
+test-driver-custom-html.test \
+test-driver-custom-no-html.test \
+test-driver-create-log-dir.test \
+test-driver-strip-vpath.test \
+test-driver-trs-suffix-registered.test \
+test-driver-fail.test \
+test-harness-vpath-rewrite.test \
+test-log.test \
+test-metadata-global-log.test \
+test-metadata-global-result.test \
+test-metadata-recheck.test \
+test-metadata-results.test \
+test-missing.test \
+test-missing2.test \
+test-trs-basic.test \
+test-trs-recover.test \
+test-trs-recover2.test \
parse.test \
percent.test \
percent2.test \
@@ -1183,15 +1142,16 @@ repeated-options.test \
rulepat.test \
self-check-cleanup.test \
self-check-dir.test \
-self-check-env-sanitize.test \
-self-check-exit.test \
+self-check-env-sanitize.tap \
+self-check-exit.tap \
self-check-explicit-skips.test \
self-check-is_newest.test \
-self-check-me.test \
+self-check-me.tap \
self-check-reexec.test \
self-check-report.test \
self-check-sanity.test \
self-check-seq.test \
+self-check-tap.test \
self-check-unindent.test \
sanity.test \
scripts.test \
@@ -1284,22 +1244,25 @@ substre2.test \
substtarg.test \
suffix.test \
suffix2.test \
-suffix3.test \
+suffix3.tap \
suffix4.test \
suffix5.test \
suffix6.test \
suffix6b.test \
suffix6c.test \
suffix7.test \
-suffix8.test \
+suffix8.tap \
suffix9.test \
-suffix10.test \
-suffix11.test \
+suffix10.tap \
+suffix11.tap \
suffix12.test \
suffix13.test \
+suffix-chain.tap \
symlink.test \
symlink2.test \
syntax.test \
+$(tap_with_common_setup_tests) \
+$(tap_other_tests) \
tags.test \
tags2.test \
tagsub.test \
@@ -1410,7 +1373,75 @@ yflags-force-override.test \
yflags-force-conditional.test \
yflags-var-expand.test
-checked_test_extensions = .test
+
+# List of tests on TAP support that use the files pre-computed by
+# `tap-common-setup.test', and sources the `tap-setup.sh' helper
+# script.
+tap_with_common_setup_tests = \
+tap-autonumber.test \
+tap-bailout.test \
+tap-color.test \
+tap-deps.test \
+tap-diagnostic.test \
+tap-empty-diagnostic.test \
+tap-empty.test \
+tap-escape-directive.test \
+tap-exit.test \
+tap-signal.test \
+tap-fancy.test \
+tap-fancy2.test \
+tap-global-log.test \
+tap-global-result.test \
+tap-html.test \
+tap-log.test \
+tap-merge-stdout-stderr.test \
+tap-no-merge-stdout-stderr.test \
+tap-message-0.test \
+tap-no-disable-hard-error.test \
+tap-no-spurious-summary.test \
+tap-no-spurious.test \
+tap-not-ok-skip.test \
+tap-numeric-description.test \
+tap-out-of-order.test \
+tap-passthrough.test \
+tap-passthrough-exit.test \
+tap-plan.test \
+tap-plan-corner.test \
+tap-plan-errors.test \
+tap-plan-middle.test \
+tap-realtime.test \
+tap-recheck-logs.test \
+tap-skip-whole-whitespace.test \
+tap-skip-whole.test \
+tap-skip-whole-lastline.test \
+tap-skip-whole-badexit.test \
+tap-todo-skip-together.test \
+tap-todo-skip-whitespace.test \
+tap-todo-skip.test \
+tap-unplanned.test \
+tap-whitespace-normalization.test \
+tap-with-and-without-number.test \
+tap-xfail-tests.test
+
+
+# Their log files.
+tap_with_common_setup_logs = $(tap_with_common_setup_tests:.test=.log)
+
+# Other tests on TAP support.
+tap_other_tests = \
+tap-common-setup.test \
+tap-bad-prog.tap \
+tap-basic.test \
+tap-diagnostic-custom.test \
+tap-doc.test \
+tap-doc2.test \
+tap-more.test \
+tap-more2.test \
+tap-recheck.test \
+tap-summary.test \
+tap-summary-color.test
+
+checked_test_extensions = .test .tap
expected_list_of_tests = $(handwritten_tests)
am__tmk = tests-in-makefile-list.tmp
am__tfs = tests-on-filesystem-list.tmp
@@ -1418,8 +1449,8 @@ am__tdf = diff-in-tests-lists.tmp
all: all-am
.SUFFIXES:
-.SUFFIXES: .depmod .html .instspc .log .ptest .shtst .test
-$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(srcdir)/parallel-tests.am
$(srcdir)/instspc-tests.am $(srcdir)/depmod-tests.am
$(srcdir)/config-shell-tests.am $(top_srcdir)/CheckListOfTests.am
$(am__configure_deps)
+.SUFFIXES: .html .log .ptest .shtst .tap .test .trs
+$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(srcdir)/parallel-tests.am
$(srcdir)/config-shell-tests.am $(top_srcdir)/CheckListOfTests.am
$(am__configure_deps)
@for dep in $?; do \
case '$(am__configure_deps)' in \
*$$dep*) \
@@ -1464,92 +1495,144 @@ CTAGS:
cscope cscopelist:
+# Recover from deleted `.trs' file; this should ensure that
+# "rm -f foo.log; make foo.trs" re-run `foo.test', and re-create
+# both `foo.log' and `foo.trs'. Break the recipe in two subshells
+# to avoid problems with "make -n".
+.log.trs:
+ rm -f $< $@
+ $(MAKE) $(AM_MAKEFLAGS) $<
+
+# Helper recipe used by $(TEST_SUITE_LOG) below, to avoid problems with
+# "make -n". Break this recipe in multiple shell invocations too, to
+# really work as expected with "make -n".
+am--redo-logs:
+ @rm -f $$redo_logs
+ @rm -f $$redo_results
+ @$(MAKE) $(AM_MAKEFLAGS) $$redo_logs
+ @st=0; \
+ errmsg="fatal: making $(TEST_SUITE_LOG): failed to create"; \
+ for i in $$redo_bases; do \
+ test -f $$i.trs && test -r $$i.trs \
+ || { echo "$$errmsg $$i.trs"; st=1; }; \
+ test -f $$i.log && test -r $$i.log \
+ || { echo "$$errmsg $$i.log"; st=1; }; \
+ done; \
+ test $$st -eq 0
+
$(TEST_SUITE_LOG): $(TEST_LOGS)
- @$(am__sh_e_setup); \
- list='$(TEST_LOGS)'; \
- results=`for f in $$list; do \
- test -r $$f && read line < $$f && echo "$$line" \
- || echo FAIL; \
- done`; \
- all=`echo "$$results" | sed '/^$$/d' | wc -l | sed -e 's/^[
]*//'`; \
- fail=`echo "$$results" | grep -c '^FAIL'`; \
- pass=`echo "$$results" | grep -c '^PASS'`; \
- skip=`echo "$$results" | grep -c '^SKIP'`; \
- xfail=`echo "$$results" | grep -c '^XFAIL'`; \
- xpass=`echo "$$results" | grep -c '^XPASS'`; \
- failures=`expr $$fail + $$xpass`; \
- all=`expr $$all - $$skip`; \
- if test "$$all" -eq 1; then tests=test; All=; \
- else tests=tests; All="All "; fi; \
- case fail=$$fail:xpass=$$xpass:xfail=$$xfail in \
- fail=0:xpass=0:xfail=0) \
- msg="$$All$$all $$tests passed. "; \
- exit=true;; \
- fail=0:xpass=0:xfail=*) \
- msg="$$All$$all $$tests behaved as expected"; \
- if test "$$xfail" -eq 1; then xfailures=failure; \
- else xfailures=failures; fi; \
- msg="$$msg ($$xfail expected $$xfailures). "; \
- exit=true;; \
- fail=*:xpass=0:xfail=*) \
- msg="$$fail of $$all $$tests failed. "; \
- exit=false;; \
- fail=*:xpass=*:xfail=*) \
- msg="$$failures of $$all $$tests did not behave as expected"; \
- if test "$$xpass" -eq 1; then xpasses=pass; \
- else xpasses=passes; fi; \
- msg="$$msg ($$xpass unexpected $$xpasses). "; \
- exit=false;; \
- *) \
- echo >&2 "incorrect case"; exit 4;; \
- esac; \
- if test "$$skip" -ne 0; then \
- if test "$$skip" -eq 1; then \
- msg="$$msg($$skip test was not run). "; \
- else \
- msg="$$msg($$skip tests were not run). "; \
- fi; \
- fi; \
+ @$(am__set_TESTS_bases); \
+ am__f_ok () { test -f "$$1" && test -r "$$1"; }; \
+ redo_bases=`for i in $$bases; do \
+ am__f_ok $$i.trs && am__f_ok $$i.log || echo $$i; \
+ done`; \
+ if test -n "$$redo_bases"; then \
+ redo_logs=`for i in $$redo_bases; do echo $$i.log; done`; \
+ redo_results=`for i in $$redo_bases; do echo $$i.trs; done`; \
+ redo_bases="$$redo_bases" \
+ redo_logs="$$redo_logs" \
+ redo_results="$$redo_results" \
+ $(MAKE) $(AM_MAKEFLAGS) am--redo-logs || exit 1; \
+ else :; fi;
+ @$(am__sh_e_setup); $(am__tty_colors); $(am__set_TESTS_bases); \
+ ws='[ ]'; \
+ results=`for b in $$bases; do echo $$b.trs; done`; \
+ test -n "$$results" || results=/dev/null; \
+ all=` grep "^$$ws*:test-result:" $$results | wc -l`; \
+ pass=` grep "^$$ws*:test-result:$$ws*PASS" $$results | wc -l`; \
+ fail=` grep "^$$ws*:test-result:$$ws*FAIL" $$results | wc -l`; \
+ skip=` grep "^$$ws*:test-result:$$ws*SKIP" $$results | wc -l`; \
+ xfail=`grep "^$$ws*:test-result:$$ws*XFAIL" $$results | wc -l`; \
+ xpass=`grep "^$$ws*:test-result:$$ws*XPASS" $$results | wc -l`; \
+ error=`grep "^$$ws*:test-result:$$ws*ERROR" $$results | wc -l`; \
+ if test `expr $$fail + $$xpass + $$error` -eq 0; then \
+ success=true; \
+ else \
+ success=false; \
+ fi; \
+ br='==================='; br=$$br$$br$$br$$br; \
+ result_count () \
+ { \
+ if test x"$$1" = x"--maybe-color"; then \
+ maybe_colorize=yes; \
+ elif test x"$$1" = x"--no-color"; then \
+ maybe_colorize=no; \
+ else \
+ echo "$@: invalid 'result_count' usage" >&2; exit 4; \
+ fi; \
+ shift; \
+ desc=$$1 count=$$2; \
+ if test $$maybe_colorize = yes && test $$count -gt 0; then \
+ color_start=$$3 color_end=$$std; \
+ else \
+ color_start= color_end=; \
+ fi; \
+ echo "$${color_start}# $$desc $$count$${color_end}"; \
+ }; \
+ create_testsuite_report () \
+ { \
+ result_count $$1 "TOTAL:" $$all "$$brg"; \
+ result_count $$1 "PASS: " $$pass "$$grn"; \
+ result_count $$1 "SKIP: " $$skip "$$blu"; \
+ result_count $$1 "XFAIL:" $$xfail "$$lgn"; \
+ result_count $$1 "FAIL: " $$fail "$$red"; \
+ result_count $$1 "XPASS:" $$xpass "$$red"; \
+ result_count $$1 "ERROR:" $$error "$$mgn"; \
+ }; \
{ \
echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \
$(am__rst_title); \
- echo "$$msg"; \
+ create_testsuite_report --no-color; \
echo; \
echo ".. contents:: :depth: 2"; \
echo; \
- for f in $$list; do \
- test -r $$f && read line < $$f || line=; \
- case $$line in \
- PASS:*|XFAIL:*);; \
- *) echo; cat $$f;; \
- esac; \
- done; \
+ for i in $$bases; do \
+ if grep "^$$ws*:copy-in-global-log:$$ws*no$$ws*$$" $$i.trs \
+ >/dev/null; then continue; \
+ fi; \
+ glob_res=`sed -n -e "s/$$ws*$$//" \
+ -e "s/^$$ws*:global-test-result:$$ws*//p" \
+ $$i.trs`; \
+ test -n "$$glob_res" || glob_res=RUN; \
+ echo "$$glob_res: $$i" | $(am__rst_section); \
+ if test ! -r $$i.log; then \
+ echo "fatal: making $@: $$i.log is unreadable" >&2; \
+ exit 1; \
+ fi; \
+ cat $$i.log; echo; \
+ done; \
} >$(TEST_SUITE_LOG).tmp; \
mv $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \
- if test "$$failures" -ne 0; then \
- msg="$${msg}See $(subdir)/$(TEST_SUITE_LOG). "; \
- if test -n "$(PACKAGE_BUGREPORT)"; then \
- msg="$${msg}Please report to $(PACKAGE_BUGREPORT). "; \
- fi; \
- fi; \
- test x"$$VERBOSE" = x || $$exit || cat $(TEST_SUITE_LOG); \
- $(am__tty_colors); \
- if $$exit; then \
+ if $$success; then \
col="$$grn"; \
else \
col="$$red"; \
+ test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \
fi; \
- echo "$$msg" | $(am__text_box) "col=$$col" "std=$$std"; \
- $$exit
+ echo "$${col}$$br$${std}"; \
+ echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \
+ echo "$${col}$$br$${std}"; \
+ create_testsuite_report --maybe-color; \
+ echo "$$col$$br$$std"; \
+ if $$success; then :; else \
+ echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \
+ if test -n "$(PACKAGE_BUGREPORT)"; then \
+ echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \
+ fi; \
+ echo "$$col$$br$$std"; \
+ fi; \
+ $$success
# Run all the tests.
check-TESTS:
@list='$(RECHECK_LOGS)'; test -z "$$list" || rm -f $$list
+ @list='$(RECHECK_LOGS:.log=.trs)'; test -z "$$list" || rm -f $$list
@test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
- @list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test .log = $$f || echo $$f; \
- done | tr '\012\015' ' '`; \
+ @list='$(TEST_LOGS)'; \
+ list=`for i in $$list; do \
+ test .log = $$i || echo $$i; \
+ done | tr '\012\015' ' '`; \
+ list=`echo "$$list" | sed 's/ *$$//'`; \
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_LOG) TEST_LOGS="$$list"
.log.html:
@@ -1577,26 +1660,36 @@ check-html:
$(MAKE) $(AM_MAKEFLAGS) $(TEST_SUITE_HTML) || exit 4; \
exit $$rv
recheck recheck-html:
- @target=`echo $@ | sed 's,^re,,'`; \
- list='$(TEST_LOGS)'; \
- list=`for f in $$list; do \
- test -f $$f || continue; \
- if test -r $$f && read line < $$f; then \
- case $$line in FAIL*|XPASS*) echo $$f;; esac; \
- else echo $$f; fi; \
+ @ws='[ ]'; \
+ target=`echo $@ | sed 's,^re,,'`; \
+ $(am__set_TESTS_bases); \
+ list=`for i in $$bases; do \
+ test -f $$i.trs || test -f $$i.log || continue; \
+ grep "^$$ws*:recheck:$$ws*no$$ws*$$" $$i.trs \
+ >/dev/null 2>&1 || echo $$i.log; \
done | tr '\012\015' ' '`; \
list=`echo "$$list" | sed 's/ *$$//'`; \
$(MAKE) $(AM_MAKEFLAGS) $$target AM_MAKEFLAGS='$(AM_MAKEFLAGS)
TEST_LOGS="'"$$list"'"'
.test.log:
- @p='$<'; $(am__check_pre) $(TEST_LOG_COMPILE) "$$tst" $(am__check_post)
+ @p='$<'; $(am__check_pre) $(TEST_LOG_DRIVER) --test-name "$$f" \
+ --log-file '$*.log' --trs-file '$*.trs' \
+ $(am__common_driver_flags) $(AM_TEST_LOG_DRIVER_FLAGS)
$(TEST_LOG_DRIVER_FLAGS) -- $(TEST_LOG_COMPILE) "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
+.tap.log:
+ @p='$<'; $(am__check_pre) $(TAP_LOG_DRIVER) --test-name "$$f" \
+ --log-file '$*.log' --trs-file '$*.trs' \
+ $(am__common_driver_flags) $(AM_TAP_LOG_DRIVER_FLAGS)
$(TAP_LOG_DRIVER_FLAGS) -- $(TAP_LOG_COMPILE) "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
.ptest.log:
- @p='$<'; $(am__check_pre) $(PTEST_LOG_COMPILE) "$$tst" $(am__check_post)
-.instspc.log:
- @p='$<'; $(am__check_pre) $(INSTSPC_LOG_COMPILE) "$$tst"
$(am__check_post)
-.depmod.log:
- @p='$<'; $(am__check_pre) $(DEPMOD_LOG_COMPILE) "$$tst"
$(am__check_post)
+ @p='$<'; $(am__check_pre) $(PTEST_LOG_DRIVER) --test-name "$$f" \
+ --log-file '$*.log' --trs-file '$*.trs' \
+ $(am__common_driver_flags) $(AM_PTEST_LOG_DRIVER_FLAGS)
$(PTEST_LOG_DRIVER_FLAGS) -- $(PTEST_LOG_COMPILE) "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
.shtst.log:
- @p='$<'; $(am__check_pre) $(SHTST_LOG_COMPILE) "$$tst" $(am__check_post)
+ @p='$<'; $(am__check_pre) $(SHTST_LOG_DRIVER) --test-name "$$f" \
+ --log-file '$*.log' --trs-file '$*.trs' \
+ $(am__common_driver_flags) $(AM_SHTST_LOG_DRIVER_FLAGS)
$(SHTST_LOG_DRIVER_FLAGS) -- $(SHTST_LOG_COMPILE) "$$tst" \
+ $(AM_TESTS_FD_REDIRECT)
distdir: $(DISTFILES)
@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
@@ -1654,7 +1747,7 @@ install-strip:
fi
mostlyclean-generic:
-test -z "$(TEST_LOGS)" || rm -f $(TEST_LOGS)
- -test -z "$(TEST_LOGS_TMP)" || rm -f $(TEST_LOGS_TMP)
+ -test -z "$(TEST_LOGS:.log=.trs)" || rm -f $(TEST_LOGS:.log=.trs)
-test -z "$(TEST_SUITE_HTML)" || rm -f $(TEST_SUITE_HTML)
-test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG)
@@ -1737,25 +1830,25 @@ uninstall-am:
.MAKE: check-am check-html install-am install-strip recheck \
recheck-html
-.PHONY: all all-am check check-TESTS check-am check-html clean \
- clean-generic clean-local distclean distclean-generic distdir \
- dvi dvi-am html html-am info info-am install install-am \
- install-data install-data-am install-dvi install-dvi-am \
- install-exec install-exec-am install-html install-html-am \
- install-info install-info-am install-man install-pdf \
- install-pdf-am install-ps install-ps-am install-strip \
- installcheck installcheck-am installdirs maintainer-clean \
- maintainer-clean-generic mostlyclean mostlyclean-generic pdf \
- pdf-am ps ps-am recheck recheck-html uninstall uninstall-am
+.PHONY: all all-am am--redo-logs check check-TESTS check-am check-html \
+ clean clean-generic clean-local distclean distclean-generic \
+ distdir dvi dvi-am html html-am info info-am install \
+ install-am install-data install-data-am install-dvi \
+ install-dvi-am install-exec install-exec-am install-html \
+ install-html-am install-info install-info-am install-man \
+ install-pdf install-pdf-am install-ps install-ps-am \
+ install-strip installcheck installcheck-am installdirs \
+ maintainer-clean maintainer-clean-generic mostlyclean \
+ mostlyclean-generic pdf pdf-am ps ps-am recheck recheck-html \
+ uninstall uninstall-am
backcompat5-p.log: backcompat5.test
check-exported-srcdir-p.log: check-exported-srcdir.test
+check-fd-redirect-p.log: check-fd-redirect.test
check-subst-prog-p.log: check-subst-prog.test
check-subst-p.log: check-subst.test
check-tests-in-builddir-p.log: check-tests-in-builddir.test
-check-tests_environment-p.log: check-tests_environment.test
check-p.log: check.test
-check10-p.log: check10.test
check11-p.log: check11.test
check12-p.log: check12.test
check2-p.log: check2.test
@@ -1773,6 +1866,7 @@ exeext4-p.log: exeext4.test
maken3-p.log: maken3.test
maken4-p.log: maken4.test
posixsubst-tests-p.log: posixsubst-tests.test
+tests-environment-p.log: tests-environment.test
$(srcdir)/parallel-tests.am: gen-parallel-tests Makefile.am
$(AM_V_GEN)($(am__cd) $(srcdir) && $(SHELL) ./gen-parallel-tests) >$@
@@ -1782,32 +1876,6 @@ $(parallel_tests:.ptest=.log): parallel-tests.sh
# This dummy declaration is required to have make actually produce
# expected log files from the `.ptest.log' suffix rule.
$(parallel_tests):
-
-$(srcdir)/instspc-tests.am: instspc-tests.sh Makefile.am
- $(AM_V_GEN)($(am__cd) $(srcdir) \
- && $(SHELL) ./instspc-tests.sh --generate-makefile) >$@
-
-# All `*.instspc' tests work by calling the script `instspc-tests.sh'.
-# Also, they all use shared data generated by the `instspc-data.test'
-# helper test (for reasons of speed).
-instspc-data.log: instspc-tests.sh
-$(instspc_tests:.instspc=.log): instspc-tests.sh instspc-data.log
-# This dummy declaration is required to have make actually produce
-# expected log files from the `.instspc.log' suffix rule.
-$(instspc_tests):
-
-$(srcdir)/depmod-tests.am: depmod-tests.sh Makefile.am
- $(AM_V_GEN)($(am__cd) $(srcdir) \
- && $(SHELL) ./depmod-tests.sh --generate-makefile) >$@
-
-# All `*.depmod' tests work by calling the script `depmod-tests.sh'.
-# Also, they all use shared data generated by the `depmod-data.test'
-# helper test (for reasons of speed).
-depmod-data.log: depmod-tests.sh
-$(depmod_tests:.depmod=.log): depmod-tests.sh depmod-data.log
-# This dummy declaration is required to have make actually produce
-# expected log files from the `.depmod.log' suffix rule.
-$(depmod_tests):
ar-lib-w.log: ar-lib.test
compile-w.log: compile.test
compile2-w.log: compile2.test
@@ -1834,6 +1902,22 @@ $(config_shell_tests:.shtst=.log): config-shell-tests.sh
# expected log files from the `.shtst.log' suffix rule.
$(config_shell_tests):
+test-driver-custom-multitest.log: trivial-test-driver
+test-driver-custom-multitest-recheck.log: trivial-test-driver
+test-driver-custom-multitest-recheck2.log: trivial-test-driver
+test-driver-custom-html.log: trivial-test-driver
+
+testsuite-summary-color.log testsuite-summary-count.log: \
+ testsuite-summary-checks.sh extract-testsuite-summary
+
+testsuite-summary-count-many.log: trivial-test-driver
+testsuite-summary-count-many.log: extract-testsuite-summary
+
+# Their dependencies.
+$(tap_with_common_setup_logs): tap-common-setup.log tap-setup.sh
+
+tap-summary.log tap-summary-color.log: tap-summary-aux.sh
+
.PHONY: maintainer-check-list-of-tests
maintainer-check-list-of-tests:
@if diff -u /dev/null /dev/null >/dev/null 2>&1; then \
@@ -1879,6 +1963,9 @@ clean-maintcheck-testslist-tmp:
# Dependencies valid for each test case.
$(TEST_LOGS): defs defs-static aclocal-$(APIVERSION) automake-$(APIVERSION)
+# FIXME: this should be made more granular once we have a cleaner
+# subdivision of the tests.
+$(TEST_LOGS): plain-functions.sh tap-functions.sh
clean-local: clean-local-check
.PHONY: clean-local-check
diff --git a/tests/README b/tests/README
index b3ff506..54ec670 100644
--- a/tests/README
+++ b/tests/README
@@ -56,15 +56,15 @@ Interpretation
Other:
SKIP - skipped tests (third party tools not available)
+ ERROR - some unexpected error condition
Getting details from failures
-----------------------------
- Each test is a shell script. In a non-VPATH build you can run the
- tests directly, they will be verbose. By default, verbose output of
- a test foo.test is retained in the log file foo.log. A summary log
- is created in the file test-suite.log.
+ By default, verbose output of a test `foo.test' or `foo.tap' is
+ retained in the log file `foo.log'. A summary log is created in
+ the file `test-suite.log'.
You can limit the set of files using the TESTS variable, and enable
detailed test output at the end of the test run with the VERBOSE
@@ -72,6 +72,30 @@ Getting details from failures
env VERBOSE=x TESTS='first.test second.test ...' make -e check
+ You can also run the tests by hand, as explained in the next subsection.
+
+
+About the tests
+---------------
+
+ There are two kinds of tests in the Automake testsuite (both implemented
+ as shell scripts). The scripts with the `.test' suffix are "simple"
+ tests, their outcome completely determined by their exit status. Those
+ with the `.tap' suffix use the TAP protocol. If you want to run a test
+ by hand, you can do so directly if it is a simple test:
+
+ ./nogzip.test
+
+ (it will be verbose by default), while if it is a TAP test you can pass
+ it to your preferred TAP runner, as in e.g.:
+
+ prove --verbose --merge ./add-missing.tap
+
+ The tests can also be run directly in a VPATH build, as with:
+
+ /path/to/srcdir/tests/nogzip.test
+ prove --verbose --merge /path/to/srcdir/tests/add-missing.tap
+
Supported shells
----------------
@@ -82,11 +106,12 @@ Supported shells
`/path/to/another/sh', the user must use:
AM_TESTS_REEXEC=no /path/to/another/sh ./foo.test
+ AM_TESTS_REEXEC=no prove -v -e /path/to/another/sh ./bar.tap
to run a test directly, and:
- make check TEST_LOG_COMPILER=/path/to/sh (GNU make)
- TEST_LOG_COMPILER=/path/to/sh make -e check (non-GNU make)
+ make check LOG_COMPILER=/path/to/sh (GNU make)
+ LOG_COMPILER=/path/to/sh make -e check (non-GNU make)
to run the test(s) through the makefile test driver.
@@ -166,11 +191,14 @@ Do
should be very very few), set the shell variable `parallel_tests' to
"no" before including ./defs.
- ./defs sets a skeleton configure.in. If possible, append to this
- file. In some cases you'll have to overwrite it, but this should
- be the exception. Note that configure.in registers Makefile.in
- but do not output anything by default. If you need ./configure
- to create Makefile, append AC_OUTPUT to configure.in.
+ ./defs brings in some commonly required files, and sets a skeleton
+ configure.in. If possible, append to this file. In some cases
+ you'll have to overwrite it, but this should be the exception. Note
+ that configure.in registers Makefile.in but do not output anything by
+ default. If you need ./configure to create Makefile, append AC_OUTPUT
+ to configure.in. In case you don't want ./defs to pre-populate your
+ test directory (which is a rare occurrence), set `$am_create_destdir'
+ to "empty" before sourcing ./defs.
By default, the testcases are run with the `errexit' shell flag on,
to make it easier to catch failures you might not have thought of.
diff --git a/tests/ac-output-old.tap b/tests/ac-output-old.tap
new file mode 100755
index 0000000..5eeb672
--- /dev/null
+++ b/tests/ac-output-old.tap
@@ -0,0 +1,181 @@
+#!/bin/sh
+# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
+# Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test for various older bugs related to quoting, escaping and
+# line breaking in the use of AC_OUTPUT. Synthesised by a bunch
+# of older tests (referenced below).
+
+. ./defs || Exit 1
+
+plan_ 22
+
+rm -f configure.in depcomp # Not required.
+
+# -----------------------------------------------------------------------
+
+# Test for bug reported by François Pinard.
+# If \ is in AC_OUTPUT, automake barfs.
+# This was the old test `acoutbs.test'.
+
+# Also test for bug reported by David A. Swierczek.
+# Another bug with \ in AC_OUTPUT (this time with whitespace).
+# This was the old test `acoutbs2.test'.
+
+acoutbs_check ()
+{
+ dir=acoutbs$1
+ mkdir $dir
+ cd $dir
+ cat > configure.in
+ touch Makefile.am zot.in
+ command_ok_ "aclocal groks '\\' in AC_OUTPUT ($dir)" $ACLOCAL
+ command_ok_ "automake groks '\\' in AC_OUTPUT ($dir)" $AUTOMAKE
+ command_ok_ "autoconf groks '\\' in AC_OUTPUT ($dir)" $AUTOCONF
+ command_ok_ "can ./configure in $dir" ./configure
+ command_ok_ "zot created in $dir" test -f zot
+ ls -a > lst || bailout_ "cannot get file listing in $dir"
+ command_ok_ "'\\' not leaked in filenames in $dir" not grep '\\' lst
+ cd ..
+}
+
+acoutbs_check 1 << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_OUTPUT(Makefile \
+zot)
+END
+
+acoutbs_check 2 << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_ARG_PROGRAM
+AC_PROG_INSTALL
+AC_PROG_MAKE_SET
+AC_OUTPUT(\
+ Makefile \
+ zot
+)
+END
+
+# -----------------------------------------------------------------------
+
+# Test for bug reported by Jerome Santini.
+# If I put this line in my configure.in:
+# AC_OUTPUT(Makefile src/Makefile tests/Makefile, echo timestamp > stamp-h)dnl
+# automake is not happy:
+# [ ... ]
+# This was the old test `acoutnoq.test'.
+
+mkdir acoutnoq
+cd acoutnoq
+
+cat > configure.in << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_OUTPUT(Makefile, [true])
+END
+
+: > Makefile.am
+
+command_ok_ "aclocal and quoted AC_OUTPUT second argument" $ACLOCAL
+command_ok_ "automake and quoted AC_OUTPUT second argument" $AUTOMAKE
+
+cd ..
+
+# -----------------------------------------------------------------------
+
+# Test for bug when AC_OUTPUT has 2 args on the same line, eg:
+# AC_OUTPUT([Makefile automake tests/Makefile],[chmod +x automake])
+# This was the old test `acoutpt.test'.
+
+mkdir acoutpt
+cd acoutpt
+
+cat > configure.in << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_OUTPUT([Makefile], [true])
+END
+
+: > Makefile.am
+
+command_ok_ "aclocal and two AC_OUTPUT arguments on same line" $ACLOCAL
+command_ok_ "automake and two AC_OUTPUT arguments on same line" $AUTOMAKE
+
+cd ..
+
+# -----------------------------------------------------------------------
+
+# Test for bug reported by Eric Magnien.
+# This was the old test `acoutpt2.test'.
+
+mkdir acoutpt2
+cd acoutpt2
+
+# Name of the current "subtest".
+cur=acoutput2
+
+cat > configure.in <<END
+AC_INIT([$cur], [1.0])
+AM_INIT_AUTOMAKE
+AC_OUTPUT([subdir/Makefile subdir/foo Makefile foo], [true])
+END
+
+mkdir subdir
+
+: > Makefile.am
+: > subdir/Makefile.am
+: > foo.in
+: > subdir/foo.in
+
+command_ok_ "aclocal and AC_OUTPUT ($cur)" $ACLOCAL
+command_ok_ "automake and AC_OUTPUT ($cur)" $AUTOMAKE
+command_ok_ "foo.in mentioned two times in Makefile.in ($cur)" \
+ test `$FGREP foo.in Makefile.in | wc -l` -eq 2
+
+# This ought to work as well.
+command_ok_ "'automake -a -f' and AC_OUTPUT ($cur)" \
+ $AUTOMAKE --add-missing --force-missing
+
+cd ..
+
+# -----------------------------------------------------------------------
+
+# Test for bug reported by François Pinard.
+# This was the old test `acoutqnl.test'.
+
+mkdir acoutqnl
+cd acoutqnl
+
+cat > configure.in << 'END'
+AC_INIT
+AM_INIT_AUTOMAKE(nonesuch, nonesuch)
+AC_OUTPUT([Makefile],
+[echo zardoz has spoken])
+END
+
+: > Makefile.am
+
+command_ok_ "aclocal and two AC_OUTPUT arguments on two lines" $ACLOCAL
+command_ok_ "automake and two AC_OUTPUT arguments on two lines" $AUTOMAKE
+
+cd ..
+
+# -----------------------------------------------------------------------
+
+# And we're done.
+Exit 0
diff --git a/tests/acoutbs.test b/tests/acoutbs.test
deleted file mode 100755
index 232267d..0000000
--- a/tests/acoutbs.test
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/sh
-# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
-# Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by François Pinard.
-# If \ is in AC_OUTPUT, automake barfs.
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_OUTPUT(Makefile \
-zot)
-END
-
-: > Makefile.am
-: > zot.in
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE
-
-./configure
-
-test -f zot || Exit 1
-ls -a | grep '\\' && Exit 1
-
-:
diff --git a/tests/acoutbs2.test b/tests/acoutbs2.test
deleted file mode 100755
index cd1cd9d..0000000
--- a/tests/acoutbs2.test
+++ /dev/null
@@ -1,47 +0,0 @@
-#! /bin/sh
-# Copyright (C) 1996, 2000, 2002, 2010, 2011 Free Software Foundation,
-# Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by David A. Swierczek.
-# Another bug with \ in AC_OUTPUT (this time with whitespace).
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_ARG_PROGRAM
-AC_PROG_INSTALL
-AC_PROG_MAKE_SET
-AC_OUTPUT(\
- Makefile \
- zot
-)
-END
-
-: > Makefile.am
-: > zot.in
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE
-
-./configure
-
-test -f zot || Exit 1
-ls -a | grep '\\' && Exit 1
-
-:
diff --git a/tests/acoutnoq.test b/tests/acoutnoq.test
deleted file mode 100755
index d49f878..0000000
--- a/tests/acoutnoq.test
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/sh
-# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
-# Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by Jerome Santini.
-# If I put this line in my configure.in:
-# AC_OUTPUT(Makefile src/Makefile tests/Makefile, echo timestamp > stamp-h)dnl
-# automake is not happy:
-# [ ... ]
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_OUTPUT(Makefile, [true])
-END
-
-: > Makefile.am
-
-$ACLOCAL
-$AUTOMAKE
diff --git a/tests/acoutpt.test b/tests/acoutpt.test
deleted file mode 100755
index 12b6427..0000000
--- a/tests/acoutpt.test
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/sh
-# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
-# Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug when AC_OUTPUT has 2 args on the same line, eg:
-# AC_OUTPUT([Makefile automake tests/Makefile],[chmod +x automake])
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_OUTPUT([Makefile], [true])
-END
-
-: > Makefile.am
-
-$ACLOCAL
-$AUTOMAKE
diff --git a/tests/acoutpt2.test b/tests/acoutpt2.test
deleted file mode 100755
index d972989..0000000
--- a/tests/acoutpt2.test
+++ /dev/null
@@ -1,44 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2001, 2002, 2010, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by Eric Magnien.
-
-. ./defs || Exit 1
-
-cat > configure.in <<END
-AC_INIT([$me], [1.0])
-AM_INIT_AUTOMAKE
-AC_OUTPUT([subdir/Makefile subdir/foo Makefile foo], [true])
-END
-
-mkdir subdir
-
-: > Makefile.am
-: > subdir/Makefile.am
-: > foo.in
-: > subdir/foo.in
-
-$ACLOCAL
-$AUTOMAKE
-
-count=`$FGREP foo.in Makefile.in | wc -l`
-test $count -eq 2
-
-# This ought to work as well.
-$ACLOCAL
-$AUTOMAKE --add-missing --force-missing
-
-:
diff --git a/tests/acoutqnl.test b/tests/acoutqnl.test
deleted file mode 100755
index 5cfac28..0000000
--- a/tests/acoutqnl.test
+++ /dev/null
@@ -1,32 +0,0 @@
-#! /bin/sh
-# Copyright (C) 1996, 2000, 2001, 2002, 2010, 2011 Free Software
-# Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test for bug reported by François Pinard.
-
-. ./defs || Exit 1
-
-cat > configure.in << 'END'
-AC_INIT
-AM_INIT_AUTOMAKE(nonesuch, nonesuch)
-AC_OUTPUT([Makefile],
-[echo zardoz has spoken])
-END
-
-: > Makefile.am
-
-$ACLOCAL
-$AUTOMAKE
diff --git a/tests/add-missing.tap b/tests/add-missing.tap
new file mode 100755
index 0000000..4c6a624
--- /dev/null
+++ b/tests/add-missing.tap
@@ -0,0 +1,361 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test that automake complains when required auxiliary files are not
+# found, and that `automake --add-missing' installs the files (and only
+# the files) it's supposed to, and that these files are symlinked by
+# default, but copied if the `--copy' option is used.
+
+. ./defs || Exit 1
+
+build_aux=build-aux
+
+# Try to improve readability of displayed diffs.
+if diff -u /dev/null /dev/null; then
+ am_diff='diff -u'
+elif diff -c /dev/null /dev/null; then
+ am_diff='diff -c'
+else
+ am_diff=diff
+fi
+
+cat > configure.stub << END
+AC_INIT([$me], [1.0])
+AC_CONFIG_AUX_DIR([$build_aux])
+AM_INIT_AUTOMAKE
+AC_CONFIG_FILES([Makefile])
+END
+
+# Pre-compute aclocal.m4, in order to save several aclocal invocations.
+cat >> configure.in <<'END'
+AC_PROG_CC
+AC_CANONICAL_BUILD
+AC_CANONICAL_HOST
+AC_CANONICAL_TARGET
+AC_CANONICAL_SYSTEM
+AM_PATH_LISPDIR
+AM_PATH_PYTHON
+END
+$ACLOCAL || framework_failure_ "cannot pre-compute aclocal.m4"
+
+rm -rf install-sh missing depcomp configure.in autom4te*.cache
+mv aclocal.m4 aclocal.stub
+
+cat configure.stub
+cat aclocal.stub
+
+# This is hacky and ugly and complex, but allow us to organize our tests
+# below in a more "declarative fashion". All in all, a good trade-off.
+check_ ()
+{
+ set +x # Temporary disable shell traces to remove noise from log files.
+ override=no
+ run_aclocal=no
+ fetch_file=
+ while test $# -gt 0; do
+ case $1 in
+ --override) override=yes;;
+ --run-aclocal) run_aclocal=yes;;
+ --fetch-file) fetch_file=$2; shift;;
+ *) framework_failure_ "check_: invalid argument '$1'";;
+ esac
+ shift
+ done
+ mkdir testdir-generic
+ cd testdir-generic
+ # Use `echo > ...', not `: > ...', here and below, as Solaris 10 /bin/sh
+ # might not execute the latter (the Autoconf manual gives more details).
+ echo > Makefile.am
+ if test $override = yes; then
+ echo > configure.in
+ else
+ cp ../configure.stub configure.in
+ fi
+ test -z "$fetch_file" || cp ../"$fetch_file" .
+ # Read description of "test scenario" from standard input.
+ what= line= name= files=
+ while read line; do
+ case $line in
+ '== Name ==') what=NAME;;
+ '== Makefile.am ==') what=Makefile.am;;
+ '== configure.in ==') what=configure.in;;
+ '== Files ==') what=LIST;;
+ '==*') framework_failure_ "invalid input line: $line";;
+ ''|'#%'*) : Empty line or ad-hoc comment, ignore. ;;
+ *)
+ if test $what = LIST; then
+ files="$files $line"
+ elif test $what = NAME; then
+ name=$line
+ else
+ printf '%s\n' "$line" >> "$what"
+ fi
+ ;;
+ esac
+ done
+ test -n "$name" || fatal_ "name of a test case not specified"
+ diag_ "Will run checks for $name."
+ if test $run_aclocal = yes; then
+ if $ACLOCAL; then
+ ok_ "[$name] aclocal.m4 rebuilt"
+ echo == aclocal.m4 ==
+ cat aclocal.m4
+ else
+ not_ok_ "[$name] aclocal failure, aclocal.m4 not rebuilt"
+ fi
+ else
+ cp ../aclocal.stub aclocal.m4
+ fi
+ echo == Makefile.am ==
+ cat Makefile.am
+ echo == configure.in ==
+ cat configure.in
+ echo Expected files: $files
+ mkdir "$build_aux"
+ cd ..
+ # End of "test scenario" setup.
+ set -x # Re-enable shell traces.
+ ls -l testdir-generic
+ # Test once with `--copy', once without.
+ for action in link copy; do
+ case $action in
+ link) opts='--add-missing';;
+ copy) opts='-a --copy';;
+ *) fatal_ "invalid value '$action' for \$action";;
+ esac
+ pfx="[$action $name]"
+ cp -R testdir-generic testdir-$action
+ cd testdir-$action
+ # If the required auxiliary files are missing, and automake is
+ # not told to install them, it should complain and error out,
+ # and also give a useful suggestion.
+ AUTOMAKE_fails -d "$pfx missing files, automake fails"
+ for f in $files; do
+ command_ok_ \
+ "$pfx warn about missing file $f" \
+ grep "required file ['\`]$build_aux/$f' not found" stderr
+ # Suggest the user to use `--add-missing'.
+ command_ok_ \
+ "$pfx suggest --add-missing for $f" \
+ grep ".*--add-missing.* install .*$f" stderr
+ done
+ # No files should be automatically installed by automake if it
+ # is not told to.
+ if ls "$build_aux" | grep .; then r='not ok'; else r=ok; fi
+ result_ "$r" "$pfx no extra files installed"
+ AUTOMAKE_run -d "$pfx automake run successfully" -- $opts
+ ls -l . $build_aux
+ # The expected files should get installed correctly (e.g., no
+ # broken symlinks).
+ for f in $files; do
+ command_ok_ \
+ "$pfx file $f installed" \
+ test -f $build_aux/$f
+ done
+ # Automake should inform about which files it's installing.
+ for f in $files; do
+ command_ok_ \
+ "$pfx report installation of $f" \
+ grep ": installing ['\`]$build_aux/$f'$" stderr
+ done
+ # Only the expected files should be installed. But automake always
+ # require `missing' and `install-sh', so account for them.
+ all_files="install-sh missing $files"
+ for f in $all_files; do echo $f; done | sort | uniq > files.exp
+ (cd $build_aux && ls) | sort > files.got
+ cat files.exp
+ cat files.got
+ command_ok_ \
+ "$pfx all and only expected files installed" \
+ $am_diff files.exp files.got
+ # The files should be copied by `--copy' and symlinked otherwise.
+ for f in $files; do
+ if test -h $build_aux/$f; then
+ is_symlink=yes
+ else
+ is_symlink=no
+ fi
+ case $action,$is_symlink in
+ link,yes) ok_ "$pfx $f has been symlinked" ;;
+ link,no) not_ok_ "$pfx $f has not been symlinked" ;;
+ copy,yes) not_ok_ "$pfx $f has been symlinked" ;;
+ copy,no) ok_ "$pfx $f has not been symlinked" ;;
+ *) fatal_ "invalid condition in case" ;;
+ esac
+ done
+ # Now that the required auxiliary files have been installed, automake
+ # should not complain anymore even if the `--add-missing' option is
+ # not used.
+ AUTOMAKE_run -d "$pfx automake finds all added files"
+ cd ..
+ done # for action in link copy
+ rm -rf testdir-*
+}
+
+# Automake should always and unconditionally require the `missing'
+# and `install-sh' scripts.
+check_ <<'END'
+== Name ==
+minimal
+== Files ==
+install-sh missing
+END
+
+check_ <<'END'
+== Name ==
+depcomp/C
+== Files ==
+depcomp
+== configure.in ==
+AC_PROG_CC
+== Makefile.am ==
+bin_PROGRAMS = foo
+END
+
+check_ <<'END'
+== Name ==
+depcomp/C++
+== Files ==
+depcomp
+== configure.in ==
+AC_PROG_CXX
+== Makefile.am ==
+bin_PROGRAMS = foo
+foo_SOURCES = foo.cc
+END
+
+check_ --run-aclocal <<'END'
+== Name ==
+compile
+== Files ==
+compile
+== configure.in ==
+# Using AM_PROG_CC_C_O in configure.in should be enough. No need to
+# use AC_PROG_CC too, nor to define xxx_PROGRAMS in Makefile.am.
+AM_PROG_CC_C_O
+END
+
+# For config.guess and config.sub.
+for mach in build host target system; do
+ MACH=`echo "$mach" | LC_ALL=C tr '[a-z]' '[A-Z]'`
+ check_ <<END
+== Name ==
+cfg-$mach
+== Files ==
+config.sub
+config.guess
+== configure.in ==
+AC_CANONICAL_$MACH
+END
+done
+
+check_ <<'END'
+== Name ==
+ylwrap/Lex
+== Files ==
+ylwrap
+== configure.in ==
+AC_PROG_CC
+AC_PROG_LEX
+== Makefile.am ==
+AUTOMAKE_OPTIONS = no-dependencies
+bin_PROGRAMS = foo
+foo_SOURCES = foo.l
+END
+
+check_ <<'END'
+== Name ==
+ylwrap/Yacc
+== Files ==
+ylwrap
+== configure.in ==
+AC_PROG_CC
+AC_PROG_YACC
+== Makefile.am ==
+AUTOMAKE_OPTIONS = no-dependencies
+bin_PROGRAMS = foo
+foo_SOURCES = foo.y
+END
+
+echo '@setfilename foo.info' > foo.texi
+check_ --fetch-file foo.texi <<'END'
+== Name ==
+Texinfo
+== Files ==
+texinfo.tex
+== Makefile.am ==
+info_TEXINFOS = foo.texi
+END
+
+echo '@include version.texi' >> foo.texi
+check_ --fetch-file foo.texi <<'END'
+== Name ==
+Texinfo/mdate-sh
+== Files ==
+mdate-sh
+texinfo.tex
+== Makefile.am ==
+info_TEXINFOS = foo.texi
+END
+
+rm -f foo.texi
+
+check_ <<'END'
+== Name ==
+elisp-comp
+== Files ==
+elisp-comp
+== configure.in ==
+AM_PATH_LISPDIR
+== Makefile.am ==
+lisp_LISP = foo.el
+END
+
+check_ <<'END'
+== Name ==
+py-compile
+== Files ==
+py-compile
+== configure.in ==
+AM_PATH_PYTHON
+== Makefile.am ==
+python_PYTHON = foo.py
+END
+
+# Try few unrelated auxiliary scripts together.
+check_ <<'END'
+== Name ==
+misc
+== Files ==
+py-compile
+depcomp
+ylwrap
+config.sub
+config.guess
+== configure.in ==
+AC_CANONICAL_BUILD
+AC_CANONICAL_HOST
+AC_PROG_CXX
+== Makefile.am ==
+PYTHON = python
+pythondir = $(prefix)/py
+YACC = bison -y
+bin_PROGRAMS = foo
+foo_SOURCES = bar.yxx baz.c++
+python_PYTHON = zardoz.py
+END
+
+:
diff --git a/tests/add-missing.test b/tests/add-missing.test
deleted file mode 100755
index 9fd858d..0000000
--- a/tests/add-missing.test
+++ /dev/null
@@ -1,315 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test that automake complains when required auxiliary files are not
-# found, and that `automake --add-missing' installs the files (and only
-# the files) it's supposed to, and that these files are symlinked by
-# default, but copied if the `--copy' option is used.
-
-. ./defs || Exit 1
-
-build_aux=build-aux
-
-# Try to improve readability of displayed diffs.
-if diff -u /dev/null /dev/null; then
- am_diff='diff -u'
-elif diff -c /dev/null /dev/null; then
- am_diff='diff -c'
-else
- am_diff=diff
-fi
-
-cat > configure.stub << END
-AC_INIT([$me], [1.0])
-AC_CONFIG_AUX_DIR([$build_aux])
-AM_INIT_AUTOMAKE
-AC_CONFIG_FILES([Makefile])
-END
-
-# Pre-compute aclocal.m4, in order to save several aclocal invocations.
-cat >> configure.in <<'END'
-AC_PROG_CC
-AC_CANONICAL_BUILD
-AC_CANONICAL_HOST
-AC_CANONICAL_TARGET
-AC_CANONICAL_SYSTEM
-AM_PATH_LISPDIR
-AM_PATH_PYTHON
-END
-$ACLOCAL || framework_failure_ "cannot pre-compute aclocal.m4"
-
-rm -rf install-sh missing depcomp configure.in autom4te*.cache
-mv aclocal.m4 aclocal.stub
-
-cat configure.stub # For debugging.
-cat aclocal.stub # Likewise.
-
-# This is hacky and ugly and complex, but allow us to organize our tests
-# below in a more "declarative fashion". All in all, a good trade-off.
-check_ ()
-{
- set +x # Temporary disable shell traces to remove noise from log files.
- override=no
- run_aclocal=no
- extra_file=
- while test $# -gt 0; do
- case $1 in
- --override) override=yes;;
- --run-aclocal) run_aclocal=yes;;
- --extra-file) extra_file=$2; shift;;
- *) framework_failure_ "check_: invalid argument '$1'";;
- esac
- shift
- done
- mkdir testdir-generic
- cd testdir-generic
- : > Makefile.am
- if test $override = yes; then
- : > configure.in
- else
- cp ../configure.stub configure.in
- fi
- test -z "$extra_file" || cp ../"$extra_file" .
- # Read description of "test scenario" from standard input.
- what=
- line=
- files='install-sh missing' # These are always installed by automake.
- while read line; do
- case $line in
- '== Makefile.am ==') what=Makefile.am;;
- '== configure.in ==') what=configure.in;;
- '== Files ==') what=LIST;;
- '==*') framework_failure_ "invalid input line: $line";;
- ''|'#%'*) : Empty line or ad-hoc comment, ignore. ;;
- *)
- if test $what = LIST; then
- files="$files $line"
- else
- printf '%s\n' "$line" >> "$what"
- fi
- ;;
- esac
- done
- if test $run_aclocal = yes; then
- $ACLOCAL
- echo == aclocal.m4 ==
- cat aclocal.m4
- else
- cp ../aclocal.stub aclocal.m4
- fi
- # For debugging.
- if test -s Makefile.am; then
- echo == Makefile.am ==
- cat Makefile.am
- fi
- echo == configure.in ==
- cat configure.in
- echo Expected files: $files
- mkdir "$build_aux"
- cd ..
- # End of "test scenario" setup.
- set -x # Re-enable shell traces.
- ls -l testdir-generic
- # Test once with `--copy', once without.
- for action in link copy; do
- case $action in
- link) opts='--add-missing' test_linked='test -h';;
- copy) opts='-a --copy' test_linked='test ! -h';;
- *) Exit 99;; # Can't happen.
- esac
- cp -R testdir-generic testdir-$action
- cd testdir-$action
- # If the required auxiliary files are missing, and automake is
- # not told to install them, it should complain and error out,
- # and also give a useful suggestion.
- AUTOMAKE_fails
- for f in $files; do
- grep "required file ['\`]$build_aux/$f' not found" stderr
- # Suggest the user to use `--add-missing'.
- grep ".*--add-missing.* install .*$f" stderr
- done
- # No files should be automatically installed by automake if it
- # is not told to.
- ls "$build_aux" | grep . && Exit 1
- $AUTOMAKE $opts 2>stderr || { cat stderr >&2; Exit 1; }
- cat stderr >&2
- ls -l . $build_aux
- # The expected files should get installed correctly (e.g., no
- # broken symlinks).
- for f in $files; do
- test -f $build_aux/$f
- done
- # Automake should inform about which files it's installing.
- for f in $files; do
- grep ": installing ['\`]$build_aux/$f'$" stderr
- done
- # Only the expected files should be installed.
- for f in $files; do echo $f; done | sort > files.exp
- (cd $build_aux && ls) | sort > files.got
- cat files.exp
- cat files.got
- $am_diff files.exp files.got
- # The files should be copied by `--copy' and symlinked otherwise.
- for f in $files; do
- $test_linked $build_aux/$f
- done
- # Now that the required auxiliary files have been installed, automake
- # should not complain anymore even if the `--add-missing' option is
- # not used.
- $AUTOMAKE
- cd ..
- done
- rm -rf testdir-*
-}
-
-: %%% install-sh and missing %%%
-check_ <<'END'
-#% 'install-sh' and 'missing' should always get installed.
-END
-
-: %%% depcomp with C %%%
-check_ <<'END'
-== Files ==
-depcomp
-== configure.in ==
-AC_PROG_CC
-== Makefile.am ==
-bin_PROGRAMS = foo
-END
-
-: %%% depcomp with C++ %%%
-check_ <<'END'
-== Files ==
-depcomp
-== configure.in ==
-AC_PROG_CXX
-== Makefile.am ==
-bin_PROGRAMS = foo
-foo_SOURCES = foo.cc
-END
-
-: %%% compile script %%%
-check_ --run-aclocal <<'END'
-== Files ==
-compile
-== configure.in ==
-# Using AM_PROG_CC_C_O in configure.in should be enough. No need to
-# use AC_PROG_CC too, nor to define xxx_PROGRAMS in Makefile.am.
-AM_PROG_CC_C_O
-END
-
-: %%% config.guess and config.sub %%%
-
-for macro in AC_CANONICAL_BUILD AC_CANONICAL_HOST \
- AC_CANONICAL_TARGET AC_CANONICAL_SYSTEM; do
- check_ <<END
-== Files ==
-config.sub
-config.guess
-== configure.in ==
-$macro
-END
-done
-
-: %%% ylwrap with Lex %%%
-check_ <<'END'
-== Files ==
-ylwrap
-== configure.in ==
-AC_PROG_CC
-AC_PROG_LEX
-== Makefile.am ==
-AUTOMAKE_OPTIONS = no-dependencies
-bin_PROGRAMS = foo
-foo_SOURCES = foo.l
-END
-
-: %%% ylwrap with Yacc %%%
-check_ <<'END'
-== Files ==
-ylwrap
-== configure.in ==
-AC_PROG_CC
-AC_PROG_YACC
-== Makefile.am ==
-AUTOMAKE_OPTIONS = no-dependencies
-bin_PROGRAMS = foo
-foo_SOURCES = foo.y
-END
-
-: %%% texinfo.tex %%%
-echo '@setfilename foo.info' > foo.texi
-check_ --extra-file foo.texi <<'END'
-== Files ==
-texinfo.tex
-== Makefile.am ==
-info_TEXINFOS = foo.texi
-END
-
-: %%% mdate-sh and texinfo.tex %%%
-echo '@include version.texi' >> foo.texi
-check_ --extra-file foo.texi <<'END'
-== Files ==
-mdate-sh
-texinfo.tex
-== Makefile.am ==
-info_TEXINFOS = foo.texi
-END
-
-rm -f foo.texi
-
-: %%% elisp-compile %%%
-check_ <<'END'
-== Files ==
-elisp-comp
-== configure.in ==
-AM_PATH_LISPDIR
-== Makefile.am ==
-lisp_LISP = foo.el
-END
-
-: %%% py-compile %%%
-check_ <<'END'
-== Files ==
-py-compile
-== configure.in ==
-AM_PATH_PYTHON
-== Makefile.am ==
-python_PYTHON = foo.py
-END
-
-: %%% few unrelated auxiliary scripts together %%%
-check_ <<'END'
-== Files ==
-py-compile
-depcomp
-ylwrap
-config.sub
-config.guess
-== configure.in ==
-AC_CANONICAL_BUILD
-AC_CANONICAL_HOST
-AC_PROG_CXX
-== Makefile.am ==
-PYTHON = python
-pythondir = $(prefix)/py
-YACC = bison -y
-bin_PROGRAMS = foo
-foo_SOURCES = bar.yxx baz.c++
-python_PYTHON = zardoz.py
-END
-
-:
diff --git a/tests/am-tests-environment.test b/tests/am-tests-environment.test
new file mode 100755
index 0000000..8ad7481
--- /dev/null
+++ b/tests/am-tests-environment.test
@@ -0,0 +1,91 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests: check AM_TESTS_ENVIRONMENT support, and its
+# interactions with TESTS_ENVIRONMENT.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_PROG_MKDIR_P
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .sh .test
+TESTS = foo.test bar.sh
+SH_LOG_COMPILER = sh
+AM_TESTS_ENVIRONMENT = \
+ $(MKDIR_P) quux.dir; \
+ if test -f $(srcdir)/test-env.sh; then \
+ . $(srcdir)/test-env.sh; \
+ fi; \
+ FOO=1; export FOO;
+EXTRA_DIST = $(TESTS) test-env.sh
+END
+
+cat > foo.test << 'END'
+#! /bin/sh
+ls -l && test -d quux.dir
+END
+chmod a+x foo.test
+
+cat > bar.sh << 'END'
+echo "FOO='$FOO'"
+echo "BAR='$BAR'"
+test x"$FOO" = x"$BAR"
+END
+
+debug_info ()
+{
+ cat test-suite.log
+ cat foo.log
+ cat bar.log
+}
+
+minicheck ()
+{
+ debug_info
+ test -d quux.dir
+}
+
+miniclean ()
+{
+ rmdir quux.dir
+ rm -f foo.log bar.log test-suite.log
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+TESTS_ENVIRONMENT='BAR=1' $MAKE check || { debug_info; Exit 1; }
+minicheck
+miniclean
+
+TESTS_ENVIRONMENT='BAR=2' $MAKE check && { debug_info; Exit 1; }
+minicheck
+miniclean
+
+echo 'BAR=1 && export BAR' > test-env.sh
+$MAKE check || { debug_info; Exit 1; }
+minicheck
+$MAKE distcheck
+
+:
diff --git a/tests/amopts-variable-expansion.test
b/tests/amopts-variable-expansion.test
index c11be87..57512cc 100755
--- a/tests/amopts-variable-expansion.test
+++ b/tests/amopts-variable-expansion.test
@@ -44,7 +44,7 @@ install:
END
$ACLOCAL
-AUTOMAKE_run 0
+AUTOMAKE_run
grep '^Makefile\.am:.*:=.*not portable' stderr
grep README stderr && Exit 1
$EGREP '(install|override)' stderr && Exit 1
diff --git a/tests/autodist.test b/tests/autodist.test
index f2b31ab..ed8dcc3 100755
--- a/tests/autodist.test
+++ b/tests/autodist.test
@@ -19,13 +19,9 @@
# Related to automake bug#7819.
# Keep this test in sync with sister test `autodist-subdir.test'.
+am_create_testdir=empty
. ./defs || Exit 1
-# Ensure we are run from the right directory.
-# (The last thing we want is to delete some random user files.)
-test -f ../defs-static && test -f ../defs || Exit 99
-rm -f *
-
cat > configure.in <<END
AC_INIT([$me], [1.0])
AC_CONFIG_AUX_DIR([.])
diff --git a/tests/automake-cmdline.tap b/tests/automake-cmdline.tap
new file mode 100755
index 0000000..9397900
--- /dev/null
+++ b/tests/automake-cmdline.tap
@@ -0,0 +1,63 @@
+#! /bin/sh
+# Copyright (C) 2004, 2006, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test Automake's command-line options.
+
+. ./defs || Exit 1
+
+plan_ 17
+
+# Usage: bad_cmdline DESCRIPTION REGEX-FOR-STDERR [ARGS-FOR-AUTOMAKE...]
+do_check ()
+{
+ test $# -ge 3 || fatal_ "do_check: invalid usage"
+ desc=$1; shift
+ regex=$1; shift
+ AUTOMAKE_fails -d "$desc (run)" -- "$@"
+ command_ok_ "$desc (stderr)" grep "$regex" stderr
+}
+
+do_check 'invalid long option' 'unrecognized option.*--voo' --voo
+
+# Older perl has a buggy Getopt::Long which makes this fail.
+if $PERL -e 'require 5.8.2;'; then
+ do_check "list of options terminated by '--'" \
+ 'input file.*--voo' -- --voo
+else
+ skip_row_ 2 -r "older perl with buggy Getopt::Long"
+fi
+
+do_check "empty argument" \
+ 'empty argument' ''
+
+do_check "missing argument for long option" \
+ 'option.*-W.*requires an argument' -W
+
+do_check "missing argument for short option" \
+ 'option.*--warnings.*requires an argument' --warnings
+
+do_check "'--help' as option argument" \
+ 'unknown warning.*--help' --warnings --help
+
+do_check "'--help' as option argument" \
+ 'unknown warning.*--help' --warnings --help
+
+do_check "ambiguous incomplete option" \
+ 'unrecognized option.*--ver' --ver
+
+command_ok_ "unambiguous incomplete long option" $AUTOMAKE --vers
+
+:
diff --git a/tests/automake.test b/tests/automake.test
deleted file mode 100755
index db8c2b3..0000000
--- a/tests/automake.test
+++ /dev/null
@@ -1,39 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2004, 2006, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test Automake's command-line options.
-. ./defs || Exit 1
-
-AUTOMAKE_fails --voo
-grep 'unrecognized option.*--voo' stderr
-# Older perl has a buggy Getopt::Long which makes this fail.
-if $PERL -e 'require 5.8.2;'; then
- AUTOMAKE_fails -- --voo
- grep 'input file.*--voo' stderr
-fi
-AUTOMAKE_fails ''
-grep 'empty argument' stderr
-AUTOMAKE_fails -W
-grep 'option.*-W.*requires an argument' stderr
-AUTOMAKE_fails --warnings
-grep 'option.*--warning.*requires an argument' stderr
-AUTOMAKE_fails --warnings --help
-grep 'unknown warning.*--help' stderr
-AUTOMAKE_fails --ver
-grep 'unrecognized option.*--ver' stderr
-$AUTOMAKE --vers
-
-:
diff --git a/tests/check-fd-redirect.test b/tests/check-fd-redirect.test
new file mode 100755
index 0000000..6da8704
--- /dev/null
+++ b/tests/check-fd-redirect.test
@@ -0,0 +1,110 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Simple Tests support: redirection of file descriptors with
+# AM_TESTS_FD_REDIRECT.
+# See also related test 'parallel-tests-fd-redirect.test'.
+
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = foo.test
+AM_TESTS_FD_REDIRECT = 3<three 4>four 5>>five 7<&0 8>&1 9>&2
+END
+
+echo '3333' > three
+chmod a-w three
+
+: > foo.test
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+./configure
+
+do_check ()
+{
+ cat foo.test # For debugging.
+ echo 'this line will be removed' > four
+ echo 'this line will not be removed' > five
+ st=0
+ echo 'ok ok ok' | $MAKE check >stdout 2>stderr || st=1
+ cat stdout
+ cat stderr >&2
+ cat four
+ test x"$parallel_tests" != x"yes" || cat foo.log
+ test $st -eq 0
+ grep '[ /]foo\.test: foofoofoo$' stdout
+ grep '[ /]foo\.test: barbarbar$' stderr
+ grep 'this line' four && Exit 1
+ grep '^3333$' four
+ grep '^this line will not be removed$' five
+ grep '^ok ok ok$' five
+ $EGREP '(foofoofoo|barbarbar|3333|ok ok ok|this line)' foo.log && Exit 1
+ :
+}
+
+# Try using both shell script and a perl script as the test, for
+# better coverage.
+
+cat > foo.test <<'END'
+#! /bin/sh
+set -e
+
+read FOO <&3
+test 3333 -eq "$FOO"
+echo "$FOO" >&4
+
+grep '^ok ok ok$' <&7 >&5
+
+echo " " $0: foofoofoo >&8
+echo " " $0: barbarbar >&9
+END
+
+do_check
+
+echo "#! $PERL -w" > foo.test
+cat >> foo.test <<'END'
+use warnings FATAL => 'all';
+use strict;
+
+open (FD3, "<&=3") or die "opening FD3: $!";
+open (FD4, ">&=4") or die "opening FD4: $!";
+open (FD5, ">&=5") or die "opening FD5: $!";
+open (FD7, "<&=7") or die "opening FD7: $!";
+open (FD8, ">&=8") or die "opening FD8: $!";
+open (FD9, ">&=9") or die "opening FD9: $!";
+
+chomp (my $FOO = <FD3>);
+die "$FOO != 3333" if not $FOO eq "3333";
+print FD4 "$FOO\n";
+
+chomp ($_ = <FD7>);
+die "$_ != 'ok ok ok'" if not $_ eq 'ok ok ok';
+print FD5 "$_\n";
+
+print FD8 " $0: foofoofoo\n";
+print FD9 " $0: barbarbar\n";
+END
+
+do_check
+
+:
diff --git a/tests/check-no-test-driver.test b/tests/check-no-test-driver.test
new file mode 100755
index 0000000..fc1e89b
--- /dev/null
+++ b/tests/check-no-test-driver.test
@@ -0,0 +1,34 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that auxiliary script 'test-driver' doesn't get needlessly
+# installed or referenced when the 'parallel-tests' option is not
+# used.
+
+parallel_tests=no
+. ./defs || Exit 1
+
+echo 'TESTS = foo.test' > Makefile.am
+
+$ACLOCAL
+
+for opts in '' '-a' '--add-missing --copy'; do
+ $AUTOMAKE $opts
+ $FGREP 'test-driver' Makefile.in && Exit 1
+ find . | $FGREP 'test-driver' && Exit 1
+done
+
+:
diff --git a/tests/check.test b/tests/check.test
index c9038ad..f18bb0a 100755
--- a/tests/check.test
+++ b/tests/check.test
@@ -22,6 +22,8 @@ cat > Makefile.am << 'END'
TESTS = frob.test
END
+test x"$parallel_tests" != x"yes" || : > test-driver
+
: > frob.test
$ACLOCAL
diff --git a/tests/check10.test b/tests/check10.test
index 0705188..37f9d08 100755
--- a/tests/check10.test
+++ b/tests/check10.test
@@ -16,6 +16,8 @@
# Check singular and plural in test summaries.
+# This test only makes sense for the older serial testsuite driver.
+parallel_tests=no
. ./defs || Exit 1
cat >> configure.in << 'END'
@@ -50,7 +52,7 @@ cp skip skip2
$ACLOCAL
$AUTOCONF
-$AUTOMAKE
+$AUTOMAKE -a
./configure
(
diff --git a/tests/check11.test b/tests/check11.test
index 5305c96..bd6be34 100755
--- a/tests/check11.test
+++ b/tests/check11.test
@@ -38,12 +38,23 @@ $AUTOCONF
$AUTOMAKE -a
./configure
+
env TESTS=skip $MAKE -e check >stdout || { cat stdout; Exit 1; }
cat stdout
-grep '1.*passed' stdout && Exit 1
+if test x"$parallel_tests" = x"yes"; then
+ count_test_results total=1 pass=0 fail=0 skip=1 xfail=0 xpass=0 error=0
+else
+ grep '1.*passed' stdout && Exit 1
+ : For shells with buggy 'set -e'.
+fi
env TESTS="skip skip2" $MAKE -e check >stdout || { cat stdout; Exit 1; }
cat stdout
-grep '2.*passed' stdout && Exit 1
+if test x"$parallel_tests" = x"yes"; then
+ count_test_results total=2 pass=0 fail=0 skip=2 xfail=0 xpass=0 error=0
+else
+ grep '2.*passed' stdout && Exit 1
+ : For shells with buggy 'set -e'.
+fi
:
diff --git a/tests/check12.test b/tests/check12.test
index 4be4b80..e36a94d 100755
--- a/tests/check12.test
+++ b/tests/check12.test
@@ -34,10 +34,12 @@ END
cat > a.test << 'END'
#!/bin/sh
+echo a.test: exit status: ${A_EXIT_STATUS-0}
exit ${A_EXIT_STATUS-0}
END
cat > b.test << 'END'
#!/bin/sh
+echo b.test: exit status: ${B_EXIT_STATUS-0}
exit ${B_EXIT_STATUS-0}
END
chmod +x a.test b.test
@@ -135,9 +137,9 @@ for vpath in : false; do
test -f spanner.sum
# This checks will be run only by the autogenerated `check12-p.test'.
if test x"$parallel_tests" = x"yes"; then
- cat test-suite.log
- grep '^PASS: a\.test (exit: 0)' a.log
- grep '^PASS: b\.test (exit: 0)' b.log
+ test -f test-suite.log
+ test -f a.log
+ test -f b.log
else
:
fi
@@ -170,9 +172,13 @@ for vpath in : false; do
# This checks will be run only by the autogenerated `check12-p.test'.
if test x"$parallel_tests" = x"yes"; then
cat test-suite.log
- grep '^PASS: a\.test (exit: 0)' a.log
- grep '^FAIL: b\.test (exit: 1)' b.log
- grep '^FAIL: b\.test (exit: 1)' test-suite.log
+ cat a.log
+ cat b.log
+ grep '^a\.test: exit status: 0$' a.log
+ grep '^b\.test: exit status: 1$' b.log
+ grep '^FAIL: b$' test-suite.log
+ grep '^b\.test: exit status: 1$' test-suite.log
+ grep '^a\.test' test-suite.log && Exit 1
else :; fi
CHECKLOCAL_EXIT_STATUS=1 $MAKE check && Exit 1
@@ -189,9 +195,13 @@ for vpath in : false; do
# This checks will be run only by the autogenerated `check12-p.test'.
if test x"$parallel_tests" = x"yes"; then
cat test-suite.log
- grep '^PASS: a\.test (exit: 0)' a.log
- grep '^FAIL: b\.test (exit: 23)' b.log
- grep '^FAIL: b\.test (exit: 23)' test-suite.log
+ cat a.log
+ cat b.log
+ grep '^a\.test: exit status: 0$' a.log
+ grep '^b\.test: exit status: 23$' b.log
+ grep '^FAIL: b$' test-suite.log
+ grep '^b\.test: exit status: 23$' test-suite.log
+ grep '^a\.test' test-suite.log && Exit 1
else :; fi
grep 'check-local failed :-(' local.log
diff --git a/tests/check2.test b/tests/check2.test
index 4edf0ee..4362424 100755
--- a/tests/check2.test
+++ b/tests/check2.test
@@ -43,6 +43,10 @@ echo.sh:
CLEANFILES = echo.sh
END
+if test x"$parallel_tests" = x"yes"; then
+ cp "$top_testsrcdir"/lib/test-driver .
+fi
+
$ACLOCAL
$AUTOCONF
$AUTOMAKE
diff --git a/tests/check3.test b/tests/check3.test
index b4f4f9d..4f32737 100755
--- a/tests/check3.test
+++ b/tests/check3.test
@@ -54,7 +54,7 @@ END
$ACLOCAL
$AUTOCONF
-$AUTOMAKE
+$AUTOMAKE -a
./configure --prefix "`pwd`/inst"
$MAKE check >stdout || { cat stdout; Exit 1; }
diff --git a/tests/check4.test b/tests/check4.test
index c7a23fe..bb9a2b6 100755
--- a/tests/check4.test
+++ b/tests/check4.test
@@ -46,7 +46,8 @@ chmod +x ok.sh dir/fail.sh
$ACLOCAL
$AUTOCONF
-$AUTOMAKE
+$AUTOMAKE --add-missing
+
./configure --prefix "`pwd`/inst"
$MAKE check >stdout && { cat stdout; Exit 1; }
diff --git a/tests/color.test b/tests/color.test
index d912aa2..12c1238 100755
--- a/tests/color.test
+++ b/tests/color.test
@@ -22,11 +22,14 @@
TERM=ansi
export TERM
-red='[0;31m'
-grn='[0;32m'
-lgn='[1;32m'
-blu='[1;34m'
-std='[m'
+esc=''
+# Escape `[' for grep, below.
+red="$esc\[0;31m"
+grn="$esc\[0;32m"
+lgn="$esc\[1;32m"
+blu="$esc\[1;34m"
+mgn="$esc\[0;35m"
+std="$esc\[m"
# Check that grep can parse nonprinting characters.
# BSD 'grep' works from a pipe, but not a seekable file.
@@ -43,7 +46,7 @@ END
cat >Makefile.am <<'END'
AUTOMAKE_OPTIONS = color-tests
TESTS = $(check_SCRIPTS)
-check_SCRIPTS = pass fail skip xpass xfail
+check_SCRIPTS = pass fail skip xpass xfail error
XFAIL_TESTS = xpass xfail
END
@@ -62,36 +65,93 @@ cat >skip <<END
exit 77
END
+cat >error <<END
+#! /bin/sh
+exit 99
+END
+
cp fail xfail
cp pass xpass
-chmod +x pass fail skip xpass xfail
+chmod +x pass fail skip xpass xfail error
$ACLOCAL
-$AUTOMAKE
$AUTOCONF
-./configure
+$AUTOMAKE --add-missing
test_color ()
{
# Not a useless use of cat; see above comments about grep.
- cat stdout | grep ": pass" | $FGREP "$grn"
- cat stdout | grep ": fail" | $FGREP "$red"
- cat stdout | grep ": xfail" | $FGREP "$lgn"
- cat stdout | grep ": xpass" | $FGREP "$red"
- cat stdout | grep ": skip" | $FGREP "$blu"
+ cat stdout | grep "^${grn}PASS${std}: .*pass"
+ cat stdout | grep "^${red}FAIL${std}: .*fail"
+ cat stdout | grep "^${blu}SKIP${std}: .*skip"
+ cat stdout | grep "^${lgn}XFAIL${std}: .*xfail"
+ cat stdout | grep "^${red}XPASS${std}: .*xpass"
+ # The old serial testsuite driver doesn't distinguish between failures
+ # and hard errors.
+ if test x"$parallel_tests" = x"yes"; then
+ cat stdout | grep "^${mgn}ERROR${std}: .*error"
+ else
+ cat stdout | grep "^${red}FAIL${std}: .*error"
+ fi
+ :
}
test_no_color ()
{
- # Not a useless use of cat; see above comments about grep.
- cat stdout | grep ": pass" | $FGREP "$grn" && Exit 1
- cat stdout | grep ": fail" | $FGREP "$red" && Exit 1
- cat stdout | grep ": xfail" | $FGREP "$lgn" && Exit 1
- cat stdout | grep ": xpass" | $FGREP "$red" && Exit 1
- cat stdout | grep ": skip" | $FGREP "$blu" && Exit 1
- :
+ # With make implementations that, like Solaris make, in case of errors
+ # print the whole failing recipe on standard output, we should content
+ # ourselves with a laxer check, to avoid false positives.
+ # Keep this in sync with lib/am/check.am:$(am__color_tests).
+ if $FGREP '= Xalways || test -t 1 ' stdout; then
+ # Extra verbose make, resort to laxer checks.
+ # Note that we also want to check that the testsuite summary is
+ # not unduly colorized.
+ (
+ set +e # In case some grepped regex below isn't matched.
+ # Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "TOTAL.*:"
+ cat stdout | grep "PASS.*:"
+ cat stdout | grep "FAIL.*:"
+ cat stdout | grep "SKIP.*:"
+ cat stdout | grep "XFAIL.*:"
+ cat stdout | grep "XPASS.*:"
+ cat stdout | grep "ERROR.*:"
+ cat stdout | grep 'test.*expected'
+ cat stdout | grep 'test.*not run'
+ cat stdout | grep '===='
+ cat stdout | grep '[Ss]ee .*test-suite\.log'
+ cat stdout | grep '[Tt]estsuite summary'
+ ) | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ else
+ cat stdout | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ fi
}
-AM_COLOR_TESTS=always $MAKE -e check >stdout && { cat stdout; Exit 1; }
-cat stdout
-test_color
+for vpath in false :; do
+
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ AM_COLOR_TESTS=always $MAKE -e check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ test_color
+
+ $MAKE -e check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ test_no_color
+
+ $MAKE distclean
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/color2.test b/tests/color2.test
index 26df9b2..96eac46 100755
--- a/tests/color2.test
+++ b/tests/color2.test
@@ -22,11 +22,14 @@
TERM=ansi
export TERM
-red='[0;31m'
-grn='[0;32m'
-lgn='[1;32m'
-blu='[1;34m'
-std='[m'
+esc=''
+# Escape `[' for grep, below.
+red="$esc\[0;31m"
+grn="$esc\[0;32m"
+lgn="$esc\[1;32m"
+blu="$esc\[1;34m"
+mgn="$esc\[0;35m"
+std="$esc\[m"
# Check that grep can parse nonprinting characters.
# BSD 'grep' works from a pipe, but not a seekable file.
@@ -54,7 +57,7 @@ END
cat >Makefile.am <<'END'
AUTOMAKE_OPTIONS = color-tests
TESTS = $(check_SCRIPTS)
-check_SCRIPTS = pass fail skip xpass xfail
+check_SCRIPTS = pass fail skip xpass xfail error
XFAIL_TESTS = xpass xfail
END
@@ -73,34 +76,68 @@ cat >skip <<END
exit 77
END
+cat >error <<END
+#! /bin/sh
+exit 99
+END
+
cp fail xfail
cp pass xpass
-chmod +x pass fail skip xpass xfail
+chmod +x pass fail skip xpass xfail error
$ACLOCAL
-$AUTOMAKE
$AUTOCONF
-./configure
+$AUTOMAKE --add-missing
test_color ()
{
# Not a useless use of cat; see above comments about grep.
- cat stdout | grep ": pass" | $FGREP "$grn"
- cat stdout | grep ": fail" | $FGREP "$red"
- cat stdout | grep ": xfail" | $FGREP "$lgn"
- cat stdout | grep ": xpass" | $FGREP "$red"
- cat stdout | grep ": skip" | $FGREP "$blu"
+ cat stdout | grep "^${grn}PASS${std}: .*pass"
+ cat stdout | grep "^${red}FAIL${std}: .*fail"
+ cat stdout | grep "^${blu}SKIP${std}: .*skip"
+ cat stdout | grep "^${lgn}XFAIL${std}: .*xfail"
+ cat stdout | grep "^${red}XPASS${std}: .*xpass"
+ # The old serial testsuite driver doesn't distinguish between failures
+ # and hard errors.
+ if test x"$parallel_tests" = x"yes"; then
+ cat stdout | grep "^${mgn}ERROR${std}: .*error"
+ else
+ cat stdout | grep "^${red}FAIL${std}: .*error"
+ fi
+ :
}
test_no_color ()
{
- # Not a useless use of cat; see above comments about grep.
- cat stdout | grep ": pass" | $FGREP "$grn" && Exit 1
- cat stdout | grep ": fail" | $FGREP "$red" && Exit 1
- cat stdout | grep ": xfail" | $FGREP "$lgn" && Exit 1
- cat stdout | grep ": xpass" | $FGREP "$red" && Exit 1
- cat stdout | grep ": skip" | $FGREP "$blu" && Exit 1
- :
+ # With make implementations that, like Solaris make, in case of errors
+ # print the whole failing recipe on standard output, we should content
+ # ourselves with a laxer check, to avoid false positives.
+ # Keep this in sync with lib/am/check.am:$(am__color_tests).
+ if $FGREP '= Xalways || test -t 1 ' stdout; then
+ # Extra verbose make, resort to laxer checks.
+ # Note that we also want to check that the testsuite summary is
+ # not unduly colorized.
+ (
+ set +e # In case some grepped regex below isn't matched.
+ # Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "TOTAL.*:"
+ cat stdout | grep "PASS.*:"
+ cat stdout | grep "FAIL.*:"
+ cat stdout | grep "SKIP.*:"
+ cat stdout | grep "XFAIL.*:"
+ cat stdout | grep "XPASS.*:"
+ cat stdout | grep "ERROR.*:"
+ cat stdout | grep 'test.*expected'
+ cat stdout | grep 'test.*not run'
+ cat stdout | grep '===='
+ cat stdout | grep '[Ss]ee .*test-suite\.log'
+ cat stdout | grep '[Tt]estsuite summary'
+ ) | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ else
+ cat stdout | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ fi
}
cat >expect-make <<'END'
@@ -109,12 +146,31 @@ eval spawn $env(MAKE) -e check
expect eof
END
-MAKE=$MAKE expect -f expect-make >stdout \
- || { cat stdout; Exit 1; }
-cat stdout
-test_color
+for vpath in false :; do
+
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ MAKE=$MAKE expect -f $srcdir/expect-make >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ test_color
+
+ AM_COLOR_TESTS=no MAKE=$MAKE expect -f $srcdir/expect-make >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ test_no_color
+
+ $MAKE distclean
+ cd $srcdir
+
+done
-AM_COLOR_TESTS=no MAKE=$MAKE expect -f expect-make >stdout \
- || { cat stdout; Exit 1; }
-cat stdout
-test_no_color
+:
diff --git a/tests/comment9.test b/tests/comment9.test
index e95d99b..e20b59f 100755
--- a/tests/comment9.test
+++ b/tests/comment9.test
@@ -30,6 +30,8 @@ TESTS = \
7.test
EOF
+: > test-driver
+
$ACLOCAL
$AUTOMAKE
diff --git a/tests/cond44.test b/tests/cond44.test
index 3d68a41..ea5e87d 100755
--- a/tests/cond44.test
+++ b/tests/cond44.test
@@ -32,7 +32,7 @@ endif
EOF
$ACLOCAL
-AUTOMAKE_run 0
+AUTOMAKE_run
grep 'libdir was already defined' stderr && Exit 1
grep '^libdir = ' Makefile.in && Exit 1
Exit 0
diff --git a/tests/cond45.test b/tests/cond45.test
index ebaa2af..05d68dc 100755
--- a/tests/cond45.test
+++ b/tests/cond45.test
@@ -33,7 +33,7 @@ endif
EOF
$ACLOCAL
-AUTOMAKE_run 0
+AUTOMAKE_run
grep 'foo was already defined' stderr && Exit 1
grep '^foo =' Makefile.in && Exit 1
Exit 0
diff --git a/tests/configure.test b/tests/configure.test
index 0e62906..f9954af 100755
--- a/tests/configure.test
+++ b/tests/configure.test
@@ -47,7 +47,7 @@ AUTOMAKE_fails -Werror
grep 'configure\.ac.*configure\.in.*both present' stderr
grep 'proceeding.*configure\.ac' stderr
-AUTOMAKE_run 0 -Wno-error
+AUTOMAKE_run -Wno-error
grep 'configure\.ac.*configure\.in.*both present' stderr
grep 'proceeding.*configure\.ac' stderr
diff --git a/tests/defs b/tests/defs
index 7fddcbf..047e22f 100644
--- a/tests/defs
+++ b/tests/defs
@@ -17,18 +17,18 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+########################################################
+### IMPORTANT NOTE: keep this file 'set -e' clean. ###
+########################################################
-# IMPORTANT NOTE: This file should execute correctly with any system's
-# /bin/sh shell, and not only with configure-time detected $CONFIG_SHELL,
-# until differently and explicitly specified.
-
+# NOTE: This file should execute correctly with any system's /bin/sh
+# shell, and not only with configure-time detected $CONFIG_SHELL,
+# *until differently and explicitly specified*.
## -------------------------------------------------------- ##
## Source static setup and definitions for the testsuite. ##
## -------------------------------------------------------- ##
-# This code needs to be 'set -e' clean.
-
# Ensure we are running from the right directory.
test -f ./defs-static || {
echo "$0: ./defs-static: not found in current directory" >&2
@@ -38,12 +38,15 @@ test -f ./defs-static || {
# Source the shell sanitization and variables' definitions.
. ./defs-static || exit 99
+# Enable the errexit shell flag early.
+set -e
+
# The name of the current test (without the `.test' suffix).
# Test scripts can override it if they need to (but this should
# be done carefully, and *before* including ./defs).
if test -z "$me"; then
# Guard against failure to spawn sed (seen on MSYS), or empty $argv0.
- me=`echo "$argv0" | sed -e 's,.*[\\/],,;s/\.test$//'` \
+ me=`echo "$argv0" | sed -e 's,.*[\\/],,;s/\.test$//;s/\.tap$//;'` \
&& test -n "$me" \
|| { echo "$argv0: failed to define \$me" >&2; exit 99; }
fi
@@ -103,9 +106,10 @@ case ${AM_TESTS_REEXEC-yes} in
*) opts=;;
esac
echo $me: exec $SHELL $opts "$0" "$*"
- exec $SHELL $opts "$0" ${1+"$@"}
- echo "$me: failed to re-execute with $SHELL" >&2
- exit 99
+ exec $SHELL $opts "$0" ${1+"$@"} || {
+ echo "$me: failed to re-execute with $SHELL" >&2
+ exit 99
+ }
;;
esac
@@ -117,6 +121,11 @@ esac
## Environment cleanup. ##
## ---------------------- ##
+# Temporarily disable this, since some shells (e.g., older version
+# of Bash) can return a non-zero exit status upon the when a non-set
+# variable is unset.
+set +e
+
# Unset some MAKE... variables that may cause $MAKE to act like a
# recursively invoked sub-make. Any $MAKE invocation in a test is
# conceptually an independent invocation, not part of the main
@@ -148,6 +157,8 @@ unset TEST_LOGS
unset RECHECK_LOGS
unset VERBOSE
+# Re-enable, it had been temporarily disabled above.
+set -e
## ---------------------------- ##
## Auxiliary shell functions. ##
@@ -170,21 +181,26 @@ Exit ()
(exit $1); exit $1
}
-# Print warnings (e.g., about skipped and failed tests) to this file
-# number. Override by putting, say:
-# stderr_fileno_=9; export stderr_fileno_; 9>&2
-# at the *end* (yes, this is mandatory; see comments in Makefile.am) of
-# the definition of AM_TESTS_ENVIRONMENT.
-# This is useful when using automake's parallel tests mode, to print the
-# reason for skip/failure to console, rather than to the *.log files.
-: ${stderr_fileno_=2}
-
-# Copied from Gnulib's `tests/init.sh'.
-warn_ () { echo "$@" 1>&$stderr_fileno_; }
-fail_ () { warn_ "$me: failed test: $@"; Exit 1; }
-skip_ () { warn_ "$me: skipped test: $@"; Exit 77; }
-fatal_ () { warn_ "$me: hard error: $@"; Exit 99; }
-framework_failure_ () { warn_ "$me: set-up failure: $@"; Exit 99; }
+if test $using_tap = yes; then
+ funcs_file_=tap-functions.sh
+else
+ funcs_file_=plain-functions.sh
+fi
+
+if test -f "$testsrcdir/$funcs_file_"; then
+ . "$testsrcdir/$funcs_file_" || {
+ echo "$me: error sourcing $testsrcdir/$funcs_file_" >&2
+ Exit 99
+ }
+else
+ echo "$me: $testsrcdir/$funcs_file_ not found, check \$testsrcdir" >&2
+ Exit 99
+fi
+unset funcs_file_
+
+# Avoid spurious TAP diagnostic. Keep this in sync with the definition
+# of $(TAP_LOG_DRIVER_FLAGS) in `tests/Makefile.am'.
+diag_string_="#%#"
# cross_compiling
# ---------------
@@ -205,28 +221,55 @@ is_newest ()
test -z "$is_newest_files"
}
-# AUTOMAKE_run status [options...]
-# --------------------------------
-# Run Automake with OPTIONS, and fail if automake
-# does not exit with STATUS.
+# AUTOMAKE_run [-e STATUS] [-d DESCRIPTION] [--] [AUTOMAKE-ARGS...]
+# -----------------------------------------------------------------
+# Run automake with AUTOMAKE-ARGS, and fail if it doesn't exit with
+# STATUS. Should be polymorphic for TAP and "plain" tests. The
+# DESCRIPTION, when provided, is used for console reporting, only if
+# the TAP protocol is in use in the current test script.
AUTOMAKE_run ()
{
- expected_exitcode=$1
- shift
- exitcode=0
- $AUTOMAKE ${1+"$@"} >stdout 2>stderr || exitcode=$?
+ am__desc=
+ am__exp_rc=0
+ while test $# -gt 0; do
+ case $1 in
+ -d) am__desc=$2; shift;;
+ -e) am__exp_rc=$2; shift;;
+ --) shift; break;;
+ # Don't fail on unknown option: assume they (and the rest of the
+ # command line) are to be passed verbatim to automake (so stop our
+ # own option parsing).
+ *) break;;
+ esac
+ shift
+ done
+ am__got_rc=0
+ $AUTOMAKE ${1+"$@"} >stdout 2>stderr || am__got_rc=$?
cat stderr >&2
cat stdout
- test $exitcode = $expected_exitcode || Exit 1
+ if test $using_tap != yes; then
+ test $am__got_rc -eq $am__exp_rc || Exit 1
+ return
+ fi
+ if test -z "$am__desc"; then
+ if test $am__got_rc -eq $am__exp_rc; then
+ am__desc="automake exited $am__got_rc"
+ else
+ am__desc="automake exited $am__got_rc, expecting $am__exp_rc"
+ fi
+ fi
+ command_ok_ "$am__desc" test $am__got_rc -eq $am__exp_rc
}
-# AUTOMAKE_fails [options...]
-# ---------------------------
-# Run Automake with OPTIONS, and fail if automake
-# does not exit with STATUS.
+# AUTOMAKE_fails [-d DESCRIPTION] [OPTIONS...]
+# --------------------------------------------
+# Run automake with OPTIONS, and fail if doesn't exit with status `1'.
+# Should be polymorphic for TAP and "plain" tests. The DESCRIPTION,
+# when provided, is used for console reporting, only if the TAP
+# protocol is in use in the current test script.
AUTOMAKE_fails ()
{
- AUTOMAKE_run 1 ${1+"$@"}
+ AUTOMAKE_run -e 1 ${1+"$@"}
}
# using_gmake
@@ -252,12 +295,46 @@ using_gmake ()
return 1
fi;;
*)
- echo "invalid value for \$am__using_gmake: '$am__using_gmake'" >&2
- Exit 99;;
+ fatal_ "invalid value for \$am__using_gmake: '$am__using_gmake'";;
esac
}
am__using_gmake="" # Avoid interferences from the environment.
+# make_can_chain_suffix_rules
+# ---------------------------
+# Return 0 if $MAKE is a make implementation that can chain suffix rules
+# automatically, return 1 otherwise. Caches the result for speed reasons.
+make_can_chain_suffix_rules ()
+{
+ if test -z "$am__can_chain_suffix_rules"; then
+ if using_gmake; then
+ am__can_chain_suffix_rules=yes
+ return 0
+ else
+ mkdir am__chain.dir$$
+ cd am__chain.dir$$
+ unindent > Makefile << 'END'
+ .u.v: ; cp $< $@
+ .v.w: ; cp $< $@
+END
+ echo make can chain suffix rules > foo.u
+ if $MAKE foo.w && diff foo.u foo.w; then
+ am__can_chain_suffix_rules=yes
+ else
+ am__can_chain_suffix_rules=no
+ fi
+ cd ..
+ rm -rf am__chain.dir$$
+ fi
+ fi
+ case $am__can_chain_suffix_rules in
+ yes) return 0;;
+ no) return 1;;
+ *) fatal_ "make_can_chain_suffix_rules: internal error";;
+ esac
+}
+am__can_chain_suffix_rules="" # Avoid interferences from the environment.
+
# seq_ - print a sequence of numbers
# ----------------------------------
# This function simulates GNU seq(1) portably. Valid usages:
@@ -294,6 +371,56 @@ seq_ ()
done
}
+# rm_rf_ [FILES OR DIRECTORIES ...]
+# ---------------------------------
+# Recursively remove the given files or directory, also handling the case
+# of non-writable subdirectories.
+rm_rf_ ()
+{
+ test $# -gt 0 || return 0
+ # Ignore failures in find, we are only interested in failures of the
+ # final rm.
+ find "$@" -type d ! -perm -700 -exec chmod u+rwx {} \; || :
+ rm -rf "$@"
+}
+
+# count_test_results total=N pass=N fail=N xpass=N xfail=N skip=N error=N
+# -----------------------------------------------------------------------
+# Check that a testsuite run driven by the parallel-tests harness has
+# had the specified numbers of test results (specified by kind).
+# This function assumes that the output of "make check" or "make recheck"
+# has been saved in the `stdout' file in the current directory, and its
+# log in the `test-suite.log' file.
+count_test_results ()
+{
+ # Use a subshell so that we won't pollute the script namespace.
+ (
+ # TODO: Do proper checks on the arguments?
+ total=ERR pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
+ eval "$@"
+ # For debugging.
+ $EGREP -i '(total|x?pass|x?fail|skip|error)' stdout || :
+ rc=0
+ # Avoid spurious failures with shells with "overly sensible"
+ # `errexit' shell flag, such as e.g., Solaris /bin/sh.
+ set +e
+ test `grep -c '^PASS:' stdout` -eq $pass || rc=1
+ test `grep -c '^XFAIL:' stdout` -eq $xfail || rc=1
+ test `grep -c '^SKIP:' stdout` -eq $skip || rc=1
+ test `grep -c '^FAIL:' stdout` -eq $fail || rc=1
+ test `grep -c '^XPASS:' stdout` -eq $xpass || rc=1
+ test `grep -c '^ERROR:' stdout` -eq $error || rc=1
+ grep "^# TOTAL: *$total$" stdout || rc=1
+ grep "^# PASS: *$pass$" stdout || rc=1
+ grep "^# XFAIL: *$xfail$" stdout || rc=1
+ grep "^# SKIP: *$skip$" stdout || rc=1
+ grep "^# FAIL: *$fail$" stdout || rc=1
+ grep "^# XPASS: *$xpass$" stdout || rc=1
+ grep "^# ERROR: *$error$" stdout || rc=1
+ test $rc -eq 0
+ )
+}
+
commented_sed_unindent_prog='
/^$/b # Nothing to do for empty lines.
x # Get x<indent> into pattern space.
@@ -348,7 +475,7 @@ require_xsi ()
{
test $# -eq 1 || fatal_ "require_xsi needs exactly one argument"
echo "$me: trying some XSI constructs with $1"
- $1 -c "$xsi_shell_code" || skip_ "$1 lacks XSI features"
+ $1 -c "$xsi_shell_code" || skip_all_ "$1 lacks XSI features"
}
# Shell code supposed to work only with XSI shells. Keep this in sync
# with libtool.m4:_LT_CHECK_SHELL_FEATURES.
@@ -385,13 +512,13 @@ do
case $tool in
:) ;;
cc)
- test "$CC" = false && skip_ "no C compiler available";;
+ test "$CC" = false && skip_all_ "no C compiler available";;
c++)
- test "$CXX" = false && skip_ "no C++ compiler available";;
+ test "$CXX" = false && skip_all_ "no C++ compiler available";;
fortran)
- test "$FC" = false && skip_ "no Fortran compiler available";;
+ test "$FC" = false && skip_all_ "no Fortran compiler available";;
fortran77)
- test "$F77" = false && skip_ "no Fortran 77 compiler available";;
+ test "$F77" = false && skip_all_ "no Fortran 77 compiler available";;
xsi-shell)
require_xsi "$SHELL";;
xsi-bin-sh)
@@ -407,19 +534,19 @@ do
# Do not use --version, bzip2 still tries to compress stdin.
echo "$me: running bzip2 --help"
bzip2 --help \
- || skip_ "required program \`bzip2' not available"
+ || skip_all_ "required program \`bzip2' not available"
;;
cl)
CC=cl
export CC
echo "$me: running $CC -?"
- $CC -? || skip_ "Microsoft C compiler \`$CC' not available"
+ $CC -? || skip_all_ "Microsoft C compiler \`$CC' not available"
;;
cscope)
# Sun cscope is interactive without redirection.
echo "$me: running cscope --version </dev/null"
cscope --version </dev/null \
- || skip_ "required program \`cscope' not available"
+ || skip_all_ "required program \`cscope' not available"
;;
etags)
# Exuberant Ctags will create a TAGS file even
@@ -428,7 +555,7 @@ do
# to make sure we do not pollute the tests/ directory.
echo "$me: running etags --version -o /dev/null"
etags --version -o /dev/null \
- || skip_ "required program \`etags' not available"
+ || skip_all_ "required program \`etags' not available"
;;
GNUmake)
for make_ in "$MAKE" gmake gnumake :; do
@@ -437,7 +564,7 @@ do
echo "$me: determine whether $MAKE is GNU make"
using_gmake && break
done
- test "$MAKE" = : && skip_ "this test requires GNU make"
+ test "$MAKE" = : && skip_all_ "this test requires GNU make"
export MAKE
unset make_
;;
@@ -449,33 +576,33 @@ do
CC=${am__tool_prefix}gcc
export CC
echo "$me: running $CC --version"
- $CC --version || skip_ "GNU C compiler not available"
+ $CC --version || skip_all_ "GNU C compiler not available"
echo "$me: running $CC -v"
- $CC -v || skip_ "botched installation for GNU C compiler"
+ $CC -v || skip_all_ "botched installation for GNU C compiler"
;;
gcj)
GCJ=${am__tool_prefix}gcj
export GCJ
echo "$me: running $GCJ --version"
- $GCJ --version || skip_ "GNU Java compiler not available"
+ $GCJ --version || skip_all_ "GNU Java compiler not available"
echo "$me: running $GCJ -v"
- $GCJ -v || skip_ "botched installation for GNU Java compiler"
+ $GCJ -v || skip_all_ "botched installation for GNU Java compiler"
;;
g++)
CXX=${am__tool_prefix}g++
export CXX
echo "$me: running $CXX --version"
- $CXX --version || skip_ "GNU C++ compiler not available"
+ $CXX --version || skip_all_ "GNU C++ compiler not available"
echo "$me: running $CXX -v"
- $CXX -v || skip_ "botched installation for GNU C++ compiler"
+ $CXX -v || skip_all_ "botched installation for GNU C++ compiler"
;;
gfortran)
FC=${am__tool_prefix}gfortran
export FC
echo "$me: running $FC --version"
- $FC --version || skip_ "GNU Fortran compiler not available"
+ $FC --version || skip_all_ "GNU Fortran compiler not available"
echo "$me: running $FC -v"
- $FC -v || skip_ "botched installation for GNU Fortran compiler"
+ $FC -v || skip_all_ "botched installation for GNU Fortran compiler"
case " $required " in
*\ g77\ *) ;;
*) F77=$FC; export F77;;
@@ -485,9 +612,9 @@ do
F77=${am__tool_prefix}g77
export F77
echo "$me: running $F77 --version"
- $F77 --version || skip_ "GNU Fortran 77 compiler not available"
+ $F77 --version || skip_all_ "GNU Fortran 77 compiler not available"
echo "$me: running $F77 -v"
- $F77 -v || skip_ "botched installation for GNU Fortran 77 compiler"
+ $F77 -v || skip_all_ "botched installation for GNU Fortran 77 compiler"
case " $required " in
*\ gfortran\ *) ;;
*) FC=$F77; export FC;;
@@ -501,7 +628,7 @@ do
# it will try link *nothing* and complain it cannot find
# main(); funny). Use -help so it does not try linking anything.
echo "$me: running $CC -V -help"
- $CC -V -help || skip_ "Intel C compiler \`$CC' not available"
+ $CC -V -help || skip_all_ "Intel C compiler \`$CC' not available"
;;
javac)
# The Java compiler from JDK 1.5 (and presumably earlier versions)
@@ -509,58 +636,62 @@ do
# telling that source files are missing. Adding also the `-help'
# option seems to solve the problem.
echo "$me: running javac -version -help"
- javac -version -help || skip_ "Sun Java compiler not available"
+ javac -version -help || skip_all_ "Sun Java compiler not available"
;;
makedepend)
echo "$me: running makedepend -f-"
makedepend -f- \
- || skip_ "required program \`makedepend' not available"
+ || skip_all_ "required program \`makedepend' not available"
;;
makeinfo-html)
# Make sure we have makeinfo, and it understands `--html'.
echo "$me: running makeinfo --html --version"
makeinfo --html --version \
- || skip_ "cannot find a makeinfo program that groks the" \
+ || skip_all_ "cannot find a makeinfo program that groks the" \
"\`--html' option"
;;
non-root)
# Skip this test case if the user is root.
# We try to append to a read-only file to detect this.
priv_check_temp=priv-check.$$
- touch $priv_check_temp || exit 99
- chmod a-w $priv_check_temp || exit 99
- (echo foo >> $priv_check_temp) >/dev/null 2>&1
- overwrite_status=$?
+ touch $priv_check_temp && chmod a-w $priv_check_temp \
+ || framework_failure_ "creating unwritable file $priv_check_temp"
+ # Not a useless use of subshell: lesser shells like Solaris /bin/sh
+ # can exit if a builtin fails.
+ overwrite_status=0
+ (echo foo >> $priv_check_temp) || overwrite_status=$?
rm -f $priv_check_temp
if test $overwrite_status -eq 0; then
- skip_ "cannot drop file write permissions"
+ skip_all_ "cannot drop file write permissions"
fi
unset priv_check_temp overwrite_status
;;
perl-threads)
if test "$WANT_NO_THREADS" = "yes"; then
- skip_ "Devel::Cover cannot cope with threads"
+ skip_all_ "Devel::Cover cannot cope with threads"
fi
;;
native)
- cross_compiling && skip_ "doesn't work in cross-compile mode"
+ cross_compiling && skip_all_ "doesn't work in cross-compile mode"
;;
python)
# Python doesn't support --version, it has -V
echo "$me: running python -V"
- python -V || skip_ "python interpreter not available"
+ python -V || skip_all_ "python interpreter not available"
;;
ro-dir)
# Skip this test case if read-only directories aren't supported
# (e.g., under DOS.)
ro_dir_temp=ro_dir.$$
- mkdir $ro_dir_temp || exit 99
- chmod a-w $ro_dir_temp || exit 99
- (: > $ro_dir_temp/probe) >/dev/null 2>/dev/null
- create_status=$?
+ mkdir $ro_dir_temp && chmod a-w $ro_dir_temp \
+ || framework_failure_ "creating unwritable directory $ro_dir_temp"
+ # Not a useless use of subshell: lesser shells like Solaris /bin/sh
+ # can exit if a builtin fails.
+ create_status=0
+ (: > $ro_dir_temp/probe) || create_status=$?
rm -rf $ro_dir_temp
if test $create_status -eq 0; then
- skip_ "cannot drop directory write permissions"
+ skip_all_ "cannot drop directory write permissions"
fi
unset ro_dir_temp create_status
;;
@@ -571,7 +702,7 @@ do
echo "$me: running $r2h --version"
$r2h --version && break 2
done
- skip_ "no proper rst2html program found"
+ skip_all_ "no proper rst2html program found"
done
unset r2h
;;
@@ -581,30 +712,30 @@ do
# DejaGnu 1.4.3 or later.
echo "$me: running runtest SOMEPROGRAM=someprogram --version"
runtest SOMEPROGRAM=someprogram --version \
- || skip_ "DejaGnu is not available"
+ || skip_all_ "DejaGnu is not available"
;;
tex)
# No all versions of Tex support `--version', so we use
# a configure check.
if test -z "$TEX"; then
- skip_ "TeX is required, but it wasn't found by configure"
+ skip_all_ "TeX is required, but it wasn't found by configure"
fi
;;
texi2dvi-o)
# Texi2dvi supports `-o' since Texinfo 4.1.
echo "$me: running texi2dvi -o /dev/null --version"
texi2dvi -o /dev/null --version \
- || skip_ "required program \`texi2dvi' not available"
+ || skip_all_ "required program \`texi2dvi' not available"
;;
flex)
# Since flex is required, we pick LEX for ./configure.
LEX=flex
export LEX
echo "$me: running flex --version"
- flex --version || skip_ "required program \`flex' not available"
+ flex --version || skip_all_ "required program \`flex' not available"
;;
lex)
- test "$LEX" = false && skip_ "no Lex program available"
+ test "$LEX" = false && skip_all_ "no Lex program available"
if test -z "$LEX"; then
# The user hasn't explicitly specified any lex program in the
# environment, so we try to use flex, skipping the test if it's
@@ -612,11 +743,11 @@ do
LEX=flex
export LEX
echo "$me: running flex --version"
- flex --version || skip_ "required program \`flex' not available"
+ flex --version || skip_all_ "required program \`flex' not available"
fi
;;
yacc)
- test "$YACC" = false && skip_ "no Yacc program available"
+ test "$YACC" = false && skip_all_ "no Yacc program available"
if test -z "$YACC"; then
# The user hasn't explicitly specified any yacc program in the
# environment, so we try to use bison, skipping the test if it's
@@ -624,7 +755,7 @@ do
YACC='bison -y'
export YACC
echo "$me: running bison --version"
- bison --version || skip_ "required program \`bison' not available"
+ bison --version || skip_all_ "required program \`bison' not available"
fi
;;
*)
@@ -633,7 +764,7 @@ do
# It is not likely but possible that $tool is a special builtin,
# in which case the shell is allowed to exit after an error.
# So, please leave the subshell here.
- ($tool --version) || skip_ "required program \`$tool' not available"
+ ($tool --version) || skip_all_ "required program \`$tool' not available"
;;
esac
done
@@ -645,7 +776,7 @@ case $testbuilddir in
*\ *|*\ *)
case " $required " in
*' libtool '* | *' libtoolize '* )
- skip_ "libtool has problems with spaces in builddir name";;
+ skip_all_ "libtool has problems with spaces in builddir name";;
esac
;;
esac
@@ -658,7 +789,7 @@ case $testsrcdir in
*\ * |*\ *)
case " $required " in
*' libtool '* | *' libtoolize '* | *' gettext '* )
- skip_ "spaces in srcdir name: libtool/gettext tests won't work";;
+ skip_all_ "spaces in srcdir name: libtool/gettext tests won't work";;
esac
;;
esac
@@ -698,7 +829,7 @@ case " $required " in
# Write more diagnostic to the log file than to the console.
echo "$me: \`libtool.m4' wasn't found in directories" \
"$aclocaldir $extra_includes"
- skip_ "libtool m4 macros won't be found by aclocal"
+ skip_all_ "libtool m4 macros won't be found by aclocal"
fi
;;
*' gettext '*)
@@ -706,7 +837,7 @@ case " $required " in
# Write more diagnostic to the log file than to the console.
echo "$me: \`gettext.m4' wasn't found in directories" \
"$aclocaldir $extra_includes"
- skip_ "gettext m4 macros won't be found by aclocal"
+ skip_all_ "gettext m4 macros won't be found by aclocal"
fi
;;
esac
@@ -726,78 +857,91 @@ esac
# Test scripts are free to override this if they need to.
distdir=$me-1.0
-# The subdirectory where the current test script will run and write its
-# temporary/data files. This will be created shortly, and will be removed
-# by the cleanup trap below if the test passes. If the test doesn't pass,
-# this directory will be kept, to facilitate debugging.
-testSubDir=$me.dir
-
-test ! -d $testSubDir || {
- find $testSubDir -type d ! -perm -700 -exec chmod u+rwx {} ";"
- rm -rf $testSubDir
-}
-mkdir $testSubDir
-
-cd ./$testSubDir
-
+# Set up the exit trap.
if test "$sh_errexit_works" = yes; then
trap 'exit_status=$?
set +e
cd "$testbuilddir"
- # This is to ensure that a test script does give a SKIP outcome just
- # because a command in it happens to exit with status 77. This
- # behaviour, while from time to time useful to developers, is not
- # meant to be enabled by default, as it could cause spurious failures
- # in the wild. Thus it will be enabled only when the variable
- # "am_explicit_skips" is set to a "true" value.
- case $am_explicit_skips in
- [yY]|[yY]es|1)
- if test $exit_status -eq 77 && test $am__test_skipped != yes; then
- exit_status=78
- fi
- ;;
- esac
- case $exit_status,$keep_testdirs in
- 0,)
- find $testSubDir -type d ! -perm -700 -exec chmod u+rwx {} ";"
- rm -rf $testSubDir
- ;;
- esac
- test "$signal" != 0 && echo "$me: caught signal $signal"
+ if test -n "$keep_testdirs"; then
+ keep_testdirs=yes
+ else
+ keep_testdirs=no
+ fi
+ if test $using_tap = yes; then
+ if test $have_tap_plan_ != yes; then
+ plan_ "now"
+ fi
+ test $exit_status -eq 0 \
+ && test $tap_xfail_count_ -eq 0 \
+ && test $tap_skip_count_ -eq 0 \
+ && test $tap_bad_count_ -eq 0 \
+ || keep_testdirs=yes
+ else
+ # This is to ensure that a test script does give a SKIP outcome just
+ # because a command in it happens to exit with status 77. This
+ # behaviour, while from time to time useful to developers, is not
+ # meant to be enabled by default, as it could cause spurious failures
+ # in the wild. Thus it will be enabled only when the variable
+ # "am_explicit_skips" is set to a "true" value.
+ case $am_explicit_skips in
+ [yY]|[yY]es|1)
+ if test $exit_status -eq 77 && test $am__test_skipped != yes; then
+ echo "$me: implicit skip turned into failure"
+ exit_status=78
+ fi;;
+ esac
+ test $exit_status -eq 0 || keep_testdirs=yes
+ fi
+ test $keep_testdirs = no && rm_rf_ $testSubDir
+ set +x
echo "$me: exit $exit_status"
exit $exit_status
' 0
for signal in 1 2 13 15; do
- trap 'signal='$signal'; { Exit 99; }' $signal
+ trap "fatal_ 'caught signal $signal'" $signal
done
- signal=0
+ unset signal
fi
-# Copy in some files we need.
-for file in install-sh missing depcomp; do
- cp "$top_testsrcdir/lib/$file" . || Exit 99
-done
-
-# Build appropriate environment in test directory. Eg create
-# configure.in, touch all necessary files, etc.
-# Don't use AC_OUTPUT, but AC_CONFIG_FILES so that appending
-# still produces a valid configure.ac. But then, tests running
-# config.status really need to append AC_OUTPUT.
-{
- echo "AC_INIT([$me], [1.0])"
- if test x"$parallel_tests" = x"yes"; then
- echo "AM_INIT_AUTOMAKE([parallel-tests])"
- else
- echo "AM_INIT_AUTOMAKE"
+# Create and populate the temporary directory, if and as required.
+if test x"$am_create_testdir" = x"no"; then
+ testSubDir=
+else
+ # The subdirectory where the current test script will run and write its
+ # temporary/data files. This will be created shortly, and will be removed
+ # by the cleanup trap below if the test passes. If the test doesn't pass,
+ # this directory will be kept, to facilitate debugging.
+ testSubDir=$me.dir
+ test ! -d $testSubDir || rm_rf_ $testSubDir \
+ || framework_failure_ "removing old test subdirectory"
+ mkdir $testSubDir \
+ || framework_failure_ "creating test subdirectory"
+ cd ./$testSubDir \
+ || framework_failure_ "cannot chdir into test subdirectory"
+ if test x"$am_create_testdir" != x"empty"; then
+ cp "$top_testsrcdir"/lib/install-sh "$top_testsrcdir"/lib/missing \
+ "$top_testsrcdir"/lib/depcomp . \
+ || framework_failure_ "fetching common files from lib/"
+ # Build appropriate environment in test directory. E.g., create
+ # configure.in, touch all necessary files, etc. Don't use AC_OUTPUT,
+ # but AC_CONFIG_FILES so that appending still produces a valid
+ # configure.in. But then, tests running config.status really need
+ # to append AC_OUTPUT.
+ {
+ echo "AC_INIT([$me], [1.0])"
+ if test x"$parallel_tests" = x"yes"; then
+ echo "AM_INIT_AUTOMAKE([parallel-tests])"
+ else
+ echo "AM_INIT_AUTOMAKE"
+ fi
+ echo "AC_CONFIG_FILES([Makefile])"
+ } >configure.in || framework_failure_ "creating configure.in skeleton"
fi
- echo "AC_CONFIG_FILES([Makefile])"
-} >configure.in
+fi
## ---------------- ##
## Ready to go... ##
## ---------------- ##
set -x
-set -e
-
pwd
diff --git a/tests/defs-static.in b/tests/defs-static.in
index 17ed284..0a7b77a 100644
--- a/tests/defs-static.in
+++ b/tests/defs-static.in
@@ -70,9 +70,16 @@ fi
# Check that the environment is properly sanitized.
# Having variables exported to the empty string is OK, since our code
# treats such variables as if they were unset.
-for var in me required parallel_tests test_prefer_config_shell \
- original_AUTOMAKE original_ACLOCAL
-do
+for var in \
+ me \
+ required \
+ use_tap \
+ parallel_tests \
+ am_create_testdir \
+ test_prefer_config_shell \
+ original_AUTOMAKE \
+ original_ACLOCAL \
+; do
if eval "test x\"\$$var\" != x" && env | grep "^$var=" >/dev/null; then
echo "$argv0: variable \`$var' is set in the environment:" \
"this is unsafe" >&2
@@ -81,6 +88,13 @@ do
done
unset var
+# See whether the current test script is expected to use TAP or not.
+# Use a sensible default, while allowing the scripts to override this
+# check.
+if test -z "$using_tap"; then
+ case $argv0 in *.tap) using_tap=yes;; *) using_tap=no;; esac
+fi
+
testsrcdir='@abs_srcdir@'
top_testsrcdir='@abs_top_srcdir@'
testbuilddir='@abs_builddir@'
diff --git a/tests/dejagnu.test b/tests/dejagnu.test
index df385b0..fe994b5 100755
--- a/tests/dejagnu.test
+++ b/tests/dejagnu.test
@@ -25,6 +25,8 @@ AUTOMAKE_OPTIONS = dejagnu
TESTS = frob.test
END
+test x"$parallel_tests" != x"yes" || : > test-driver
+
$ACLOCAL
$AUTOMAKE
diff --git a/tests/depmod-data.test b/tests/depmod-data.test
deleted file mode 100755
index 297c739..0000000
--- a/tests/depmod-data.test
+++ /dev/null
@@ -1,32 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Helper testcase which generate input data for the other tests
-# `*.depmod'. It basically delegates the work to the helper script
-# `depmod-test.sh'.
-
-# Ensure proper definition of $testsrcdir.
-# FIXME: we employ useless quoting below to please maintainer-check.
-. ./defs-static || 'exit' 99
-
-# Sanity check.
-if test x"$testsrcdir" = x; then
- echo "$0: \$testsrcdir not set after sourcing of ./defs-static" >&2
- # FIXME: we employ useless quoting below to please maintainer-check.
- 'exit' 99
-fi
-
-exec $SHELL $testsrcdir/depmod-tests.sh --generate-data
diff --git a/tests/depmod-tests.sh b/tests/depmod-tests.sh
deleted file mode 100755
index a21f618..0000000
--- a/tests/depmod-tests.sh
+++ /dev/null
@@ -1,226 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Driver script to generate and run tests checking that dependency
-# tracking in various flavours works with VPATH builds.
-#
-# This script fulfills a threefold role:
-# 1. It is called to generate a Makefile.am snippet, containing the
-# definition of proper lists of tests.
-# 2. It is called to set up a directory containing some common data
-# files and autotools-generated files used by the aforementioned
-# tests (this is done for speed reasons only).
-# 3. It is called to properly run those tests, one at a time.
-#
-# Examples of reported failures follows ...
-
-# <http://lists.gnu.org/archive/html/automake-patches/2011-04/msg00028.html>
-#
-# Here's the bug: makedepend will prefix VPATH to the object file name,
-# thus the second make will invoke depcomp with object='../../src/foo.o',
-# causing errors such as:
-# touch: cannot touch `../../src/.deps/foo.TPo': No such file or directory
-# makedepend: error: cannot open "../../src/.deps/foo.TPo"
-# ../../depcomp: line 560: ../../src/.deps/foo.TPo: No such file or directory
-
-# <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8473>
-# <http://lists.gnu.org/archive/html/automake-patches/2011-04/msg00079.html>
-#
-# Here's the bug: hp depmode will prefix VPATH to the object file name,
-# thus the second gmake will invoke depcomp with object='../../src/foo.o',
-# causing errors such as (broken on multiple lines for clarity):
-# cpp: "", line 0: error 4066: Cannot create
-# "../../gllib/.deps/nonblocking.TPo" file for
-# "-M../../gllib/.deps/nonblocking.TPo" option.
-# (No such file or directory[errno=2])
-
-# Be more Bourne compatible (snippet copied from `tests/defs').
-DUALCASE=1; export DUALCASE # for MKS sh
-if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
- emulate sh
- NULLCMD=:
- # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
- # is contrary to our usage. Disable this feature.
- alias -g '${1+"$@"}'='"$@"'
- setopt NO_GLOB_SUBST
-else
- case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac
-fi
-
-set -e
-
-# We need this early. It will be overridden when we source ./defs below,
-# which will offer a more proper implementation.
-fatal_ () { echo "$0: $*" >&2; exit 99; }
-
-case $# in
- 0) fatal_ "missing argument";;
- 1) ;;
- *) fatal_ "too many arguments";;
-esac
-
-case $1 in
- --generate-makefile|--generate-data)
- action=`expr x"$1" : x'--\(.*\)'`
- ;;
- depcomp-*.depmod|*/depcomp-*.depmod)
- action=run-test
- depmode=`expr /"$1" : '.*/depcomp-\(.*\)\.depmod'`
- ;;
- *)
- fatal_ "invalid argument '$1'"
- ;;
-esac
-
-# Helper subroutines for creation of input data files.
-
-create_input_data ()
-{
- mkdir src src/sub
-
- unindent > configure.in << 'END'
- AC_INIT([depcomp], [1.0])
- AM_INIT_AUTOMAKE
- AC_CONFIG_FILES([Makefile src/Makefile])
- AC_PROG_CC
- AM_PROG_CC_C_O
- AC_OUTPUT
-END
-
- echo 'SUBDIRS = src' > Makefile.am
-
- unindent > src/Makefile.am << 'END'
- AUTOMAKE_OPTIONS = subdir-objects
- bin_PROGRAMS = foo
- foo_SOURCES = foo.c foo.h sub/subfoo.c
-END
-
- echo 'extern int subfoo (void);' > src/foo.h
-
- unindent > src/foo.c << 'END'
- #include "foo.h"
- int main (void)
- {
- return subfoo ();
- }
-END
-
- # We include subfoo only to be sure that we don't remove too much
- # from the object file name.
- unindent > src/sub/subfoo.c << 'END'
- #include "foo.h"
- int subfoo (void)
- {
- return 0;
- }
-END
-
- $ACLOCAL
- $AUTOCONF
- $AUTOMAKE -a
-
- # Sanity check: make sure the cache variable we force is used
- # by configure.
- grep am_cv_CC_dependencies_compiler_type configure
-
- : > success
-}
-
-# Usage: get_depmodes DEPCOMP-FILE
-get_depmodes ()
-{
- # Keep this in sync with the contents of depend.m4.
- all_depmodes=`sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < "$1" \
- | grep -v '^none$'` \
- && : Turn newlines and tabs into spaces, and strip extra whitespace. \
- && all_depmodes=`echo $all_depmodes` \
- && test -n "$all_depmodes" \
- || fatal_ "can't extract list of valid depmodes from '$1'"
-}
-
-if test x"$action" = x"generate-makefile"; then
- # We must generate a makefile fragment on stdout. It must refer
- # to all tests at once, hence the loop below.
- get_depmodes ../lib/depcomp
- echo '## Generated by depmode-tests.sh. DO NOT EDIT!'
- echo 'depmod_tests ='
- for depmode in $all_depmodes; do
- echo "depmod_tests += depcomp-$depmode.depmod"
- done
- exit 0
-fi
-
-# We'll need the full setup provided by `tests/defs'. Temporarily disable
-# the errexit flag, since the setup code might not be prepared to deal
-# with it. Also pre-set `$me' for `tests/defs', so that different calls
-# to `depmod-tests.sh' won't try to use the same temporary directory.
-# The actual tests require a C compiler, so require it for them; but do
-# not require it when generating data, as it's not needed then.
-if test x"$action" = x"generate-data"; then
- me=depmod-data
-else
- required=cc
- me=depcomp-$depmode
-fi
-set +e
-. ./defs || Exit 99
-set -e
-
-# The directory set up by the `generate-data' action should contain all
-# the files we need. So remove the other files created by ./defs. And
-# check we really are in a temporary `*.dir' directory in the build tree,
-# since the last thing we want is to remove some random user files!
-test -f ../defs-static && test -f ../defs \
- && case `pwd` in *.dir) :;; *) false;; esac \
- || fatal_ "running from the wrong directory"
-rm -f *
-
-if test x"$action" = x"generate-data"; then
- # We must *not* remove the test directory, since its contents must be
- # used by following dependent tests.
- keep_testdirs=yes
- create_input_data
- Exit 0
-fi
-
-get_depmodes "$top_testsrcdir/lib/depcomp"
-case " $all_depmodes " in
- *" $depmode "*) ;;
- *) fatal_ "invalid depmode '$depmode'";;
-esac
-
-### If we are still here, we have to run a test ...
-
-if test ! -f ../depmod-data.dir/success; then
- framework_failure_ "depmod-data.test failure"
-fi
-
-../depmod-data.dir/configure am_cv_CC_dependencies_compiler_type=$depmode
-
-# Do not error out with the first make, as the depmode we've forced might
-# not actually work, but we have overridden the _AM_DEPENDENCIES tests.
-$MAKE || skip_ "forced depmode '$depmode' doesn't work"
-
-# We must clean and rebuild, as the actual error only happens the second
-# time the objects are built because 'depcomp' has silently messed up the
-# .Po files the first time.
-$MAKE clean
-
-$MAKE >out 2>&1 || { cat out; Exit 1; }
-cat out
-grep 'src/[._]deps' out && Exit 1
-
-:
diff --git a/tests/depmod.tap b/tests/depmod.tap
new file mode 100755
index 0000000..82fd844
--- /dev/null
+++ b/tests/depmod.tap
@@ -0,0 +1,174 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that dependency tracking in various flavours works with VPATH
+# builds. Examples of reported failures follows ...
+
+# <http://lists.gnu.org/archive/html/automake-patches/2011-04/msg00028.html>
+#
+# Here's the bug: makedepend will prefix VPATH to the object file name,
+# thus the second make will invoke depcomp with object='../../src/foo.o',
+# causing errors such as:
+# touch: cannot touch `../../src/.deps/foo.TPo': No such file or directory
+# makedepend: error: cannot open "../../src/.deps/foo.TPo"
+# ../../depcomp: line 560: ../../src/.deps/foo.TPo: No such file or directory
+
+# <http://debbugs.gnu.org/cgi/bugreport.cgi?bug=8473>
+# <http://lists.gnu.org/archive/html/automake-patches/2011-04/msg00079.html>
+#
+# Here's the bug: hp depmode will prefix VPATH to the object file name,
+# thus the second gmake will invoke depcomp with object='../../src/foo.o',
+# causing errors such as (broken on multiple lines for clarity):
+# cpp: "", line 0: error 4066: Cannot create
+# "../../gllib/.deps/nonblocking.TPo" file for
+# "-M../../gllib/.deps/nonblocking.TPo" option.
+# (No such file or directory[errno=2])
+#
+
+# <http://lists.gnu.org/archive/html/automake-patches/2011-04/msg00140.html>
+# <http://lists.gnu.org/archive/html/automake-patches/2011-05/msg00019.html>
+#
+# A partial failure of an earlier version of this test; some bad
+# post-processing of the `*.Po' files led to the following broken
+# contents of `src/sub/.deps/subfoo.Po':
+# > sub/subfoo.o: ../../depmod-data.dir/src/sub/subfoo.c \
+# > ../../depmod-data.dir/src/foo.h
+# > ../../depmod-data.dir/src/sub/subfoo.c \:
+# > ../../depmod-data.dir/src/foo.h:
+# which caused make to die with an error like:
+# "sub/.deps/subfoo.Po:3: *** missing separator. Stop."
+
+required=cc
+. ./defs || Exit 99
+
+# Per each valid depmode three kinds of VPATH, and per each of them
+# a test on "./configure" and one on "make & remake"; so there are
+# 17 * 3 * 2 = 102.
+plan_ 102
+
+ocwd=`pwd` || fatal_ "cannot get current working directory"
+
+# Keep this in sync with the contents of depend.m4.
+all_depmodes=`\
+ sed -n 's/^#*\([a-zA-Z0-9]*\))$/\1/p' < "$top_testsrcdir/lib/depcomp" \
+ | grep -v '^none$'` \
+ && : Turn newlines and tabs into spaces, and strip extra whitespace. \
+ && all_depmodes=`echo $all_depmodes` \
+ && test -n "$all_depmodes" \
+ || fatal_ "can't extract list of valid depmodes"
+
+srctree=depmod-1.0
+mkdir $srctree
+mv missing install-sh depcomp configure.in $srctree
+cd $srctree
+
+mkdir src src/sub
+
+cat >> configure.in << 'END'
+AC_CONFIG_FILES([src/Makefile])
+AC_PROG_CC
+AM_PROG_CC_C_O
+AC_OUTPUT
+END
+
+echo 'SUBDIRS = src' > Makefile.am
+
+cat > src/Makefile.am << 'END'
+AUTOMAKE_OPTIONS = subdir-objects
+bin_PROGRAMS = foo
+foo_SOURCES = foo.c foo.h sub/subfoo.c
+END
+
+echo 'extern int subfoo (void);' > src/foo.h
+
+cat > src/foo.c << 'END'
+#include "foo.h"
+int main (void)
+{
+ return subfoo ();
+}
+END
+
+# We include subfoo only to be sure that we don't remove too much
+# from the object file name.
+cat > src/sub/subfoo.c << 'END'
+#include "foo.h"
+int subfoo (void)
+{
+ return 0;
+}
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+# Sanity check: make sure the cache variable we force is used
+# by configure.
+grep am_cv_CC_dependencies_compiler_type configure \
+ || fatal_ "configure lacks required cache variable"
+
+cd "$ocwd" || fatal_ "cannot chdir back to top directory"
+
+longpath=this-is/a-path/which-have/quite-a/long_long_name
+$srctree/install-sh -d $longpath || fatal_ "cannot create long path"
+cp -pR $srctree/* $longpath || fatal_ "cannot populate long path"
+cp -pR $srctree/* . || fatal_ "cannot populate top-level directory"
+
+for vpath in simple long absolute; do
+ for depmode in $all_depmodes; do
+ case $vpath in
+ simple)
+ srcdir=..;;
+ long)
+ srcdir=../$longpath;;
+ absolute)
+ srcdir=$ocwd;;
+ *)
+ fatal_ "bad \$vpath value '$vpath'";;
+ esac
+
+ pfx="$depmode [$vpath VPATH]"
+
+ mkdir $depmode-$vpath.d
+ cd $depmode-$vpath.d
+
+ command_ok_ "$pfx configure" \
+ "$srcdir"/configure am_cv_CC_dependencies_compiler_type=$depmode
+
+ # Do not fail with the first call to make fails, as the depmode
+ # we've forced might not actually work, but we have overridden the
+ # _AM_DEPENDENCIES tests.
+ if $MAKE; then
+ # We must clean and rebuild, as the actual error only happens the
+ # second time the objects are built because 'depcomp' has silently
+ # messed up the .Po files the first time.
+ r=ok
+ $MAKE clean \
+ && { e=0; $MAKE >output 2>&1 || e=1; cat output; test $e -eq 0; } \
+ && { if grep 'src/[._]deps' output; then false; else :; fi; } \
+ || r='not ok'
+ result_ "$r" "$pfx make & remake"
+ else
+ skip_ -r "can't force depmode" "$pfx make & remake"
+ fi
+
+ cd "$ocwd" || fatal_ "cannot chdir back to top directory"
+
+ done
+done
+
+:
diff --git a/tests/exeext4.test b/tests/exeext4.test
index e78a508..1ee8016 100755
--- a/tests/exeext4.test
+++ b/tests/exeext4.test
@@ -51,7 +51,7 @@ END
$ACLOCAL
$AUTOCONF
-$AUTOMAKE
+$AUTOMAKE --add-missing --copy
./configure
$MAKE print-bin > output
cat output
diff --git a/tests/extract-testsuite-summary b/tests/extract-testsuite-summary
new file mode 100644
index 0000000..a3a0d87
--- /dev/null
+++ b/tests/extract-testsuite-summary
@@ -0,0 +1,15 @@
+#! /usr/bin/env perl
+# Extract the testsuite summary generated by the parallel-tests harness
+# from the output of "make check".
+
+use warnings FATAL => 'all';
+use strict;
+
+my $br = '=' x 76;
+my @sections = ('');
+while (<>)
+ {
+ push @sections, $_, '' if /$br/;
+ $sections[-1] .= $_ if !/$br/;
+ }
+print @sections[1..$#sections-1];
diff --git a/tests/getopt.test b/tests/getopt.test
index 9889e3d..41aca0c 100755
--- a/tests/getopt.test
+++ b/tests/getopt.test
@@ -24,7 +24,7 @@ AUTOMAKE_fails -Wnonexistent
grep ':.*nonexistent' stderr
# ... but this should not.
-AUTOMAKE_run 0 -Wnonexistent --help
+AUTOMAKE_run -Wnonexistent --help
grep ':.*nonexistent' stderr && Exit 1
@@ -33,7 +33,7 @@ AUTOMAKE_fails --nonexistent
grep ':.*nonexistent' stderr
# ... but this should not.
-AUTOMAKE_run 0 --nonexistent --help
+AUTOMAKE_run --nonexistent --help
grep ':.*nonexistent' stderr && Exit 1
:
diff --git a/tests/instspc-data.test b/tests/instspc-data.test
deleted file mode 100755
index 7b222ff..0000000
--- a/tests/instspc-data.test
+++ /dev/null
@@ -1,31 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Helper testcase which generate input data for the `*.instspc' tests.
-# It basically delegates the work to the driver script `instspc-test.sh'.
-
-# Ensure proper definition of $testsrcdir.
-# FIXME: we employ useless quoting below to please maintainer-check.
-. ./defs-static || 'exit' 99
-
-# Sanity check.
-if test x"$testsrcdir" = x; then
- echo "$0: \$testsrcdir not set after sourcing of ./defs-static" >&2
- # FIXME: we employ useless quoting below to please maintainer-check.
- 'exit' 99
-fi
-
-exec $SHELL $testsrcdir/instspc-tests.sh --generate-data
diff --git a/tests/instspc-tests.sh b/tests/instspc-tests.sh
deleted file mode 100755
index 9c6c968..0000000
--- a/tests/instspc-tests.sh
+++ /dev/null
@@ -1,343 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2010, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-#
-# Driver script to run tests checking that building from, or installing
-# to, directories with shell metacharacters succeed.
-#
-# Original report from James Amundson about file names with spaces.
-# Other characters added by Paul Eggert.
-#
-# This script fulfills a threefold role:
-# 1. It is called to generate a Makefile.am snippet, containing the
-# definition of proper lists of tests.
-# 2. It is called to set up a directory containing some common data
-# files and autotools-generated files used by the aforementioned
-# tests (this is done for speed reasons only).
-# 3. It is called to properly run those tests, one at a time.
-#
-
-# Be more Bourne compatible (snippet copied from `tests/defs').
-DUALCASE=1; export DUALCASE # for MKS sh
-if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then
- emulate sh
- NULLCMD=:
- # Pre-4.2 versions of Zsh do word splitting on ${1+"$@"}, which
- # is contrary to our usage. Disable this feature.
- alias -g '${1+"$@"}'='"$@"'
- setopt NO_GLOB_SUBST
-else
- case `(set -o) 2>/dev/null` in *posix*) set -o posix;; esac
-fi
-
-set -e
-
-# We need this early. It will be overridden when we source ./defs below,
-# which will offer a more proper implementation.
-fatal_ () { echo "$0: $*" >&2; exit 99; }
-
-case $# in
- 0) fatal_ "missing argument";;
- 1) ;;
- *) fatal_ "too many arguments";;
-esac
-
-case $1 in
- --generate-makefile|--generate-data)
- instspc_action=`expr x"$1" : x'--\(.*\)'`
- ;;
- build-*.instspc|*/build-*.instspc)
- instspc_action=test-build
- instspc_test_name=`expr /"$1" : '.*/build-\(.*\)\.instspc'`
- ;;
- install-*.instspc|*/install-*.instspc)
- instspc_action=test-install
- instspc_test_name=`expr /"$1" : '.*/install-\(.*\)\.instspc'`
- ;;
- *)
- fatal_ "invalid argument '$1'"
- ;;
-esac
-
-# Helper subroutine for test data definition.
-# Usage: define_problematic_string NAME STRING
-define_problematic_string ()
-{
- tst=$1
- shift
- eval "instspc__$tst=\$1" \
- || fatal_ "define_problematic_string: bad argument: '$tst'"
- shift
- instspc_names_list="$instspc_names_list $tst"
- # Some of the "problematic" characters cannot be used in the name of
- # a build or install directory on a POSIX host. These lists should
- # be empty, but are not due to limitations in Autoconf, Automake, Make,
- # M4, or the shell.
- case " $* " in *' fail-build '*|*' build-fail '*)
- instspc_xfail_builds_list="$instspc_xfail_builds_list $tst";;
- esac
- case " $* " in *' fail-install '*|*' install-fail '*)
- instspc_xfail_installs_list="$instspc_xfail_installs_list $tst";;
- esac
-}
-
-# Helper subroutines for creation of input data files.
-
-create_input_data ()
-{
- mkdir sub
-
- unindent > configure.in << 'EOF'
- AC_INIT([instspc], [1.0])
- AM_INIT_AUTOMAKE
- AC_CONFIG_FILES([Makefile])
- AC_PROG_CC
- AC_PROG_RANLIB
- AC_OUTPUT
-EOF
-
- : > sub/base.h
- : > sub/nobase.h
- : > sub/base.dat
- : > sub/nobase.dat
- : > sub/base.sh
- : > sub/nobase.sh
-
- unindent > source.c << 'EOF'
- int
- main (int argc, char **argv)
- {
- return 0;
- }
-EOF
-
- unindent > Makefile.am << 'EOF'
- foodir = $(prefix)/foo
- fooexecdir = $(prefix)/foo
-
- foo_HEADERS = sub/base.h
- nobase_foo_HEADERS = sub/nobase.h
-
- dist_foo_DATA = sub/base.dat
- nobase_dist_foo_DATA = sub/nobase.dat
-
- dist_fooexec_SCRIPTS = sub/base.sh
- nobase_dist_fooexec_SCRIPTS = sub/nobase.sh
-
- fooexec_PROGRAMS = sub/base
- nobase_fooexec_PROGRAMS = sub/nobase
- sub_base_SOURCES = source.c
- sub_nobase_SOURCES = source.c
-
- fooexec_LIBRARIES = sub/libbase.a
- nobase_fooexec_LIBRARIES = sub/libnobase.a
- sub_libbase_a_SOURCES = source.c
- sub_libnobase_a_SOURCES = source.c
-
- .PHONY: test-install-sep
- test-install-sep: install
- test -f '$(DESTDIR)/$(file)-prefix/foo/sub/nobase.h'
- test ! -f '$(DESTDIR)/$(file)-prefix/foo/nobase.h'
- test -f '$(DESTDIR)/$(file)-prefix/foo/base.h'
- test -f '$(DESTDIR)/$(file)-prefix/foo/sub/nobase.dat'
- test ! -f '$(DESTDIR)/$(file)-prefix/foo/nobase.dat'
- test -f '$(DESTDIR)/$(file)-prefix/foo/base.dat'
- test -f '$(DESTDIR)/$(file)-prefix/foo/sub/nobase.sh'
- test ! -f '$(DESTDIR)/$(file)-prefix/foo/nobase.sh'
- test -f '$(DESTDIR)/$(file)-prefix/foo/base.sh'
- test -f '$(DESTDIR)/$(file)-prefix/foo/sub/nobase$(EXEEXT)'
- test ! -f '$(DESTDIR)/$(file)-prefix/foo/nobase$(EXEEXT)'
- test -f '$(DESTDIR)/$(file)-prefix/foo/base$(EXEEXT)'
- test -f '$(DESTDIR)/$(file)-prefix/foo/sub/libnobase.a'
- test ! -f '$(DESTDIR)/$(file)-prefix/foo/libnobase.a'
- test -f '$(DESTDIR)/$(file)-prefix/foo/libbase.a'
-EOF
-
- $ACLOCAL
- $AUTOCONF
- $AUTOMAKE -a
-
- : > success
-}
-
-# Be sure to avoid interferences from the environment.
-instspc_names_list=''
-instspc_xfail_builds_list=''
-instspc_xfail_installs_list=''
-
-
-# ================= #
-# Test data begin #
-# ----------------- #
-
-# Some control characters that are white space.
-bs='' # back space
-cr='
' # carriage return
-ff=' ' # form feed
-ht=' ' # horizontal tab
-lf='
-' # line feed (aka newline)
-
-# Hack to save typing and make code visually clearer.
-def=define_problematic_string
-
-$def squote \' fail-build fail-install
-$def dquote '"' fail-build fail-install
-$def bquote '`' fail-build fail-install
-$def sharp '#' fail-build fail-install
-$def dollar '$' fail-build fail-install
-$def bang '!'
-$def bslash '\' fail-build
-$def ampersand '&' fail-build
-$def percent '%'
-$def leftpar '('
-$def rightpar ')'
-$def pipe '|'
-$def caret '^'
-$def tilde '~'
-$def qmark '?'
-$def star '*'
-$def plus '+'
-$def minus '-'
-$def comma ','
-$def colon ':'
-$def semicol ';'
-$def equal '='
-$def less '<'
-$def more '>'
-$def at '@'
-$def lqbrack '['
-$def rqbrack ']'
-$def lcbrack '{'
-$def rcbrack '}'
-$def space ' '
-$def tab "$ht"
-$def linefeed "$lf" fail-build fail-install
-$def backspace "$bs"
-$def formfeed "$ff"
-$def carriageret "$cr"
-$def quadrigraph0 '@&t@' fail-build
-$def quadrigraph1 '@<:@'
-$def quadrigraph2 '@:>@'
-$def quadrigraph3 '@S|@'
-$def quadrigraph4 '@%:@'
-$def a_b 'a b'
-$def a__b 'a b'
-$def a_lf_b "a${lf}b" fail-build fail-install
-$def dotdotdot '...'
-$def dosdrive 'a:'
-$def miscglob1 '?[a-z]*'
-$def miscglob2 '.*?[0-9]'
-
-unset def
-
-# --------------- #
-# Test data end #
-# =============== #
-
-
-if test x"$instspc_action" = x"generate-makefile"; then
- # We must generate a makefile fragment on stdout. It must refer
- # to all tests at once, hence the loop below.
- echo '## Generated by instspc-tests.sh. DO NOT EDIT!'
- echo 'instspc_tests ='
- echo 'instspc_xfail_tests ='
- for test_name in $instspc_names_list; do
- echo "instspc_tests += build-$test_name.instspc"
- echo "instspc_tests += install-$test_name.instspc"
- done
- for test_name in $instspc_xfail_builds_list; do
- echo "instspc_xfail_tests += build-$test_name.instspc"
- done
- for test_name in $instspc_xfail_installs_list; do
- echo "instspc_xfail_tests += install-$test_name.instspc"
- done
- exit 0
-fi
-
-# We'll need the full setup provided by `tests/defs'. Temporarily disable
-# the errexit flag, since the setup code might not be prepared to deal
-# with it. Also pre-set `$me' for `tests/defs', so that different calls
-# to `instspc-tests.sh' won't try to use the same temporary directory.
-# The actual tests require a C compiler, so require it for them; but do
-# not require it when generating data, as it's not needed then.
-if test x"$instspc_action" = x"generate-data"; then
- me=instspc-data
-else
- required=cc
- me=$instspc_action-$instspc_test_name
-fi
-set +e
-. ./defs || Exit 99
-set -e
-
-# The directory set up by the `generate-data' action should contain all
-# the files we need. So remove the other files created by ./defs. And
-# check we really are in a temporary `*.dir' directory in the build tree,
-# since the last thing we want is to remove some random user files!
-test -f ../defs-static && test -f ../defs \
- && case `pwd` in *.dir) :;; *) false;; esac \
- || fatal_ "running from the wrong directory"
-rm -f *
-
-if test x"$instspc_action" = x"generate-data"; then
- # We must *not* remove the test directory, since its contents must be
- # used by following dependent tests.
- keep_testdirs=yes
- create_input_data
- Exit 0
-fi
-
-### If we are still here, we have to run a test ...
-
-eval "instspc_test_string=\${instspc__$instspc_test_name}" \
- && test x"$instspc_test_string" != x \
- || fatal_ "invalid test name: '$instspc_test_name'"
-
-if test ! -f ../instspc-data.dir/success; then
- framework_failure_ "instspc-data.test failure"
-fi
-
-# Skip if this system doesn't support these characters in file names.
-mkdir "./$instspc_test_string" || Exit 77
-
-case $instspc_action in
- test-build)
- dest=`pwd`/_dest
- relbuilddir=../..
- cd "./$instspc_test_string"
- ;;
- test-install)
- dest=`pwd`/$instspc_test_string
- relbuilddir=..
- ;;
- *)
- fatal_ "invalid action '$instspc_action'"
- ;;
-esac
-
-$relbuilddir/instspc-data.dir/configure \
- --prefix "/$instspc_test_string-prefix"
-$MAKE
-# Some make implementations eliminate leading and trailing whitespace
-# from macros passed on the command line, and some eliminate leading
-# whitespace from macros set from environment variables, so prepend
-# './' and use the latter here.
-# Tru64 sh -e needs '|| Exit' in order to work correctly.
-DESTDIR="$dest" file="./$instspc_test_string" $MAKE -e test-install-sep \
- || Exit 1
-
-:
diff --git a/tests/instspc.tap b/tests/instspc.tap
new file mode 100755
index 0000000..dc52299
--- /dev/null
+++ b/tests/instspc.tap
@@ -0,0 +1,278 @@
+#! /bin/sh
+# Copyright (C) 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that building from, or installing to, directories with shell
+# metacharacters succeed.
+# Original report from James Amundson about file names with spaces.
+# Other characters added by Paul Eggert.
+
+. ./defs || Exit 99
+
+plan_ 94 # Two tests per "problematic string".
+
+# Usage: is_in_list ITEM [LIST...]
+is_in_list ()
+{
+ item=$1; shift;
+ case " $* " in
+ *[\ \ ]"$item"[\ \ ]*) return 0;;
+ *) return 1;;
+ esac
+}
+
+# Helper subroutine for test data definition.
+# Usage: define_problematic_string NAME STRING
+define_problematic_string ()
+{
+ tst=$1; shift
+ eval "instspc__$tst=\$1" \
+ || fatal_ "define_problematic_string: bad argument: '$tst'"
+ shift
+ test_names_list="$test_names_list $tst"
+ # Some of the "problematic" characters cannot be used in the name of
+ # a build or install directory on a POSIX host. These lists should
+ # be empty, but are not due to limitations in Autoconf, Automake, Make,
+ # M4, or the shell.
+ if is_in_list fail-builddir "$@"; then
+ builddir_xfails="$builddir_xfails $tst"
+ fi
+ if is_in_list fail-destdir "$@"; then
+ destdir_xfails="$destdir_xfails $tst"
+ fi
+}
+
+# Be sure to avoid interferences from the environment.
+test_names_list=''
+builddir_xfails=''
+destdir_xfails=''
+
+expected_to_fail ()
+{
+ case $1 in
+ build) is_in_list "$2" $builddir_xfails;;
+ dest) is_in_list "$2" $destdir_xfails;;
+ *) fatal_ "incorrect 'expected_to_fail' usage";;
+ esac
+}
+
+# Helper subroutines for creation of input data files.
+create_input_data ()
+{
+ mkdir sub
+
+ unindent >> configure.in << 'EOF'
+ AC_PROG_CC
+ AC_PROG_RANLIB
+ AC_OUTPUT
+EOF
+
+ : > sub/base.h
+ : > sub/nobase.h
+ : > sub/base.dat
+ : > sub/nobase.dat
+ : > sub/base.sh
+ : > sub/nobase.sh
+
+ unindent > source.c << 'EOF'
+ int
+ main (int argc, char **argv)
+ {
+ return 0;
+ }
+EOF
+
+ unindent > Makefile.am << 'EOF'
+ foodir = $(prefix)/foo
+ fooexecdir = $(prefix)/foo
+
+ foo_HEADERS = sub/base.h
+ nobase_foo_HEADERS = sub/nobase.h
+
+ dist_foo_DATA = sub/base.dat
+ nobase_dist_foo_DATA = sub/nobase.dat
+
+ dist_fooexec_SCRIPTS = sub/base.sh
+ nobase_dist_fooexec_SCRIPTS = sub/nobase.sh
+
+ fooexec_PROGRAMS = sub/base
+ nobase_fooexec_PROGRAMS = sub/nobase
+ sub_base_SOURCES = source.c
+ sub_nobase_SOURCES = source.c
+
+ fooexec_LIBRARIES = sub/libbase.a
+ nobase_fooexec_LIBRARIES = sub/libnobase.a
+ sub_libbase_a_SOURCES = source.c
+ sub_libnobase_a_SOURCES = source.c
+
+ .PHONY: test-inst
+ test-inst: install
+ test -f '$(DESTDIR)/$(file)-prefix/foo/sub/nobase.h'
+ test ! -f '$(DESTDIR)/$(file)-prefix/foo/nobase.h'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/base.h'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/sub/nobase.dat'
+ test ! -f '$(DESTDIR)/$(file)-prefix/foo/nobase.dat'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/base.dat'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/sub/nobase.sh'
+ test ! -f '$(DESTDIR)/$(file)-prefix/foo/nobase.sh'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/base.sh'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/sub/nobase$(EXEEXT)'
+ test ! -f '$(DESTDIR)/$(file)-prefix/foo/nobase$(EXEEXT)'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/base$(EXEEXT)'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/sub/libnobase.a'
+ test ! -f '$(DESTDIR)/$(file)-prefix/foo/libnobase.a'
+ test -f '$(DESTDIR)/$(file)-prefix/foo/libbase.a'
+EOF
+
+ $ACLOCAL
+ $AUTOCONF
+ $AUTOMAKE -a
+}
+
+# ================= #
+# Test data begin #
+# ----------------- #
+
+# Some control characters that are white space.
+bs='' # back space
+cr='
' # carriage return
+ff=' ' # form feed
+ht=' ' # horizontal tab
+lf='
+' # line feed (aka newline)
+
+# Hack to save typing and make code visually clearer.
+def=define_problematic_string
+
+$def squote \' fail-builddir fail-destdir
+$def dquote '"' fail-builddir fail-destdir
+$def bquote '`' fail-builddir fail-destdir
+$def sharp '#' fail-builddir fail-destdir
+$def dollar '$' fail-builddir fail-destdir
+$def bang '!'
+$def bslash '\' fail-builddir
+$def ampersand '&' fail-builddir
+$def percent '%'
+$def leftpar '('
+$def rightpar ')'
+$def pipe '|'
+$def caret '^'
+$def tilde '~'
+$def qmark '?'
+$def star '*'
+$def plus '+'
+$def minus '-'
+$def comma ','
+$def colon ':'
+$def semicol ';'
+$def equal '='
+$def less '<'
+$def more '>'
+$def at '@'
+$def lqbrack '['
+$def rqbrack ']'
+$def lcbrack '{'
+$def rcbrack '}'
+$def space ' '
+$def tab "$ht"
+$def linefeed "$lf" fail-builddir fail-destdir
+$def backspace "$bs"
+$def formfeed "$ff"
+$def carriageret "$cr"
+$def quadrigraph0 '@&t@' fail-builddir
+$def quadrigraph1 '@<:@'
+$def quadrigraph2 '@:>@'
+$def quadrigraph3 '@S|@'
+$def quadrigraph4 '@%:@'
+$def a_b 'a b'
+$def a__b 'a b'
+$def a_lf_b "a${lf}b" fail-builddir fail-destdir
+$def dotdotdot '...'
+$def dosdrive 'a:'
+$def miscglob1 '?[a-z]*'
+$def miscglob2 '.*?[0-9]'
+
+unset def
+
+# --------------- #
+# Test data end #
+# =============== #
+
+ocwd=`pwd` || fatal_ "cannot get current working directory"
+
+create_input_data
+
+for test_name in $test_names_list; do
+
+ eval "test_string=\${instspc__$test_name}" \
+ && test x"$test_string" != x \
+ || fatal_ "invalid test name: '$test_name'"
+
+ # Skip the next checks if this system doesn't support the required
+ # characters in file names.
+
+ mkdir "./$test_string" || \
+ skip_ -r "mkdir failed" "$test_name"
+
+ # Where are the "weird" characters going to be used, in $(builddir)
+ # or in $(DESTDIR)? They are always going to be used in $(prefix)
+ # though; should we maybe separate this into a dedicated check?
+ for where in build dest; do
+
+ case $where in
+ build)
+ build=./$test_string
+ dest=$ocwd/dest-$test_name
+ ;;
+ dest)
+ build=build-$test_name
+ dest=$ocwd/$test_string
+ mkdir "$build" || fatal_ "cannot create '$build'"
+ ;;
+ *)
+ fatal_ "invalid where '$where'"
+ ;;
+ esac
+
+ cd "$build" || fatal_ "cannot chdir into '$build'"
+
+ # Some make implementations eliminate leading and trailing whitespace
+ # from macros passed on the command line, and some eliminate leading
+ # whitespace from macros set from environment variables, so prepend
+ # './' and use the latter here.
+ r=ok
+ ../configure --prefix "/$test_string-prefix" \
+ && $MAKE all \
+ && DESTDIR="$dest" file="./$test_string" $MAKE -e test-inst \
+ || r='not ok'
+
+ description="$test_name in ${where}dir"
+ if expected_to_fail "$where" "$test_name"; then
+ directive=TODO
+ reason="long-standing limitation"
+ else
+ directive=
+ reason=
+ fi
+ # Test case outcome is here.
+ result_ "$r" -D "$directive" -r "$reason" -- "$description"
+
+ cd "$ocwd" || fatal_ "cannot chdir back to test directory"
+
+ done # $instspc_action
+
+done # $test_name
+
+:
diff --git a/tests/license2.test b/tests/license2.test
index 34bfdf8..ca6a249 100755
--- a/tests/license2.test
+++ b/tests/license2.test
@@ -32,7 +32,7 @@ $ACLOCAL
AUTOMAKE_fails
grep 'COPYING' stderr
-AUTOMAKE_run 0 --add-missing
+AUTOMAKE_run --add-missing
grep 'COPYING' stderr
grep 'GNU General Public License' stderr
grep 'Consider adding.*version control' stderr
diff --git a/tests/maken3.test b/tests/maken3.test
index 5a77f1a..eb54d35 100755
--- a/tests/maken3.test
+++ b/tests/maken3.test
@@ -149,7 +149,7 @@ check_targets ()
done
}
-$AUTOMAKE -Wno-override
+$AUTOMAKE -a -Wno-override
./configure
check_targets || Exit 1
diff --git a/tests/maken4.test b/tests/maken4.test
index bef1389..bdecb84 100755
--- a/tests/maken4.test
+++ b/tests/maken4.test
@@ -169,7 +169,7 @@ check_targets ()
done
}
-$AUTOMAKE -Wno-override
+$AUTOMAKE -a -Wno-override
./configure
check_targets || Exit 1
diff --git a/tests/parallel-am.test b/tests/parallel-am.test
index 0182b44..453f7a5 100755
--- a/tests/parallel-am.test
+++ b/tests/parallel-am.test
@@ -81,7 +81,7 @@ $ACLOCAL
# Generate expected output using the non-threaded code.
unset AUTOMAKE_JOBS || :
-AUTOMAKE_run 0 --add-missing
+AUTOMAKE_run --add-missing
mv stderr expected
Makefile_ins=`find . -name Makefile.in`
for file in $Makefile_ins; do
@@ -93,7 +93,7 @@ export AUTOMAKE_JOBS
for run in 1 2 3 4 5 6 7; do
rm -f build-aux/* sub*/Makefile.in
- AUTOMAKE_run 0 --add-missing
+ AUTOMAKE_run --add-missing
diff stderr expected
for file in $Makefile_ins; do
diff $file $file.exp
diff --git a/tests/parallel-am3.test b/tests/parallel-am3.test
index 6db3ca7..732bd41 100755
--- a/tests/parallel-am3.test
+++ b/tests/parallel-am3.test
@@ -57,7 +57,7 @@ $ACLOCAL
# Generate expected output using the non-threaded code.
unset AUTOMAKE_JOBS || :
-AUTOMAKE_run 0 --add-missing
+AUTOMAKE_run --add-missing
mv stderr expected
mv Makefile.in Makefile.in.exp
@@ -66,7 +66,7 @@ export AUTOMAKE_JOBS
for run in 1 2 3 4 5 6 7; do
rm -f build-aux/* sub*/Makefile.in
- AUTOMAKE_run 0 --add-missing
+ AUTOMAKE_run --add-missing
diff stderr expected
diff Makefile.in Makefile.in.exp
done
diff --git a/tests/parallel-tests-am_tests_environment.test
b/tests/parallel-tests-am_tests_environment.test
deleted file mode 100755
index eff02a6..0000000
--- a/tests/parallel-tests-am_tests_environment.test
+++ /dev/null
@@ -1,95 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# parallel-tests: check AM_TESTS_ENVIRONMENT support, and its
-# interactions with TESTS_ENVIRONMENT.
-
-parallel_tests=yes
-. ./defs || Exit 1
-
-cat >> configure.in << 'END'
-AC_PROG_MKDIR_P
-AC_OUTPUT
-END
-
-cat > Makefile.am << 'END'
-TEST_EXTENSIONS = .sh .test
-TESTS = foo.test bar.sh
-SH_LOG_COMPILER = sh
-AM_TESTS_ENVIRONMENT = $(MKDIR_P) quux.dir; exec 9>&2; fd=9; export fd;
-EXTRA_DIST = $(TESTS)
-END
-
-cat > foo.test << 'END'
-#! /bin/sh
-echo "FOO='$FOO'"
-echo "BAR='$BAR'"
-test x"$FOO" = x"$BAR"
-END
-chmod a+x foo.test
-
-cat > bar.sh << 'END'
-: ${fd=2}
-eval "echo 'diagnostic message from test bar' >&${fd}"
-END
-
-minicheck ()
-{
- cat foo.log
- cat bar.log
- test -d quux.dir
-}
-
-miniclean ()
-{
- rmdir quux.dir
- rm -f foo.log bar.log test-suite.log
-}
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE -a
-
-./configure
-
-FOO=1; export FOO
-
-TESTS_ENVIRONMENT='BAR=1' $MAKE check || { cat test-suite.log; Exit 1; }
-minicheck
-miniclean
-TESTS_ENVIRONMENT='BAR=2' $MAKE check && { cat test-suite.log; Exit 1; }
-minicheck
-miniclean
-
-BAR=$FOO; export BAR
-
-$MAKE check 2>stderr || { cat stderr >&2; Exit1; }
-cat stderr >&2
-minicheck
-grep 'diagnostic message from test bar' stderr
-grep 'diagnostic message from test bar' bar.log && Exit 1
-miniclean
-
-TESTS_ENVIRONMENT='fd=2' $MAKE check 2>stderr || { cat stderr >&2; Exit1; }
-cat stderr >&2
-minicheck
-grep 'diagnostic message from test bar' stderr && Exit 1
-grep 'diagnostic message from test bar' bar.log
-miniclean
-
-$MAKE distcheck
-
-:
diff --git a/tests/parallel-tests-console-output.test
b/tests/parallel-tests-console-output.test
new file mode 100755
index 0000000..8122bfd
--- /dev/null
+++ b/tests/parallel-tests-console-output.test
@@ -0,0 +1,104 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests: some checks on console output about testsuite
+# progress.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+XFAIL_TESTS = sub/xpass.test xfail.test error.test
+TESTS = $(XFAIL_TESTS) fail.test pass.test a/b/skip.test sub/error2.test
+pass.log: fail.log
+error.log: pass.log
+sub/xpass.log: error.log
+sub/error2.log: xfail.log
+a/b/skip.log: sub/error2.log
+END
+
+cat > exp <<'END'
+FAIL: fail.test
+PASS: pass.test
+ERROR: error.test
+XPASS: sub/xpass.test
+XFAIL: xfail.test
+ERROR: sub/error2.test
+SKIP: a/b/skip.test
+END
+
+mkdir sub a a/b
+
+cat > pass.test << 'END'
+#!/bin/sh
+exit 0
+END
+cp pass.test sub/xpass.test
+
+cat > fail.test << 'END'
+#!/bin/sh
+exit 1
+END
+
+cat > xfail.test << 'END'
+#!/bin/sh
+# The sleep should ensure expected execution order of tests
+# even when make is run in parallel mode.
+# FIXME: quotes below required by maintainer-check.
+sleep '10'
+exit 1
+END
+
+cat > error.test << 'END'
+#!/bin/sh
+exit 99
+END
+cp error.test sub/error2.test
+
+cat > a/b/skip.test << 'END'
+#!/bin/sh
+exit 77
+END
+
+chmod a+x pass.test fail.test xfail.test sub/xpass.test \
+ a/b/skip.test error.test sub/error2.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+ $srcdir/configure
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ LC_ALL=C grep '^[A-Z][A-Z]*:' stdout > got
+ cat got
+ diff $srcdir/exp got
+ cd $srcdir
+done
+
+:
diff --git a/tests/parallel-tests-driver-install.test
b/tests/parallel-tests-driver-install.test
new file mode 100755
index 0000000..10455fb
--- /dev/null
+++ b/tests/parallel-tests-driver-install.test
@@ -0,0 +1,93 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that auxiliary script 'test-driver' gets automatically installed
+# in the correct directory by 'parallel-tests' option.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+: Try first with parallel-tests defined in AM_INIT_AUTOMAKE.
+
+mkdir am-init-automake
+cd am-init-automake
+
+cat > configure.in <<END
+AC_INIT([$me], [1.0])
+AC_CONFIG_AUX_DIR([my_aux_dir])
+AM_INIT_AUTOMAKE([parallel-tests])
+AC_CONFIG_FILES([Makefile sub/Makefile])
+AC_OUTPUT
+END
+
+mkdir sub my_aux_dir
+
+cat > Makefile.am <<END
+SUBDIRS = sub
+TESTS = foo.test
+END
+
+cat > sub/Makefile.am <<END
+TESTS = bar.test
+END
+
+$ACLOCAL
+$AUTOMAKE -a 2>stderr || { cat stderr >&2; Exit 1; }
+cat stderr >&2
+
+ls -l . sub my_aux_dir # For debugging.
+test -f my_aux_dir/test-driver
+test ! -r test-driver
+test ! -r sub/test-driver
+
+grep '^configure\.in:3:.*installing.*my_aux_dir/test-driver' stderr
+
+cd ..
+
+: Now try with parallel-tests defined in AUTOMAKE_OPTIONS.
+
+mkdir automake-options
+cd automake-options
+
+cat > configure.in <<END
+AC_INIT([$me], [1.0])
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE
+AC_CONFIG_FILES([dir/GNUmakefile])
+AC_OUTPUT
+END
+
+mkdir build-aux dir
+
+cat > dir/GNUmakefile.am <<END
+TESTS = foo.test
+AUTOMAKE_OPTIONS = parallel-tests
+TESTS += bar.test
+END
+
+$ACLOCAL
+$AUTOMAKE --add-missing --copy dir/GNUmakefile 2>stderr \
+ || { cat stderr >&2; Exit 1; }
+cat stderr >&2
+
+ls -l . dir build-aux # For debugging.
+test -f build-aux/test-driver
+test ! -r test-driver
+test ! -r dir/test-driver
+
+grep '^dir/GNUmakefile\.am:2:.*installing.*build-aux/test-driver' stderr
+
+:
diff --git a/tests/parallel-tests-dry-run.test
b/tests/parallel-tests-dry-run.test
new file mode 100755
index 0000000..888d888
--- /dev/null
+++ b/tests/parallel-tests-dry-run.test
@@ -0,0 +1,112 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check `make -n' for testsuite-related targets, when `parallel-tests'
+# is in use.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TESTS = foo.test bar.test
+TEST_LOG_COMPILER = $(SHELL)
+END
+
+$ACLOCAL
+$AUTOMAKE -a
+$AUTOCONF
+
+./configure
+
+make_n_ ()
+{
+ st=0
+ $MAKE -n "$@" >output 2>&1 || { cat output; ls -l; Exit 1; }
+ cat output
+ # Look out for possible errors from common tools used by recipes.
+ $EGREP -i ' (exist|permission|denied|no .*(such|file))' output && Exit 1
+ $EGREP '(mv|cp|rm|cat|grep|sed|awk): ' output && Exit 1
+ :
+}
+
+: > output
+
+files='foo.log bar.log foo.trs bar.trs'
+
+for target in check recheck test-suite.log; do
+ test ! -f foo.log
+ test ! -f foo.trs
+ test ! -f bar.log
+ test ! -f bar.trs
+ test ! -f test-suite.log
+done
+
+# FIXME: creative quoting below to please maintainer-check.
+echo exit '0' > foo.test
+echo exit '1' > bar.test
+
+$MAKE check && Exit 1
+
+chmod a-w .
+
+make_n_ clean
+test -f foo.log
+test -f foo.trs
+test -f foo.log
+test -f bar.trs
+
+# FIXME: creative quoting below to please maintainer-check.
+cat > foo.test <<END
+echo this is bad
+exit 1
+END
+echo exit '0' > bar.test
+
+for target in check recheck test-suite.log; do
+ make_n_ $target
+ grep '^:test-result: *FAIL' bar.trs
+ grep 'this is bad' foo.log test-suite.log && Exit 1
+done
+
+chmod a-rw $files
+if test -r foo.log; then
+ : You can still read unreadable files! Skip these checks.
+else
+ for target in check recheck test-suite.log; do
+ make_n_ $target
+ for f in $files; do
+ test -f $f && test ! -r $f || Exit 1
+ done
+ done
+fi
+chmod u+r $files
+
+chmod u+w .
+rm -f foo.log bar.trs
+chmod a-w .
+for target in check recheck test-suite.log $files; do
+ make_n_ $target
+ test ! -f foo.log
+ test -f foo.trs
+ test ! -f bar.trs
+ test -f bar.log
+done
+
+:
diff --git a/tests/parallel-tests-empty-testlogs.test
b/tests/parallel-tests-empty-testlogs.test
new file mode 100755
index 0000000..e3d77ea
--- /dev/null
+++ b/tests/parallel-tests-empty-testlogs.test
@@ -0,0 +1,86 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - empty TESTS
+# - empty TEST_LOGS
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_CONFIG_FILES([sub1/Makefile sub2/Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+SUBDIRS = sub1 sub2
+END
+
+mkdir sub1 sub2
+
+cat > sub1/Makefile.am << 'END'
+TESTS =
+check-local:
+ echo $(TEST_LOGS) | grep . && exit 1; exit 0
+END
+
+cat > sub2/Makefile.am << 'END'
+TESTS = foo.test
+END
+
+cat > sub2/foo.test <<'END'
+#! /bin/sh
+exit 0
+END
+chmod a+x sub2/foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+no_test_has_run ()
+{
+ ls -1 *.log | grep -v '^test-suite\.log$' | grep . && Exit 1
+ grep '^# TOTAL: *0$' test-suite.log
+ :
+}
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+ $srcdir/configure
+ cd sub1
+ VERBOSE=yes $MAKE check
+ no_test_has_run
+ cd ../sub2
+ VERBOSE=yes TESTS='' $MAKE -e check
+ no_test_has_run
+ VERBOSE=yes TEST_LOGS='' $MAKE -e check
+ no_test_has_run
+ cd ..
+ $MAKE check
+ cat sub2/foo.log
+ $MAKE distclean
+ cd $srcdir
+done
+
+:
diff --git a/tests/parallel-tests-exit-statuses.test
b/tests/parallel-tests-exit-statuses.test
new file mode 100755
index 0000000..840a14a
--- /dev/null
+++ b/tests/parallel-tests-exit-statuses.test
@@ -0,0 +1,102 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features: normal and special exit statuses
+# in the test scripts.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+# $failure_statuses should be defined to the list of all integers between
+# 1 and 255 (inclusive), excluded 77 and 99.
+failure_statuses=`seq_ 1 255 | $EGREP -v '^(77|99)$' | tr "$nl" ' '`
+# For debugging.
+echo "failure_statuses: $failure_statuses"
+# Sanity check.
+test `for st in $failure_statuses; do echo $st; done | wc -l` -eq 253 \
+ || fatal_ "initializing list of exit statuses for simple failures"
+
+cat > Makefile.am <<END
+LOG_COMPILER = ./do-exit
+fail_tests = $failure_statuses
+TESTS = 0 77 99 $failure_statuses
+\$(TESTS):
+END
+
+cat > do-exit <<'END'
+#!/bin/sh
+echo "$0: $1"
+case $1 in
+ [0-9]|[0-9][0-9]|[0-9][0-9][0-9]) st=$1;;
+ */[0-9]|*/[0-9][0-9]|*/[0-9][0-9][0-9]) st=`echo x"$1" | sed 's|.*/||'`;;
+ *) st=99;;
+esac
+exit $st
+END
+chmod a+x do-exit
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+{
+ echo PASS: 0
+ echo SKIP: 77
+ echo ERROR: 99
+ for st in $failure_statuses; do
+ echo "FAIL: $st"
+ done
+} | LC_ALL=C sort > exp-fail
+
+sed 's/^FAIL:/XFAIL:/' exp-fail | LC_ALL=C sort > exp-xfail-1
+sed '/^ERROR:/d' exp-xfail-1 > exp-xfail-2
+
+sort exp-fail
+sort exp-xfail-1
+sort exp-xfail-2
+
+./configure
+
+st=1
+$MAKE check >stdout && st=0
+cat stdout
+cat test-suite.log
+test $st -gt 0 || Exit 1
+LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-fail
+diff exp-fail got-fail
+
+st=1
+XFAIL_TESTS="$failure_statuses 99" $MAKE -e check >stdout && st=0
+cat stdout
+cat test-suite.log
+test $st -gt 0 || Exit 1
+LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-xfail-1
+diff exp-xfail-1 got-xfail-1
+
+st=0
+XFAIL_TESTS="$failure_statuses" TESTS="0 77 $failure_statuses" \
+ $MAKE -e check >stdout || st=$?
+cat stdout
+cat test-suite.log
+test $st -eq 0 || Exit 1
+LC_ALL=C grep '^[A-Z][A-Z]*:' stdout | LC_ALL=C sort > got-xfail-2
+diff exp-xfail-2 got-xfail-2
+
+:
diff --git a/tests/parallel-tests-ext-driver-prog.test
b/tests/parallel-tests-ext-driver-prog.test
deleted file mode 100755
index e92d262..0000000
--- a/tests/parallel-tests-ext-driver-prog.test
+++ /dev/null
@@ -1,94 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Check parallel-tests features:
-# - per-extension test drivers (xxx_LOG_COMPILER and xxx_LOG_FLAGS),
-# when some tests are PROGRAMS.
-# See also related test 'parallel-tests-ext-driver.test'.
-
-parallel_tests=yes
-required='cc native'
-. ./defs || Exit 1
-
-cat >> configure.in << 'END'
-AC_PROG_CC
-AC_OUTPUT
-END
-
-cat > Makefile.am << 'END'
-## Note that automake should not match the '/test' part
-## of 'sub/test' as '.test' suffix.
-TESTS = foo.chk bar.test $(check_PROGRAMS) sub/test
-check_PROGRAMS = baz bla.test bli.suff
-TEST_EXTENSIONS = .chk .test
-CHK_LOG_COMPILER = ./chk-driver
-TEST_LOG_COMPILER = ./test-driver
-LOG_COMPILER = ./noext-driver
-AM_CHK_LOG_FLAGS = 1
-CHK_LOG_FLAGS = 2
-AM_TEST_LOG_FLAGS = 3
-TEST_LOG_FLAGS = 4
-AM_LOG_FLAGS = 5
-LOG_FLAGS = 6
-END
-
-mkdir sub
-
-cat >chk-driver <<'END'
-#! /bin/sh
-echo $0 "$@"
-shift
-shift
-exec "$@"
-exit 127
-END
-chmod a+x chk-driver
-cp chk-driver test-driver
-cp chk-driver noext-driver
-
-cat >foo.chk << 'END'
-#! /bin/sh
-exit 0
-END
-chmod a+x foo.chk
-cp foo.chk bar.test
-cp foo.chk sub/test
-
-cat >baz.c << 'END'
-int main (void)
-{
- return 0;
-}
-END
-cp baz.c bla.c
-cp baz.c bli.c
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE -a
-
-./configure
-$MAKE check || { cat test-suite.log; Exit 1; }
-ls -l . sub
-cat test-suite.log
-grep 'chk-driver *1 *2 ' foo.log
-grep 'test-driver *3 *4 ' bar.log
-grep 'noext-driver *5 *6 ' baz.log
-grep 'test-driver *3 *4 ' bla.log
-grep 'noext-driver *5 *6 ' bli.suff.log
-grep 'noext-driver *5 *6 ' sub/test.log
-
-:
diff --git a/tests/parallel-tests-ext-driver.test
b/tests/parallel-tests-ext-driver.test
deleted file mode 100755
index 44d7606..0000000
--- a/tests/parallel-tests-ext-driver.test
+++ /dev/null
@@ -1,136 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Check parallel-tests features:
-# - per-extension test drivers (xxx_LOG_COMPILER and xxx_LOG_FLAGS),
-# also with AC_SUBST'd stuff.
-# See also related test 'parallel-tests-ext-driver-prog.test'.
-
-parallel_tests=yes
-. ./defs || Exit 1
-
-cat >> configure.in << 'END'
-AC_SUBST([acsubst_driver], [t-driver])
-AC_CONFIG_LINKS([chk-driver:chk-driver])
-AC_OUTPUT
-END
-
-cat > Makefile.am << 'END'
-# Note that automake should not match the '/test' part of 'sub/test'
-# as '.test' suffix.
-TESTS = foo.chk bar.test baz bla.t bli.suff sub/test
-
-TEST_EXTENSIONS = .chk .test .t
-
-CHK_LOG_COMPILER = CHK=ok; export CHK; ./chk-driver
-TEST_LOG_COMPILER = CHK=ok $(SHELL) $(srcdir)/test-driver
-T_LOG_COMPILER = @acsubst_driver@
-LOG_COMPILER = @SHELL@ -x $(srcdir)/noext-driver
-
-CHK_LOG_FLAGS = --chk
-
-AM_TEST_LOG_FLAGS = --am-test
-
-T_LOG_FLAGS = --t
-AM_T_LOG_FLAGS = --am-t
-
-AM_LOG_FLAGS = -a
-LOG_FLAGS = -b
-
-# `t-driver' is deliberately not distributed; it should be available
-# in PATH also from "make distcheck".
-EXTRA_DIST = $(TESTS) chk-driver test-driver noext-driver
-END
-
-unset CHK || :
-
-mkdir sub
-
-cat > chk-driver <<'END'
-#! /bin/sh
-test x"$1" = x"--chk" || {
- echo "$0: invalid command line: $*" >&2
- exit 1
-}
-test x"$CHK" = x"ok" || {
- echo "$0: bad \$CHK value '$CHK'" >&2
- exit 1
-}
-shift
-exec "$@"
-exit 127
-END
-sed 's/--chk/--am-test/' chk-driver > test-driver
-
-mkdir bin
-PATH=`pwd`/bin$PATH_SEPARATOR$PATH; export PATH
-
-cat > bin/t-driver <<'END'
-#! /bin/sh
-if test x"$1,$2" != x"--am-t,--t"; then
- echo "$0: invalid command line: $*" >&2
- exit 1
-fi
-if test x${CHK+"set"} = x"set"; then
- echo "$0: $CHK is set in the environment" >&2
- exit 1
-fi
-shift; shift
-exec "$@"
-exit 127
-END
-sed 's/--am-t,--t/-a,-b/' bin/t-driver > noext-driver
-
-chmod a+x chk-driver chk-driver test-driver bin/t-driver noext-driver
-
-cat >foo.chk << 'END'
-#! /bin/sh
-exit 0
-END
-chmod a+x foo.chk
-cp foo.chk bar.test
-cp foo.chk baz
-cp foo.chk bla.t
-cp foo.chk bli.suff
-cp foo.chk sub/test
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE -a
-
-./configure
-$MAKE check || { cat test-suite.log; Exit 1; }
-ls -l . sub
-cat test-suite.log
-test -f foo.log
-test -f bar.log
-test -f baz.log
-test -f bla.log
-test -f bli.suff.log
-test -f sub/test.log
-
-T_LOG_FLAGS=--bad $MAKE -e check && Exit 1
-cat test-suite.log
-cat bla.log
-# With the above flag overridden, bla.t should fail ...
-$EGREP '(^ *|/)t-driver:.* invalid .*--bad' bla.log
-# ... but no other test should.
-grep -v '^FAIL: bla\.t ' bla.log | grep 'FAIL:' && Exit 1
-
-# Try also with a VPATH build.
-$MAKE distcheck
-
-:
diff --git a/tests/parallel-tests-extra-programs.test
b/tests/parallel-tests-extra-programs.test
new file mode 100755
index 0000000..f309b4c
--- /dev/null
+++ b/tests/parallel-tests-extra-programs.test
@@ -0,0 +1,174 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Parallel test harness: check that $(TESTS) can lazily depend on
+# (or even be) $(EXTRA_PROGRAMS).
+
+required='cc native'
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AC_OUTPUT
+END
+
+# Will be extended later.
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .bin .test
+EXTRA_PROGRAMS =
+TESTS =
+END
+
+#
+# Now try various kinds of test dependencies ...
+#
+
+# 1. A program that is also a test, and whose source files
+# already exist.
+
+cat >> Makefile.am <<'END'
+EXTRA_PROGRAMS += foo.bin
+TESTS += foo.bin
+foo_bin_SOURCES = foo.c
+END
+
+cat > foo.c <<'END'
+#include <stdio.h>
+int main (void)
+{
+ printf ("foofoofoo\n");
+ return 0;
+}
+END
+
+# 2. A program that is also a test, and whose source files
+# are buildable by make.
+cat >> Makefile.am <<'END'
+EXTRA_PROGRAMS += bar.bin
+TESTS += bar.bin
+bar_bin_SOURCES = bar.c
+bar.c: foo.c
+ sed -e 's/foofoofoo/barbarbar/' foo.c > $@
+END
+
+# 3. A test script that already exists, whose execution depends
+# on a program whose source files already exist and which is
+# not itself a test.
+cat >> Makefile.am <<'END'
+EXTRA_PROGRAMS += y
+TESTS += baz.test
+baz.log: y$(EXEEXT)
+END
+
+cat > baz.test <<'END'
+#!/bin/sh
+$srcdir/y "$@" | sed 's/.*/&ep&ep&ep/'
+END
+chmod a+x baz.test
+
+cat > y.c <<'END'
+#include <stdio.h>
+int main (void)
+{
+ printf ("y\n");
+ return 0;
+}
+END
+
+# 4. A program that is also a test, but whose source files
+# do not exit and are not buildable by make.
+
+cat >> Makefile.am <<'END'
+EXTRA_PROGRAMS += none.bin
+TESTS += none.bin
+none_bin_SOURCES = none.c
+END
+
+#
+# Setup done, go with the tests.
+#
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+# What we check now:
+# 1. even if we cannot build the `none.bin' program, all the other
+# test programs should be built, and all the other tests should
+# be run;
+# 2. still, since we cannot create the `none.log' file, the
+# `test-suite.log' file shouldn't be created (as it depends
+# on *all* the test logs).
+
+st=0
+$MAKE check >stdout 2>stderr || st=$?
+cat stdout
+cat stderr >&2
+ls -l
+test $st -gt 0 || Exit 1
+
+# Files that should have been created, with the expected content.
+cat bar.c
+grep foofoofoo foo.log
+grep barbarbar bar.log
+grep yepyepyep baz.log
+# Files that shouldn't have been created.
+test ! -f none.log
+test ! -f test-suite.log
+# Expected testsuite progress output.
+grep '^PASS: baz\.test$' stdout
+# Don't anchor the end of the next two patterns, to allow for non-empty
+# $(EXEEXT).
+grep '^PASS: foo\.bin' stdout
+grep '^PASS: bar\.bin' stdout
+# Expected error messages from make.
+$EGREP 'none\.(bin|o|c)' stderr
+
+# What we check now:
+# 1. if we make the last EXTRA_PROGRAM buildable, the failed tests
+# pass;
+# 2. on a lazy re-run, the passed tests are not re-run, and
+# 3. their log files are not updated or touched.
+
+: > stamp
+$sleep
+
+echo 'int main (void) { return 0; }' > none.c
+
+st=0
+RECHECK_LOGS= $MAKE -e check >stdout || st=$?
+cat stdout
+ls -l
+test $st -eq 0 || Exit 1
+
+# For debugging.
+stat stamp foo.log bar.log baz.log || :
+
+# Files that shouldn't have been updated or otherwise touched.
+is_newest stamp foo.log bar.log baz.log
+# Files that should have been created now.
+test -f none.log
+test -f test-suite.log
+# Tests that shouldn't have been re-run.
+$EGREP '(foo|bar)\.bin|baz\.test$' stdout && Exit 1
+# Tests that should have been run. Again, we don't anchor the end
+# of the next pattern, to allow for non-empty $(EXEEXT).
+grep '^PASS: none\.bin' stdout
+
+:
diff --git a/tests/parallel-tests-fd-redirect.test
b/tests/parallel-tests-fd-redirect.test
new file mode 100755
index 0000000..e00581c
--- /dev/null
+++ b/tests/parallel-tests-fd-redirect.test
@@ -0,0 +1,109 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests support: redirection of file descriptors with
+# AM_TESTS_FD_REDIRECT, even when using tests without suffix, or
+# which are binary executables.
+# We use some tricks to ensure that all code paths in `lib/am/check2.am'
+# are covered, even on platforms where $(EXEEXT) would be naturally empty.
+# See also the more generic test 'check-fd-redirect.test'.
+
+required='cc native'
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AM_CONDITIONAL([real_EXEEXT], [test -n "$EXEEXT"])
+test -n "$EXEEXT" || EXEEXT=.bin
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+AM_TESTS_FD_REDIRECT = 9>&1
+TEST_EXTENSIONS = .test .sh
+TESTS = foo.sh bar $(check_PROGRAMS)
+check_PROGRAMS = baz qux.test
+qux_test_SOURCES = zardoz.c
+
+## Sanity check.
+if !real_EXEEXT
+check-local:
+ test -f baz.bin
+ test -f qux.test.bin
+endif
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+cat > foo.sh <<'END'
+#!/bin/sh
+echo " foofoofoo" >&9
+END
+chmod a+x foo.sh
+
+cat > bar <<'END'
+#!/bin/sh
+echo " barbarbar" >&9
+END
+
+chmod a+x foo.sh bar
+
+cat > baz.c <<'END'
+#include <unistd.h>
+int main (void)
+{
+ write (9, " bazbazbaz\n", 11);
+ return 0;
+}
+END
+
+cat > zardoz.c <<'END'
+#include <unistd.h>
+int main (void)
+{
+ write (9, " quxquxqux\n", 11);
+ return 0;
+}
+END
+
+./configure
+
+# Sanity checks.
+st=0
+grep '^bar\.log:.*bar' Makefile || st=1
+grep '^baz\.log:.*baz\$(EXEEXT)' Makefile || st=1
+grep '^\.test\$(EXEEXT)\.log:' Makefile || st=1
+$EGREP '^(foo|qux)\.log:' Makefile && st=1
+test $st -eq 0 || fatal_ "doesn't cover expected code paths"
+
+st=0
+$MAKE check >stdout || st=1
+cat stdout
+cat foo.log
+cat bar.log
+cat baz.log
+cat qux.log
+test $st -eq 0
+grep "^ foofoofoo$" stdout
+grep "^ barbarbar$" stdout
+grep "^ bazbazbaz$" stdout
+grep "^ quxquxqux$" stdout
+$EGREP '(foofoofoo|barbarbar|bazbazbaz|quxquxqux)' *.log && Exit 1
+
+:
diff --git a/tests/parallel-tests-harderror.test
b/tests/parallel-tests-harderror.test
index dedef36..f49b79b 100755
--- a/tests/parallel-tests-harderror.test
+++ b/tests/parallel-tests-harderror.test
@@ -67,13 +67,13 @@ DISABLE_HARD_ERRORS=x $MAKE check
$MAKE check DISABLE_HARD_ERRORS='' && Exit 1
cat test-suite.log
-grep '^FAIL: foo\.test .*exit.*99' test-suite.log
+grep '^ERROR: foo$' test-suite.log
cd sub
# The `-e' is wanted here.
DISABLE_HARD_ERRORS='' $MAKE -e check && Exit 1
cat test-suite.log
-grep '^FAIL: bar\.test .*exit.*99' test-suite.log
+grep '^ERROR: bar$' test-suite.log
cd ..
# Check the distributions.
@@ -88,7 +88,7 @@ $AUTOMAKE Makefile
./config.status Makefile
VERBOSE=yes $MAKE check && Exit 1
grep '^FAIL' test-suite.log && Exit 1
-grep '^FAIL: bar\.test .*exit.*99' sub/test-suite.log
+grep '^ERROR: bar$' sub/test-suite.log
echo 'DISABLE_HARD_ERRORS = zardoz' >> sub/Makefile
VERBOSE=yes $MAKE check
diff --git a/tests/parallel-tests-interrupt.test
b/tests/parallel-tests-interrupt.test
index 45e55d4..ed40412 100755
--- a/tests/parallel-tests-interrupt.test
+++ b/tests/parallel-tests-interrupt.test
@@ -28,19 +28,27 @@ END
cat > Makefile.am << 'END'
TESTS = foo.test
-## Ugly, but required by foo.test. See below.
-TEST_LOG_COMPILER = echo $$$$ > pid && exec 9>&2 && $(SHELL) -x
+## Provide more debugging info.
+TEST_LOG_COMPILER = $(SHELL) -x
+## Required by foo.test; see below.
+AM_TESTS_FD_REDIRECT = 9>&2
END
# This is hacky and ugly, but has the great advantage of avoiding us a lot
# of pain with background processes and related synchronization issues.
+
+cat - "$top_testsrcdir"/lib/test-driver > test-driver <<'END'
+#!/bin/sh
+echo $$ > pid
+END
+
cat > foo.test << 'END'
#!/bin/sh
exec 2>&9
echo "foo is starting to run"
ls -l >&2
-cat foo.log-t >&2 || : > fail
-grep '^foo is starting to run$' foo.log-t >&2 || : > fail
+cat foo.log >&2 || : > fail
+grep '^foo is starting to run$' foo.log >&2 || : > fail
cat pid >&2 || : > fail
kill -$signum `cat pid` || : > fail
END
@@ -48,7 +56,7 @@ chmod a+x foo.test
$ACLOCAL
$AUTOCONF
-$AUTOMAKE -a
+$AUTOMAKE
./configure
@@ -57,15 +65,15 @@ $AUTOMAKE -a
trapped_signals='1 2 13 15'
for signum in $trapped_signals; do
- rm -f pid fail *.log *.log-t
+ rm -f pid fail *.log
env signum=$signum $MAKE check && { ls -l; Exit 1; }
ls -l
- cat foo.log-t || :
+ # These files shouldn't exist, but in case they do, their content might
+ # provide helpful information about the causes of the failure(s).
cat foo.log || :
cat test-suite.log || :
test -f fail && Exit 1
- test -f foo.log-t && Exit 1
- test -f foo.log && Exit 1
+ ls | $EGREP 'foo.*\.(log|tmp)' && Exit 1
done
:
diff --git a/tests/parallel-tests-log-compiler-1.test
b/tests/parallel-tests-log-compiler-1.test
new file mode 100755
index 0000000..887fcee
--- /dev/null
+++ b/tests/parallel-tests-log-compiler-1.test
@@ -0,0 +1,126 @@
+#! /bin/sh
+# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - per-extension "test runners" a.k.a. "log compilers" (xxx_LOG_COMPILER
+# and xxx_LOG_FLAGS), also with AC_SUBST'd stuff.
+# See also related test 'parallel-tests-log-compiler-2.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_SUBST([acsubst_compiler], [t-compiler])
+AC_CONFIG_LINKS([chk-compiler:chk-compiler])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+# Note that automake should not match the '/test' part of 'sub/test'
+# as '.test' suffix.
+TESTS = foo.chk bar.test baz bla.t bli.suff sub/test
+
+TEST_EXTENSIONS = .chk .test .t
+
+CHK_LOG_COMPILER = ./chk-compiler
+TEST_LOG_COMPILER = $(SHELL) $(srcdir)/test-compiler
+T_LOG_COMPILER = @acsubst_compiler@
+LOG_COMPILER = @SHELL@ -x $(srcdir)/noext-compiler
+
+CHK_LOG_FLAGS = --chk
+
+AM_TEST_LOG_FLAGS = --am-test
+
+T_LOG_FLAGS = --t
+AM_T_LOG_FLAGS = --am-t
+
+AM_LOG_FLAGS = -a
+LOG_FLAGS = -b
+
+# `t-compiler' is deliberately not distributed; it should be available
+# in PATH also from "make distcheck".
+EXTRA_DIST = $(TESTS) chk-compiler test-compiler noext-compiler
+END
+
+mkdir sub
+
+cat > chk-compiler <<'END'
+#! /bin/sh
+test x"$1" = x"--chk" || {
+ echo "$0: invalid command line: $*" >&2
+ exit 1
+}
+shift
+exec "$@"
+exit 127
+END
+sed 's/--chk/--am-test/' chk-compiler > test-compiler
+
+mkdir bin
+PATH=`pwd`/bin$PATH_SEPARATOR$PATH; export PATH
+
+cat > bin/t-compiler <<'END'
+#! /bin/sh
+if test x"$1,$2" != x"--am-t,--t"; then
+ echo "$0: invalid command line: $*" >&2
+ exit 1
+fi
+shift; shift
+exec "$@"
+exit 127
+END
+sed 's/--am-t,--t/-a,-b/' bin/t-compiler > noext-compiler
+
+chmod a+x chk-compiler chk-compiler test-compiler bin/t-compiler noext-compiler
+
+cat >foo.chk << 'END'
+#! /bin/sh
+exit 0
+END
+chmod a+x foo.chk
+cp foo.chk bar.test
+cp foo.chk baz
+cp foo.chk bla.t
+cp foo.chk bli.suff
+cp foo.chk sub/test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+$MAKE check || { cat test-suite.log; Exit 1; }
+ls -l . sub
+cat test-suite.log
+test -f foo.log
+test -f bar.log
+test -f baz.log
+test -f bla.log
+test -f bli.suff.log
+test -f sub/test.log
+
+T_LOG_FLAGS=--bad $MAKE -e check && Exit 1
+cat test-suite.log
+cat bla.log
+# With the above flag overridden, bla.t should fail ...
+$EGREP '(^ *|/)t-compiler:.* invalid .*--bad' bla.log
+# ... but no other test should.
+grep -v '^FAIL: bla\.t ' bla.log | grep 'FAIL:' && Exit 1
+
+# Try also with a VPATH build.
+$MAKE distcheck
+
+:
diff --git a/tests/parallel-tests-log-compiler-2.test
b/tests/parallel-tests-log-compiler-2.test
new file mode 100755
index 0000000..0099b7f
--- /dev/null
+++ b/tests/parallel-tests-log-compiler-2.test
@@ -0,0 +1,96 @@
+#! /bin/sh
+# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - per-extension "test runners" a.k.a. "log compilers" (xxx_LOG_COMPILER
+# and xxx_LOG_FLAGS), when some tests are PROGRAMS.
+# See also related test 'parallel-tests-log-compiler-1.test'.
+
+parallel_tests=yes
+required='cc native'
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_PROG_CC
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+## Note that automake should not match the '/test' part
+## of 'sub/test' as '.test' suffix.
+TESTS = foo.chk bar.test $(check_PROGRAMS) sub/test
+check_PROGRAMS = baz bla.test bli.suff
+TEST_EXTENSIONS = .chk .test
+CHK_LOG_COMPILER = ./chk-compiler
+TEST_LOG_COMPILER = ./test-compiler
+LOG_COMPILER = ./noext-compiler
+AM_CHK_LOG_FLAGS = 1
+CHK_LOG_FLAGS = 2
+AM_TEST_LOG_FLAGS = 3
+TEST_LOG_FLAGS = 4
+AM_LOG_FLAGS = 5
+LOG_FLAGS = 6
+END
+
+mkdir sub
+
+cat >chk-compiler <<'END'
+#! /bin/sh
+echo $0 "$@"
+shift
+shift
+exec "$@"
+exit 127
+END
+chmod a+x chk-compiler
+cp chk-compiler test-compiler
+cp chk-compiler noext-compiler
+
+cat >foo.chk << 'END'
+#! /bin/sh
+exit 0
+END
+chmod a+x foo.chk
+cp foo.chk bar.test
+cp foo.chk sub/test
+
+cat >baz.c << 'END'
+int main (void)
+{
+ return 0;
+}
+END
+cp baz.c bla.c
+cp baz.c bli.c
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE
+$MAKE check || { cat test-suite.log; Exit 1; }
+ls -l . sub
+cat test-suite.log
+grep 'chk-compiler *1 *2' foo.log
+grep 'test-compiler *3 *4' bar.log
+grep 'noext-compiler *5 *6' baz.log
+grep 'test-compiler *3 *4' bla.log
+grep 'noext-compiler *5 *6' bli.suff.log
+grep 'noext-compiler *5 *6' sub/test.log
+
+:
diff --git a/tests/parallel-tests-log-override-2.test
b/tests/parallel-tests-log-override-2.test
index b790c41..ef37a2c 100755
--- a/tests/parallel-tests-log-override-2.test
+++ b/tests/parallel-tests-log-override-2.test
@@ -65,6 +65,7 @@ do
$MAKE -e check >stdout || { cat stdout; Exit 1; }
cat stdout
ls -l
+ count_test_results total=2 pass=1 fail=0 skip=1 xfail=0 xpass=0 error=0
cat pass.log
cat skip.log
cat partial.log
@@ -72,14 +73,12 @@ do
test ! -f pass2.log
test ! -f skip2.log
test ! -f fail.log
- grep '^PASS: .*pass\.test' stdout
- grep '^SKIP: .*skip\.test' stdout
- $FGREP 'skip.test' partial.log
+ grep '^PASS: pass\.test$' stdout
+ grep '^SKIP: skip\.test$' stdout
+ $FGREP 'SKIP: skip' partial.log
$FGREP '% skipped test %' partial.log
- for t in pass2 skip2 fail; do
- $FGREP "$t.test" stdout && Exit 1
- $FGREP "$t.test" partial.log && Exit 1
- done
+ $EGREP '(pass2|skip2|fail)\.test' stdout && Exit 1
+ $EGREP '(pass2|skip2|fail)' partial.log && Exit 1
rm -f *.log
done
diff --git a/tests/parallel-tests-log-override-recheck.test
b/tests/parallel-tests-log-override-recheck.test
index 0739cbe..5996895 100755
--- a/tests/parallel-tests-log-override-recheck.test
+++ b/tests/parallel-tests-log-override-recheck.test
@@ -25,22 +25,22 @@ AC_OUTPUT
END
cat > Makefile.am <<'END'
-TESTS = foo.test bar.test baz.test
+TESTS = foofoo.test barbar.test bazbaz.test
END
-cat > foo.test <<'END'
+cat > foofoo.test <<'END'
#! /bin/sh
echo "this is $0"
exit 0
END
-cat > bar.test <<'END'
+cat > barbar.test <<'END'
#! /bin/sh
echo "this is $0"
exit 99
END
-cat > baz.test <<'END'
+cat > bazbaz.test <<'END'
#! /bin/sh
echo "this is $0"
exit ${BAZ_EXIT_STATUS-1}
@@ -54,6 +54,13 @@ $ACLOCAL
$AUTOCONF
$AUTOMAKE -a
+# Filter make output before grepping it, for make implementations that,
+# like Solaris' one, print the whole of the failed recipe on failure.
+filter_stdout ()
+{
+ grep -v 'bases=.*;' stdout > t && mv -f t stdout
+}
+
./configure
$MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
@@ -63,11 +70,12 @@ TEST_SUITE_LOG=my.log $MAKE -e recheck >stdout \
&& { cat stdout; Exit 1; }
cat stdout
ls -l
-grep '^2 of 2 .*failed' stdout
+filter_stdout
+count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
for x in stdout my.log; do
- $FGREP foo.test $x && Exit 1
- $FGREP bar.test $x
- $FGREP baz.test $x
+ $FGREP foofoo $x && Exit 1
+ $FGREP barbar $x
+ $FGREP bazbaz $x
done
chmod a-rw my.log
@@ -75,16 +83,17 @@ BAZ_EXIT_STATUS=0 TEST_SUITE_LOG=my2.log $MAKE -e recheck
>stdout \
&& { cat stdout; Exit 1; }
cat stdout
ls -l
-grep '^1 of 2 .*failed' stdout
-$FGREP foo.test stdout && Exit 1
-$FGREP bar.test stdout
-$FGREP baz.test stdout
-$FGREP foo.test my2.log && Exit 1
-$FGREP bar.test my2.log
-$FGREP baz.test my2.log && Exit 1
+count_test_results total=2 pass=1 fail=0 skip=0 xfail=0 xpass=0 error=1
+filter_stdout
+$FGREP foofoo stdout && Exit 1
+$FGREP barbar stdout
+$FGREP bazbaz stdout
+$FGREP foofoo my2.log && Exit 1
+$FGREP barbar my2.log
+$FGREP bazbaz my2.log && Exit 1
chmod u+r test-suite.log my.log
-$FGREP baz.test test-suite.log
-$FGREP baz.test my.log
+$FGREP bazbaz test-suite.log
+$FGREP bazbaz my.log
:
diff --git a/tests/parallel-tests-no-color-in-log.test
b/tests/parallel-tests-no-color-in-log.test
new file mode 100755
index 0000000..5d33ac5
--- /dev/null
+++ b/tests/parallel-tests-no-color-in-log.test
@@ -0,0 +1,63 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Colorized output from the testsuite report shouldn't end up in log files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+esc=''
+
+# Check that grep can parse nonprinting characters.
+# BSD 'grep' works from a pipe, but not a seekable file.
+# GNU or BSD 'grep -a' works on files, but is not portable.
+case `echo "$esc" | $FGREP "$esc"` in
+ "$esc") ;;
+# FIXME: creative quoting below to please maintainer-check.
+ *) echo "$me: f""grep can't parse nonprinting characters" >&2; Exit 77;;
+esac
+
+TERM=ansi; export TERM
+
+cat >>configure.in <<END
+AC_OUTPUT
+END
+
+cat >Makefile.am <<'END'
+LOG_COMPILER = $(SHELL)
+AUTOMAKE_OPTIONS = color-tests parallel-tests
+TESTS = pass fail skip xpass xfail error
+XFAIL_TESTS = xpass xfail
+END
+
+# FIXME: creative quoting to please maintainer-check.
+echo exit '0' > pass
+echo exit '0' > xpass
+echo exit '1' > fail
+echo exit '1' > xfail
+echo exit '77' > skip
+echo exit '99' > error
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+./configure
+mv config.log config-log # Avoid possible false positives below.
+AM_COLOR_TESTS=always $MAKE -e check && Exit 1
+$FGREP "$esc" *.log && Exit 1
+
+:
diff --git a/tests/parallel-tests-no-spurious-summary.test
b/tests/parallel-tests-no-spurious-summary.test
new file mode 100755
index 0000000..9ef7715
--- /dev/null
+++ b/tests/parallel-tests-no-spurious-summary.test
@@ -0,0 +1,68 @@
+#! /bin/sh
+# Copyright (C) 2009, 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that `:test-results:' directives in test scripts' output doesn't
+# originate spurious results in the testsuite summary.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+echo :test-result:XFAIL
+echo :test-result: SKIP
+echo :test-result:ERROR
+exit 0
+END
+cat > bar.test <<'END'
+#! /bin/sh
+echo :test-result: ERROR
+echo :test-result:FAIL
+echo :test-result: XPASS
+exit 0
+END
+chmod a+x foo.test bar.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+st=0
+$MAKE check >stdout || st=$?
+cat stdout
+cat test-suite.log
+cat foo.log
+cat bar.log
+test $st -eq 0 || Exit $st
+
+grep '^:test-result:XFAIL$' foo.log
+grep '^:test-result: SKIP$' foo.log
+grep '^:test-result:FAIL$' bar.log
+grep '^:test-result: XPASS$' bar.log
+
+count_test_results total=2 pass=2 fail=0 skip=0 xfail=0 xpass=0 error=0
+
+:
diff --git a/tests/parallel-tests-once.test b/tests/parallel-tests-once.test
new file mode 100755
index 0000000..d676a12
--- /dev/null
+++ b/tests/parallel-tests-once.test
@@ -0,0 +1,48 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Tests shouldn't be run multiple times by a simple "make check" in a
+# clean directory. An early implementation of the `.trs' intermediate
+# files incurred a similar problem.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+test -f foo.run && Exit 1
+: > foo.run
+END
+chmod a+x foo.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check
+test -f foo.run # Sanity check.
+
+:
diff --git a/tests/parallel-tests-unreadable-log.test
b/tests/parallel-tests-unreadable-log.test
deleted file mode 100755
index 8560c37..0000000
--- a/tests/parallel-tests-unreadable-log.test
+++ /dev/null
@@ -1,60 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Check that the testsuite driver copes well with unreadable test logs.
-
-parallel_tests=yes
-. ./defs || Exit 1
-
-cat >> configure.in << 'END'
-AC_OUTPUT
-END
-
-cat > Makefile.am << 'END'
-TESTS = foo.test bar.test
-XFAIL_TESTS = bar.test
-END
-
-cat > foo.test << 'END'
-#! /bin/sh
-exit 0
-END
-cat > bar.test << 'END'
-#! /bin/sh
-exit 1
-END
-chmod a+x foo.test bar.test
-
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE -a
-
-# The testsuite driver will use this variable, so ensure it sanitizes
-# it and do not allow in spurious values from the environment.
-line=PASS; export line
-
-./configure
-
-$MAKE foo.log
-$MAKE bar.log
-chmod a-r foo.log bar.log
-test ! -r foo.log || skip_ "cannot drop file read permissions"
-$MAKE test-suite.log >stdout && { cat stdout; Exit 1; }
-cat stdout
-grep '^2 of 2 tests failed *$' stdout
-grep '^2 of 2 tests failed\. *$' test-suite.log
-
-:
diff --git a/tests/parallel-tests-unreadable.test
b/tests/parallel-tests-unreadable.test
new file mode 100755
index 0000000..6192a13
--- /dev/null
+++ b/tests/parallel-tests-unreadable.test
@@ -0,0 +1,75 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that the testsuite driver copes well with unreadable `.log'
+# and `.trs' files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+: > t
+chmod a-r t && test ! -r t || skip_ "you can still read unreadable files"
+rm -f t
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test
+END
+
+cat > foo.test << 'END'
+#! /bin/sh
+echo foofoofoo
+exit 0
+END
+
+cat > bar.test << 'END'
+#! /bin/sh
+echo barbarbar
+exit 77
+END
+
+chmod a+x foo.test bar.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+for files in \
+ 'foo.log bar.log' \
+ 'foo.trs bar.trs' \
+ 'foo.trs bar.log' \
+ 'foo.log bar.trs' \
+; do
+ $MAKE check
+ rm -f test-suite.log
+ chmod a-r $files
+ $MAKE test-suite.log || { ls -l; Exit 1; }
+ ls -l
+ grep '^foofoofoo$' foo.log
+ grep '^:test-result: PASS' foo.trs
+ grep '^barbarbar$' bar.log
+ grep '^:test-result: SKIP' bar.trs
+ grep '^SKIP: bar' test-suite.log
+ grep '^barbarbar$' test-suite.log
+ $EGREP ':.*foo|foofoofoo' test-suite.log && Exit 1
+done
+
+:
diff --git a/tests/parallel-tests.test b/tests/parallel-tests.test
index b1e157c..014adfc 100755
--- a/tests/parallel-tests.test
+++ b/tests/parallel-tests.test
@@ -14,14 +14,14 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-# Check parallel-tests features:
-# - VERBOSE
-# - clean
-# - TEST_SUITE_LOG
-# - dependencies between tests
-# - TESTS
-# - TEST_LOGS
-# - RECHECK_LOGS
+# Basic checks on parallel-tests support:
+# - console output
+# - log files, and what goes in 'test-suite.log'
+# - make clean
+# - dependencies between tests
+# - TESTS redefinition at runtime
+# - TEST_LOGS redefinition at runtime
+# - RECHECK_LOGS redefinition at runtime
parallel_tests=yes
. ./defs || Exit 1
@@ -31,7 +31,6 @@ AC_OUTPUT
END
cat > Makefile.am << 'END'
-TEST_SUITE_LOG = mylog.log
TESTS = foo.test bar.test baz.test
XFAIL_TESTS = bar.test
foo.log: bar.log
@@ -68,35 +67,36 @@ $AUTOMAKE -a
$MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
-# There should be two errors: bar.test is a hard error.
-test `grep -c '^FAIL' stdout` -eq 2
-test -f mylog.log
-test `grep -c '^FAIL' mylog.log` -eq 2
+count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
+test -f test-suite.log
+cat test-suite.log
+test `grep -c '^FAIL:' test-suite.log` -eq 1
+test `grep -c '^ERROR:' test-suite.log` -eq 1
+$EGREP '^(X?PASS|XFAIL|SKIP)' test-suite.log && Exit 1
test -f baz.log
test -f bar.log
test -f foo.log
-# The summary should be formatted correctly.
-grep 'failedn' stdout && Exit 1
-# clean should remove all log files (but not more).
-: > unrelated.log
$MAKE clean
test ! -f baz.log
test ! -f bar.log
test ! -f foo.log
-test ! -f mylog.log
-test -f unrelated.log
+test ! -f test-suite.log
# Check dependencies: baz.test needs to run before bar.test,
# but foo.test is not needed.
# Note that this usage has a problem: the summary will only
# take bar.log into account, because the $(TEST_SUITE_LOG) rule
# does not "see" baz.log. Hmm.
-env TESTS='bar.test' $MAKE -e check && Exit 1
+env TESTS='bar.test' $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^FAIL: baz\.test$' stdout
+grep '^ERROR: bar\.test$' stdout
+
test -f baz.log
test -f bar.log
test ! -f foo.log
-test -f mylog.log
+test -f test-suite.log
# Upon a lazy rerun, foo.test should be run, but the others shouldn't.
# Note that the lazy rerun still exits with a failure, due to the previous
@@ -107,10 +107,12 @@ test -f mylog.log
env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
cat stdout
test -f foo.log
-grep foo.test stdout
+grep '^PASS: foo\.test$' stdout
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
# Now, explicitly retry with all test logs already updated, and ensure
# that the summary is still displayed.
@@ -119,7 +121,9 @@ cat stdout
grep foo.test stdout && Exit 1
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
# Lazily rerunning only foo should only rerun this one test.
env RECHECK_LOGS=foo.log $MAKE -e check > stdout && { cat stdout; Exit 1; }
@@ -127,13 +131,9 @@ cat stdout
grep foo.test stdout
grep bar.test stdout && Exit 1
grep baz.test stdout && Exit 1
-grep '2.*tests.*failed' stdout
-
-# Test VERBOSE.
-env VERBOSE=yes $MAKE -e check > stdout && { cat stdout; Exit 1; }
-cat stdout
-grep 'this is.*bar.test' stdout
-grep 'this is.*baz.test' stdout
+grep '^# PASS: *1$' stdout
+grep '^# FAIL: *1$' stdout
+grep '^# ERROR: *1$' stdout
$MAKE clean
env TEST_LOGS=baz.log $MAKE -e check > stdout && { cat stdout; Exit 1; }
diff --git a/tests/parallel-tests2.test b/tests/parallel-tests2.test
index 85256c8..ab390f8 100755
--- a/tests/parallel-tests2.test
+++ b/tests/parallel-tests2.test
@@ -15,8 +15,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Check parallel-tests features:
-# - check-html
-# - recheck-html
+# - check-html
+# - recheck-html
+# Keep this in sync with sister test `test-driver-custom-html.test'.
parallel_tests=yes
required=rst2html
@@ -41,16 +42,19 @@ echo "this is $0"
test -f bla || exit 1
exit 0
END
+
cat > bar.test <<'END'
#! /bin/sh
echo "this is $0"
exit 99
END
+
cat > baz.test <<'END'
#! /bin/sh
echo "this is $0"
exit 1
END
+
chmod a+x foo.test bar.test baz.test
$ACLOCAL
@@ -58,27 +62,35 @@ $AUTOCONF
$AUTOMAKE -a
./configure
-$MAKE check-html >stdout && { cat stdout; Exit 1; }
-cat stdout
+
+$MAKE check-html && Exit 1
test -f mylog.html
+# check-html should cause check_SCRIPTS to be created.
+test -f bla
+
+# "make clean" should remove HTML files.
+$MAKE clean
+test ! -f mylog.html
+test ! -f bla
# Always create the HTML output, even if there were no failures.
rm -f mylog.html
-env TESTS=foo.test $MAKE -e check-html >stdout || { cat stdout; Exit 1; }
-cat stdout
+env TESTS=foo.test $MAKE -e check-html
test -f mylog.html
-# Create HTML output also with recheck-html
+# Create summarizing HTML output also with recheck-html.
rm -f mylog.html
-env TESTS=foo.test $MAKE -e recheck-html >stdout || { cat stdout; Exit 1; }
-cat stdout
+env TESTS=foo.test $MAKE -e recheck-html
test -f mylog.html
-# check-html and recheck-html should cause check_SCRIPTS to be created,
-# and recheck-html should rerun no tests if check has not been run.
+# check-html should cause check_SCRIPTS to be created.
$MAKE clean
-env TESTS=foo.test $MAKE -e check-html
+env TEST_LOGS=foo.log $MAKE -e check-html
test -f bla
+test -f foo.log
+test -f mylog.html
+# recheck-html should cause check_SCRIPTS to be created, and should rerun
+# no tests if it appears that check has not been run.
$MAKE clean
env TESTS=foo.test $MAKE -e recheck-html
test -f bla
diff --git a/tests/parallel-tests3.test b/tests/parallel-tests3.test
index a8004e7..a140988 100755
--- a/tests/parallel-tests3.test
+++ b/tests/parallel-tests3.test
@@ -68,7 +68,7 @@ $MAKE -j4 check >> stdout
cd ..
kill $!
cat parallel/stdout
-test `grep -c PASS parallel/stdout` -eq 8
+test `grep -c '^PASS:' parallel/stdout` -eq 8
# Wait long enough so that there are no open files any more
# when the post-test cleanup runs.
diff --git a/tests/parallel-tests6.test b/tests/parallel-tests6.test
index 5992250..e0c535e 100755
--- a/tests/parallel-tests6.test
+++ b/tests/parallel-tests6.test
@@ -36,6 +36,8 @@ $AUTOMAKE -a
./configure
$MAKE check >stdout || { cat stdout; Exit 1; }
cat stdout
-grep '0 tests' stdout
+for x in TOTAL PASS FAIL XPASS FAIL SKIP ERROR; do
+ grep "^# $x: *0$" stdout
+done
Exit 0
diff --git a/tests/parallel-tests9.test b/tests/parallel-tests9.test
index b06d364..fb68d51 100755
--- a/tests/parallel-tests9.test
+++ b/tests/parallel-tests9.test
@@ -58,21 +58,14 @@ $AUTOMAKE -a
./configure
$MAKE check >stdout && { cat stdout; Exit 1; }
cat stdout
+count_test_results total=3 pass=1 fail=1 skip=0 xfail=0 xpass=0 error=1
$MAKE recheck >stdout && { cat stdout; Exit 1; }
cat stdout
-grep foo.test stdout && Exit 1
-grep bar.test stdout || Exit 1
-grep baz.test stdout || Exit 1
-grep '2 of 2.*failed' stdout
-
-# If we cannot read the log file, then redo it as well.
-chmod a-r foo.log
-if test ! -r foo.log; then
- $MAKE recheck >stdout && { cat stdout; Exit 1; }
- cat stdout
- grep foo.test stdout || Exit 1
-fi
+count_test_results total=2 pass=0 fail=1 skip=0 xfail=0 xpass=0 error=1
+grep 'foo\.test' stdout && Exit 1
+grep '^ERROR: bar\.test$' stdout
+grep '^FAIL: baz\.test$' stdout
# Ensure that recheck builds check_SCRIPTS, and that
# recheck reruns nothing if check has not been run.
diff --git a/tests/plain-functions.sh b/tests/plain-functions.sh
new file mode 100644
index 0000000..233f65d
--- /dev/null
+++ b/tests/plain-functions.sh
@@ -0,0 +1,40 @@
+# -*- shell-script -*-
+#
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Helper functions used by "plain" tests of the Automake testsuite
+# (i.e., tests that don't use any test protocol).
+
+# Print warnings (e.g., about skipped and failed tests) to this file
+# number. Override by putting, say:
+# AM_TESTS_ENVIRONMENT = stderr_fileno_=9; export stderr_fileno_;
+# AM_TESTS_FD_REDIRECT = 9>&2
+# in your Makefile.am.
+# This is useful when using automake's parallel tests mode, to print the
+# reason for skip/failure to console, rather than to the *.log files.
+: ${stderr_fileno_=2}
+
+# Copied from Gnulib's `tests/init.sh'.
+warn_ () { echo "$@" 1>&$stderr_fileno_; }
+fail_ () { warn_ "$me: failed test: $@"; Exit 1; }
+skip_ () { warn_ "$me: skipped test: $@"; Exit 77; }
+fatal_ () { warn_ "$me: hard error: $@"; Exit 99; }
+framework_failure_ () { warn_ "$me: set-up failure: $@"; Exit 99; }
+
+# For compatibility with TAP functions.
+skip_all_ () { skip_ "$@"; }
+
+:
diff --git a/tests/posixsubst-tests.test b/tests/posixsubst-tests.test
index ea9a21a..68eaa46 100755
--- a/tests/posixsubst-tests.test
+++ b/tests/posixsubst-tests.test
@@ -50,7 +50,7 @@ chmod +x foo1.test bary
$ACLOCAL
$AUTOCONF
-$AUTOMAKE
+$AUTOMAKE -a
./configure
$MAKE check >out 2>&1 && { cat out; Exit 1; }
diff --git a/tests/repeated-options.test b/tests/repeated-options.test
index 999c9e1..1609aa9 100755
--- a/tests/repeated-options.test
+++ b/tests/repeated-options.test
@@ -58,7 +58,7 @@ int main (void)
}
END
-cp "$top_testsrcdir"/lib/compile .
+cp "$top_testsrcdir"/lib/compile "$top_testsrcdir"/lib/test-driver .
$ACLOCAL
$AUTOMAKE --foreign --foreign -Wall 2>stderr || { cat stderr >&2; Exit 1; }
diff --git a/tests/self-check-cleanup.test b/tests/self-check-cleanup.test
index 7660316..fb5b055 100755
--- a/tests/self-check-cleanup.test
+++ b/tests/self-check-cleanup.test
@@ -57,6 +57,7 @@ test ! -r $dir
# Check that post-test cleanup works also with directories with
# "null" permissions, and containing broken symlinks.
$SHELL -c '
+ stderr_fileno_=2
. ./defs || Exit 1
set -e
mkdir dir dir/sub
@@ -95,6 +96,7 @@ if $have_symlinks; then
$SHELL -c '
ocwd=`pwd` || exit 1
+ stderr_fileno_=2
. ./defs || Exit 1
ln -s "$ocwd/dir" "$ocwd/file" .
' dummy.test
@@ -125,7 +127,12 @@ for bailout_command in \
'kill -13 $$' \
'kill -15 $$' \
; do
- $SHELL -c ". ./defs; : > foo; $bailout_command" dummy.test && Exit 1
+ $SHELL -c "
+ stderr_fileno_=2
+ . ./defs
+ : > foo
+ $bailout_command
+ " dummy.test && Exit 1
test -f dummy.dir/foo
done
diff --git a/tests/self-check-dir.test b/tests/self-check-dir.test
index 0b4dd4e..1aae601 100755
--- a/tests/self-check-dir.test
+++ b/tests/self-check-dir.test
@@ -16,26 +16,51 @@
# Sanity check for the automake testsuite.
# Check that tests using `./defs' create a proper temporary directory,
-# and run in it.
+# and run in it. Also check that we can prevent the pre-population
+# and the creation of such directory if we want.
. ./defs-static || exit 1
set -ex
-
AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC
-this=S_dir
-outcome=:
-pwd
+# This needs to be consistent with what $SHELL deems to be the
+# current working directory.
+cwd=`$SHELL -c 'pwd'` || exit 99
+echo "$cwd"
+
+rm -f _self.tmp
-rm -f $this.tmp
+st=0
+$SHELL -c '
+ . ./defs || Exit 1
+ pwd; ls -l; # For debugging.
+ : > _self.tmp
+ test -f install-sh
+ test -f configure.in
+ case `pwd` in '"$cwd"'/_self.dir);; *) Exit 1;; esac
+' _self.test || st=1
+test -f _self.tmp && { rm -f _self.tmp; st=1; }
+test $st -eq 0 || exit 1
-$SHELL -c "
+st=0
+$SHELL -c '
+ am_create_testdir=empty
. ./defs || Exit 1
- : > $this.tmp
- case \`pwd\` in '`pwd`'/$this.dir);; *) Exit 1;; esac
-" $this.test || outcome=false
+ pwd; ls -l; # For debugging.
+ ls -a | grep -v "^\\.\\{1,2\\}$" | grep . && Exit 1
+ : > _self.tmp
+ case `pwd` in '"$cwd"'/_self.dir);; *) Exit 1;; esac
+ :
+' _self.test || st=1
+test -f _self.tmp && { rm -f _self.tmp; st=1; }
+test $st -eq 0 || exit 1
-test -f $this.tmp && { rm -f $this.tmp; outcome=false; }
+$SHELL -c '
+ am_create_testdir=no
+ . ./defs || Exit 1
+ test -f self-check-dir.test
+ case `pwd` in '"$cwd"');; *) Exit 1;; esac
+' _self.test || exit 1
-$outcome
+:
diff --git a/tests/self-check-env-sanitize.tap
b/tests/self-check-env-sanitize.tap
new file mode 100755
index 0000000..4556e47
--- /dev/null
+++ b/tests/self-check-env-sanitize.tap
@@ -0,0 +1,56 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Sanity check for the automake testsuite.
+# Make sure that the testsuite initialization code complains when
+# some testsuite-influential variables are set in the environment.
+
+am_create_testdir=no
+. ./defs || Exit 1
+
+set -x
+exec 5>&1
+
+plan_ 16 # Two times the number of variable names in $vars.
+
+vars='
+ me
+ parallel_tests
+ required
+ use_tap
+ am_create_testdir
+ test_prefer_config_shell
+ original_AUTOMAKE
+ original_ACLOCAL
+'
+
+do_run ()
+{
+ env "$1=foo" $SHELL -c '. ./defs' foo.test
+}
+
+do_grep ()
+{
+ env "$1=foo" $SHELL -c '. ./defs' foo.test 2>&1 1>&5 \
+ | grep "foo\.test:.* variable \`$1'.* in the environment.*unsafe"
+}
+
+for var in $vars; do
+ command_ok_ "$var [err status]" not do_run $var
+ command_ok_ "$var [err message]" do_grep $var
+done
+
+:
diff --git a/tests/self-check-env-sanitize.test
b/tests/self-check-env-sanitize.test
deleted file mode 100755
index 9dc97be..0000000
--- a/tests/self-check-env-sanitize.test
+++ /dev/null
@@ -1,34 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Sanity check for the automake testsuite.
-# Make sure that the testsuite initialization code complains when
-# some testsuite-influential variables are set in the environment.
-
-. ./defs-static || exit 1
-
-set -x
-exec 5>&1
-
-for var in me parallel_tests required original_AUTOMAKE original_ACLOCAL \
- test_prefer_config_shell
-do
- env "$var=foo" $SHELL -c '. ./defs' foo.test && exit 1
- env "$var=foo" $SHELL -c '. ./defs' foo.test 2>&1 1>&5 \
- | grep "foo\.test:.* variable \`$var'.* in the environment.*unsafe" ||
exit 1
-done
-
-:
diff --git a/tests/self-check-exit.tap b/tests/self-check-exit.tap
new file mode 100755
index 0000000..fc3fdca
--- /dev/null
+++ b/tests/self-check-exit.tap
@@ -0,0 +1,93 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Sanity check for the automake testsuite.
+# Check that, in case of failing commands, the correct exit status is
+# passed to the exit trap installed by the `./defs' script.
+# Also check that the `errexit' shell flag is active.
+
+am_create_testdir=no
+. ./defs || Exit 99
+
+plan_ 34
+
+AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC
+
+init='. ./defs; stderr_fileno_=2; unset am_explicit_skips'
+
+for st in 1 2 3 4 5 77 99 126 127 128 129 130 255; do
+ for exit_cmd in "Exit $st" "sh -c 'exit $st'"; do
+ rc=0; $SHELL -c "$init; $exit_cmd; :" || rc=$?
+ command_ok_ "$exit_cmd" test $rc -eq $st
+ done
+done
+
+for sig in 1 2 13 15; do
+ if test $sig -eq 2; then
+ # Some Korn shells might otherwise get a spurious SIGINT
+ # signal when one is sent to the child $SHELL.
+ trap : 2
+ fi
+ rc=0; $SHELL -c "$init; kill -$sig \$\$; :" || rc=$?
+ if test $sig -eq 2; then
+ # Reset default SIGINT handler as portably as possible.
+ trap 2 || trap - 2
+ fi
+ if test x"$sh_errexit_works" = x"yes"; then
+ # The exit trap should turn into an hard errors any failure
+ # caused by signals.
+ command_ok_ "kill -$sig" test $rc -eq 99
+ else
+ # The exit trap is not installed, so that the shell should exit
+ # with status 128+n when receiving signal number n. But don't
+ # be too strict in the check, as POSIX only says that "The exit
+ # status of a command that terminated because it received a
+ # signal shall be reported as greater than 128".
+ command_ok_ "kill -$sig" test $rc -gt 128
+ fi
+
+done
+
+: Non-existent program.
+# Solaris 10 /bin/sh erroneously exit with success right away when the
+# following three conditions are met at the same time:
+# 1. the `errexit' flag is active,
+# 2. an exit trap is installed, and
+# 3. a non-existing command is issued.
+# Note that the non-existent command is issued as the last command to
+# the shell in the next line; this is deliberate.
+if $SHELL -c 'set -e; trap "exit \$?" 0; non-existent-program'; then
+ maybe_todo=TODO reason="known Solaris /bin/sh bug"
+else
+ maybe_todo="" reason=""
+fi
+rc=0; $SHELL -c "$init; non-existent-program; :" || rc=$?
+command_ok_ "command not found" -D "$maybe_todo" -r "$reason" \
+ -- test $rc -gt 0
+
+: Non-executable command.
+test -f Makefile && test ! -x Makefile || \
+ framowork_failure_ "no proper Makefile in the current directory"
+rc=0; $SHELL -c "$init; ./Makefile; :" || rc=$?
+command_ok_ "permission denied" test $rc -gt 0
+
+: Syntax errors in the test code.
+rc=0; $SHELL -c "$init; if :; then" || rc=$?
+command_ok_ "syntax error 1" test $rc -gt 0
+rc=0; $SHELL -c "$init; fi" || rc=$?
+command_ok_ "syntax error 2" test $rc -gt 0
+
+:
diff --git a/tests/self-check-exit.test b/tests/self-check-exit.test
deleted file mode 100755
index 69aabcd..0000000
--- a/tests/self-check-exit.test
+++ /dev/null
@@ -1,83 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Sanity check for the automake testsuite.
-# Check that, in case of failing commands, the correct exit status is
-# passed to the exit trap installed by the `./defs' script.
-# Also check that the `errexit' shell flag is active.
-
-. ./defs-static || exit 99
-
-unset am_explicit_skips
-AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC
-
-for st in 1 2 3 4 5 77 99 126 127 128 129 130 255; do
-
- echo "* Try: Exit $st"
- $SHELL -c ". ./defs; Exit $st; :"
- rc=$?
- echo "* rc=$rc"
- echo
- test $rc -eq $st || exit 1
-
- echo "* Try: sh -c 'exit $st'"
- $SHELL -c ". ./defs; sh -c 'exit $st'; :"
- rc=$?
- echo "* rc=$rc"
- echo
- test $rc -eq $st || exit 1
-
-done
-
-echo "* Try: non-existent-program"
-$SHELL -c ". ./defs; non-existent-program; :"
-rc=$?
-echo "* rc=$rc"
-echo
-test $rc -eq 127 || exit 1
-
-for sig in 1 2 13 15; do
-
- echo "* Try: kill -$sig \$\$"
- if test $sig -eq 2; then
- # Some Korn shells might otherwise get a spurious SIGINT
- # signal when one is sent to the child $SHELL.
- trap : 2
- fi
- $SHELL -c ". ./defs; kill -$sig \$\$; :"
- rc=$?
- if test $sig -eq 2; then
- # Reset default SIGINT handler as portably as possible.
- trap 2 || trap - 2
- fi
- echo "* rc=$rc"
- echo
- if test x"$sh_errexit_works" = x"yes"; then
- # The exit trap should turn into an hard errors any failure
- # caused by signals.
- test $rc -eq 99 || exit 1
- else
- # The exit trap is not installed, so that the shell should exit
- # with status 128+n when receiving signal number n. But don't
- # be too strict in the check, as POSIX only says that "The exit
- # status of a command that terminated because it received a
- # signal shall be reported as greater than 128".
- test $rc -gt 128 || exit 1
- fi
-
-done
-
-:
diff --git a/tests/self-check-me.tap b/tests/self-check-me.tap
new file mode 100755
index 0000000..503a1d0
--- /dev/null
+++ b/tests/self-check-me.tap
@@ -0,0 +1,75 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Sanity check for the automake testsuite.
+# Make sure that $me gets automatically defined by `./defs', and that it
+# can be overridden by the test script.
+
+am_create_testdir=no
+. ./defs || Exit 1
+
+plan_ 9
+
+AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC
+set +e
+
+do_check ()
+{
+ if $SHELL -c '. ./defs && echo me=$me' "$1" | grep "^me=$2$"; then
+ r='ok'
+ else
+ r='not ok'
+ fi
+ result_ "$r" "me=$1"
+ unset r
+}
+
+do_check foo-bar-.test 'foo-bar-'
+do_check _foo__bar.test '_foo__bar'
+do_check 012.test '012'
+do_check foo.bar 'foo\.bar'
+do_check a.b.c.test 'a\.b\.c'
+
+# If we override $me, ./defs should not modify it.
+
+if s=`$SHELL -c 'me=foo.test && . ./defs && echo me=$me' bad.test`; then
+ r='ok'
+else
+ r='not ok'
+fi
+result_ "$r" "override of \$me before ./defs causes no error"
+
+r='ok'
+if printf '%s\n' "$s" | grep '^me=foo\.test$'; then :; else r='not ok'; fi
+if printf '%s\n' "$s" | grep 'me=bad'; then r='not ok'; else :; fi
+result_ "$r" "override of \$me before defs is honored"
+
+# Overriding $me after sourcing ./defs-static should work.
+
+if s=`$SHELL -c '. ./defs-static && me=zardoz &&
+ . ./defs && echo me=$me' bad.test`; then
+ r='ok'
+else
+ r='not ok'
+fi
+result_ "$r" "override of \$me after defs-static causes no error"
+
+r='ok'
+if printf '%s\n' "$s" | grep '^me=zardoz$'; then :; else r='not ok'; fi
+if printf '%s\n' "$s" | grep 'me=bad'; then r='not ok'; else :; fi
+result_ "$r" "override of \$me after ./defs-static is honored"
+
+:
diff --git a/tests/self-check-me.test b/tests/self-check-me.test
deleted file mode 100755
index e8b1361..0000000
--- a/tests/self-check-me.test
+++ /dev/null
@@ -1,45 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Sanity check for the automake testsuite.
-# Make sure that $me gets automatically defined by `./defs', and that it
-# can be overridden by the test script.
-
-. ./defs-static || exit 1
-
-set -ex
-
-AM_TESTS_REEXEC=no; export AM_TESTS_REEXEC
-
-$SHELL -c '. ./defs && echo me=$me' foo-bar-.test | grep '^me=foo-bar-$'
-$SHELL -c '. ./defs && echo me=$me' _foo__bar.test | grep '^me=_foo__bar$'
-$SHELL -c '. ./defs && echo me=$me' 012.test | grep '^me=012$'
-$SHELL -c '. ./defs && echo me=$me' foo.bar | grep '^me=foo\.bar$'
-$SHELL -c '. ./defs && echo me=$me' a.b.c.test | grep '^me=a\.b\.c$'
-
-# Overriding $me after sourcing ./defs-static should work.
-s=`$SHELL -c '. ./defs-static && me=zardoz && . ./defs && echo me=$me' \
- bad.test` || exit 1
-printf '%s\n' "$s" | grep '^me=zardoz$'
-printf '%s\n' "$s" | grep 'me=bad' && exit 1
-
-# If we override $me, ./defs should not modify it.
-s=`$SHELL -c 'me=foo.test && . ./defs && echo me=$me' \
- bad.test` || exit 1
-printf '%s\n' "$s" | grep '^me=foo\.test$'
-printf '%s\n' "$s" | grep 'me=bad' && exit 1
-
-:
diff --git a/tests/self-check-tap.test b/tests/self-check-tap.test
new file mode 100755
index 0000000..43a8bac
--- /dev/null
+++ b/tests/self-check-tap.test
@@ -0,0 +1,34 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Sanity check for the automake testsuite.
+# Make sure that $using_tap gets automatically defined by `./defs-static',
+# but can be overridden by the individual tests.
+
+. ./defs-static || exit 1
+
+set -ex
+
+$SHELL -c '. ./defs-static && test $using_tap = yes' foo.tap
+$SHELL -c '. ./defs-static && test $using_tap = no' foo.test
+$SHELL -c '. ./defs-static && test $using_tap = no' tap
+$SHELL -c '. ./defs-static && test $using_tap = no' tap.test
+$SHELL -c '. ./defs-static && test $using_tap = no' foo-tap
+
+$SHELL -c 'using_tap=no; . ./defs-static; test $using_tap = no' foo.tap
+$SHELL -c 'using_tap=yes; . ./defs-static; test $using_tap = yes' foo.test
+
+:
diff --git a/tests/suffix-chain.tap b/tests/suffix-chain.tap
new file mode 100755
index 0000000..680d3f0
--- /dev/null
+++ b/tests/suffix-chain.tap
@@ -0,0 +1,64 @@
+#! /bin/sh
+# Copyright (C) 2002, 2003, 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that Automake can emit code that work round the inability of
+# some make implementations to automatically chain suffix rules.
+# See automake bug#7824 and bug#7670.
+
+required=cc
+. ./defs || Exit 1
+
+plan_ 8
+
+cat >> configure.in <<'END'
+AC_PROG_CC
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+bin_PROGRAMS = foo
+foo_SOURCES = foo.c1
+.c1.c0:
+ (echo 'int main (void)' && echo '{' && cat $<) > $@
+.c0.c:
+ (cat $< && echo '}') > $@
+CLEANFILES = foo.c0 foo.c
+END
+
+echo 'return 0;' > foo.c1
+
+command_ok_ "aclocal" $ACLOCAL
+command_ok_ "automake" $AUTOMAKE
+command_ok_ "autoconf" $AUTOCONF
+command_ok_ "configure" ./configure
+
+directive=''; make_can_chain_suffix_rules || directive=TODO
+
+for target in all distcheck; do
+ command_ok_ "make $target" \
+ -D "$directive" -r "suffix rules not chained" \
+ $MAKE $target
+done
+
+command_ok_ "clean" $MAKE clean
+# Sanity check.
+cat >> Makefile <<'END'
+foo.c: foo.c0
+foo.c0: foo.c1
+END
+command_ok_ "make with explicit dependencies" $MAKE
+
+:
diff --git a/tests/suffix10.tap b/tests/suffix10.tap
new file mode 100755
index 0000000..d71450d
--- /dev/null
+++ b/tests/suffix10.tap
@@ -0,0 +1,68 @@
+#! /bin/sh
+# Copyright (C) 2002, 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Make sure that derivations work with .lo too.
+# (related to PR/37)
+
+required='cc libtoolize yacc'
+. ./defs || Exit 1
+
+cat >>configure.in <<EOF
+AC_PROG_CC
+AC_PROG_YACC
+AC_PROG_LIBTOOL
+AC_OUTPUT
+EOF
+
+cat >Makefile.am << 'END'
+lib_LTLIBRARIES = libfoo.la
+libfoo_la_SOURCES = foo.x_
+.x_.y:
+ rm -f $@ address@hidden
+## The leading `:;' works around a bug in bash <= 3.2.
+ :; { echo '/* autogenerated */' \
+ && echo '%{' \
+ && echo 'int yylex () {return 0;}' \
+ && echo 'void yyerror (char *s) {}' \
+ && echo '%}' \
+ && echo '%%' \
+ && echo 'WORD: "foo";' \
+ && echo '%%' \
+## Account for VPATH issues on weaker make implementations.
+ && cat `test -f '$<' || echo $(srcdir)/`$<; \
+ } > address@hidden
+ mv -f address@hidden $@
+.PHONY: test
+test:
+ echo $(libfoo_la_OBJECTS) | grep '^foo\.lo$$'
+END
+
+cat > foo.x_ << 'END'
+int foo (void) { return yyparse(); }
+END
+
+command_ok_ "libtoolize" libtoolize --force
+command_ok_ "aclocal" $ACLOCAL
+command_ok_ "autoconf" $AUTOCONF
+command_ok_ "automake" $AUTOMAKE --add-missing
+command_ok_ "configure" ./configure
+command_ok_ "make test" $MAKE test
+
+directive=''; make_can_chain_suffix_rules || directive=TODO
+command_ok_ "make all" -D "$directive" -r "suffix rules not chained" \
+ $MAKE all
+
+:
diff --git a/tests/suffix10.test b/tests/suffix10.test
deleted file mode 100755
index 9f36c55..0000000
--- a/tests/suffix10.test
+++ /dev/null
@@ -1,66 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2002, 2010, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Make sure that derivations work with .lo too.
-# (related to PR/37)
-
-required='cc libtoolize yacc'
-. ./defs || Exit 1
-
-cat >>configure.in <<EOF
-AC_PROG_CC
-AC_PROG_YACC
-AC_PROG_LIBTOOL
-AC_OUTPUT
-EOF
-
-cat >Makefile.am << 'END'
-lib_LTLIBRARIES = libfoo.la
-libfoo_la_SOURCES = foo.x_
-.x_.y:
- rm -f $@ address@hidden
-## The leading `:;' works around a bug in bash <= 3.2.
- :; { echo '/* autogenerated */' \
- && echo '%{' \
- && echo 'int yylex () {return 0;}' \
- && echo 'void yyerror (char *s) {}' \
- && echo '%}' \
- && echo '%%' \
- && echo 'WORD: "foo";' \
- && echo '%%' \
-## Account for VPATH issues on weaker make implementations.
- && cat `test -f '$<' || echo $(srcdir)/`$<; \
- } > address@hidden
- mv -f address@hidden $@
-.PHONY: test
-test:
- echo $(libfoo_la_OBJECTS) | grep '^foo\.lo$$'
-END
-
-cat > foo.x_ << 'END'
-int foo (void) { return yyparse(); }
-END
-
-libtoolize --force
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE --add-missing
-
-./configure
-$MAKE test
-$MAKE all
-
-:
diff --git a/tests/suffix11.tap b/tests/suffix11.tap
new file mode 100755
index 0000000..e740f6c
--- /dev/null
+++ b/tests/suffix11.tap
@@ -0,0 +1,82 @@
+#! /bin/sh
+# Copyright (C) 2002, 2003, 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Tests that Automake understands multiple suffix rules on the same line.
+# PR/371 and PR/372: Reported by Duncan Gibson.
+# We also check for suffixes containing `-'.
+
+required=cc
+. ./defs || Exit 1
+
+plan_ 10
+
+cat >>configure.in <<EOF
+AC_PROG_CC
+AC_OUTPUT
+EOF
+
+cat >Makefile.am << 'END'
+bin_PROGRAMS = foo bar baz
+foo_SOURCES = foo.x_
+bar_SOURCES = bar.y-z
+baz_SOURCES = baz1.x_ baz2.y-z
+
+.y-z.c .x_.c:
+## Account for VPATH issues on weaker make implementations (e.g. IRIX 6.5).
+ sed 's/INTEGER/int/g' `test -f '$<' || echo $(srcdir)/`$< >$@
+
+CLEANFILES = foo.c bar.c baz1.c baz2.c
+
+.PHONY: test-real test-fake
+test-fake:
+ echo $(foo_OBJECTS) | grep '^foo\.foo$$'
+ echo $(bar_OBJECTS) | grep '^bar\.foo$$'
+ echo $(baz_OBJECTS) | grep '^baz1\.foo baz2\.foo$$'
+test-real:
+ echo $(foo_OBJECTS) | grep '^foo\.$(OBJEXT)$$'
+ echo $(bar_OBJECTS) | grep '^bar\.$(OBJEXT)$$'
+ echo $(baz_OBJECTS) | grep '^baz1\.$(OBJEXT) baz2\.$(OBJEXT)$$'
+check-local: test-real
+END
+
+echo 'INTEGER main(void) { return 0; }' > foo.x_
+echo 'INTEGER main(void) { return 0; }' > bar.y-z
+echo 'INTEGER main(void) { INTEGER baz(void); return baz(); }' > baz1.x_
+echo 'INTEGER baz(void) { return 0; }' > baz2.y-z
+
+command_ok_ "aclocal" $ACLOCAL
+command_ok_ "autoconf" $AUTOCONF
+
+# What we do is not portable. Automake should warn.
+AUTOMAKE_fails -a
+command_ok_ "warn about unportable make usage" \
+ grep '[iI]nference rules can have only one target before the colon' stderr
+# But this should work anyway.
+command_ok_ "automake" $AUTOMAKE -a -Wno-portability
+
+command_ok_ "configure" ./configure
+command_ok_ "make test-fake" env OBJEXT=foo $MAKE -e test-fake
+command_ok_ "make test-real" $MAKE test-real
+
+directive=''; make_can_chain_suffix_rules || directive=TODO
+
+for target in '' distcheck; do
+ command_ok_ "make $target" \
+ -D "$directive" -r "suffix rules not chained" \
+ $MAKE $target
+done
+
+:
diff --git a/tests/suffix11.test b/tests/suffix11.test
deleted file mode 100755
index 3f0542b..0000000
--- a/tests/suffix11.test
+++ /dev/null
@@ -1,75 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2002, 2003, 2010, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Tests that Automake understands multiple suffix rules on the same line.
-# PR/371 and PR/372: Reported by Duncan Gibson.
-# We also check for suffixes containing `-'.
-
-required=cc
-. ./defs || Exit 1
-
-cat >>configure.in <<EOF
-AC_PROG_CC
-AC_OUTPUT
-EOF
-
-cat >Makefile.am << 'END'
-bin_PROGRAMS = foo bar baz
-foo_SOURCES = foo.x_
-bar_SOURCES = bar.y-z
-baz_SOURCES = baz1.x_ baz2.y-z
-
-.y-z.c .x_.c:
-## Account for VPATH issues on weaker make implementations (e.g. IRIX 6.5).
- sed 's/INTEGER/int/g' `test -f '$<' || echo $(srcdir)/`$< >$@
-
-CLEANFILES = foo.c bar.c baz1.c baz2.c
-
-.PHONY: test-real test-fake
-test-fake:
- echo $(foo_OBJECTS) | grep '^foo\.foo$$'
- echo $(bar_OBJECTS) | grep '^bar\.foo$$'
- echo $(baz_OBJECTS) | grep '^baz1\.foo baz2\.foo$$'
-test-real:
- echo $(foo_OBJECTS) | grep '^foo\.$(OBJEXT)$$'
- echo $(bar_OBJECTS) | grep '^bar\.$(OBJEXT)$$'
- echo $(baz_OBJECTS) | grep '^baz1\.$(OBJEXT) baz2\.$(OBJEXT)$$'
-check-local: test-real
-END
-
-echo 'INTEGER main(void) { return 0; }' > foo.x_
-echo 'INTEGER main(void) { return 0; }' > bar.y-z
-echo 'INTEGER main(void) { INTEGER baz(void); return baz(); }' > baz1.x_
-echo 'INTEGER baz(void) { return 0; }' > baz2.y-z
-
-$ACLOCAL
-$AUTOCONF
-
-# What we do is not portable. Automake should warn.
-AUTOMAKE_fails -a
-grep '[iI]nference rules can have only one target before the colon' stderr
-
-# But this should work anyway.
-$AUTOMAKE -a -Wno-portability
-
-./configure
-
-OBJEXT=foo $MAKE -e test-fake
-$MAKE test-real
-$MAKE
-$MAKE distcheck
-
-:
diff --git a/tests/suffix3.tap b/tests/suffix3.tap
new file mode 100755
index 0000000..f1596ce
--- /dev/null
+++ b/tests/suffix3.tap
@@ -0,0 +1,84 @@
+#! /bin/sh
+# Copyright (C) 1999, 2001, 2002, 2010, 2011 Free Software Foundation,
+# Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test to make sure that suffix rules chain.
+
+required=c++
+. ./defs || Exit 1
+
+plan_ 10
+
+cat >> configure.in << 'END'
+AC_PROG_CXX
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+SUFFIXES = .zoo
+.zoo.cc:
+ sed 's/INTEGER/int/g' `test -f '$<' || echo $(srcdir)/`$< >$@
+bin_PROGRAMS = foo
+foo_SOURCES = foo.zoo
+# This is required by "make distcheck". The useless indirection is
+# reequired to avoid false positives by the grepping checks below.
+FOO = foo
+CLEANFILES = $(FOO).cc
+END
+
+command_ok_ "aclocal" $ACLOCAL
+command_ok_ "automake" $AUTOMAKE
+
+# The foo.cc intermediate step is implicit, it's a mistake if
+# Automake requires this file somewhere. Also, Automake should
+# not require the file `foo.c' anywhere.
+command_ok_ "intermediate files not mentioned" \
+ not $FGREP foo.c Makefile.in
+# However Automake must figure that foo.zoo is eventually
+# transformed into foo.o, and use this latter file (to link foo).
+command_ok_ "final object file figured out" \
+ $FGREP 'foo.$(OBJEXT)' Makefile.in
+
+command_ok_ "autoconf" $AUTOCONF
+command_ok_ "configure" ./configure
+
+# This is deliberately valid C++, but invalid C.
+cat > foo.zoo <<'END'
+#include <iostream>
+using namespace std;
+INTEGER main (void)
+{
+ return 0;
+}
+END
+
+directive=''; make_can_chain_suffix_rules || directive=TODO
+
+for target in all distcheck; do
+ command_ok_ "make $target" \
+ -D "$directive" -r "suffix rules not chained" \
+ $MAKE $target
+done
+
+# FIXME: should we check that intermediate file `foo.cc' has
+# been removed? Or is this requiring too much from the make
+# implementation?
+
+# Intermediate files should not be distributed.
+command_ok_ "make distdir" $MAKE distdir
+command_ok_ "intermediate file not distributed" test ! -r $me-1.0/foo.cc
+
+:
diff --git a/tests/suffix3.test b/tests/suffix3.test
deleted file mode 100755
index 70385fe..0000000
--- a/tests/suffix3.test
+++ /dev/null
@@ -1,76 +0,0 @@
-#! /bin/sh
-# Copyright (C) 1999, 2001, 2002, 2010, 2011 Free Software Foundation,
-# Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test to make sure that suffix rules chain.
-
-required=c++
-. ./defs || Exit 1
-
-cat >> configure.in << 'END'
-AC_PROG_CXX
-AC_OUTPUT
-END
-
-cat > Makefile.am << 'END'
-SUFFIXES = .zoo
-.zoo.cc:
- sed 's/INTEGER/int/g' `test -f '$<' || echo $(srcdir)/`$< >$@
-bin_PROGRAMS = foo
-foo_SOURCES = foo.zoo
-# This is required by "make distcheck". The useless indirection is
-# reequired to avoid false positives by the grepping checks below.
-FOO = foo
-CLEANFILES = $(FOO).cc
-END
-
-$ACLOCAL
-$AUTOMAKE
-
-# The foo.cc intermediate step is implicit, it's a mistake if
-# Automake requires this file somewhere. Also, Automake should
-# not require the file `foo.c' anywhere.
-$FGREP foo.c Makefile.in && Exit 1
-# However Automake must figure that foo.zoo is eventually
-# transformed into foo.o, and use this latter file (to link foo).
-$FGREP 'foo.$(OBJEXT)' Makefile.in
-
-$AUTOCONF
-./configure
-
-# This is deliberately valid C++, but invalid C.
-cat > foo.zoo <<'END'
-#include <iostream>
-using namespace std;
-INTEGER main(void)
-{
- return 0;
-}
-END
-
-$MAKE
-# FIXME: should we check that intermediate file `foo.cc' has
-# been removed? Or is this requiring too much from the make
-# implementation?
-
-# Intermediate files should not be distributed.
-$MAKE distdir
-test ! -r $me-1.0/foo.cc
-
-# Check the distribution.
-$MAKE distcheck
-
-:
diff --git a/tests/suffix8.tap b/tests/suffix8.tap
new file mode 100755
index 0000000..7b07496
--- /dev/null
+++ b/tests/suffix8.tap
@@ -0,0 +1,91 @@
+#! /bin/sh
+# Copyright (C) 2002, 2003, 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test to make sure Automake supports multiple derivations for the
+# same suffix.
+# From PR/37.
+
+required='cc libtoolize'
+. ./defs || Exit 1
+
+plan_ 10
+
+cat >>configure.in <<'END'
+AM_PROG_LIBTOOL
+AC_OUTPUT
+END
+
+cat >Makefile.am << 'END'
+# $(LINK) is not defined automatically by Automake, since the *_SOURCES
+# variables don't contain any known extension (.c, .cc, .f ...),
+# So we need this hack.
+LINK = :
+
+bin_PROGRAMS = foo
+lib_LTLIBRARIES = libfoo.la
+
+foo_SOURCES = foo.x_
+libfoo_la_SOURCES = bar.x_
+
+# The elaborate cp commands below account for VPATH issues on
+# weaker make implementations (e.g. IRIX 6.5).
+.x_.y_:
+ cp `test -f '$<' || echo $(srcdir)/`$< $@
+.y_.o:
+ cp `test -f '$<' || echo $(srcdir)/`$< $@
+.y_.z_:
+ cp `test -f '$<' || echo $(srcdir)/`$< $@
+.z_.lo:
+ cp `test -f '$<' || echo $(srcdir)/`$< $@
+
+# Some make implementations don't remove intermediate files
+# automatically, thus causing "make distcheck" to fail if
+# this is not added.
+MOSTLYCLEANFILES = *.y_ *.z_
+
+.PHONY: test0 test1 test2
+test0:
+ echo $(foo_OBJECTS) | grep '^foo\.foo$$'
+ echo $(libfoo_la_OBJECTS) | grep '^bar\.lo$$'
+test1:
+ echo $(foo_OBJECTS) | grep '^foo\.$(OBJEXT)$$'
+ echo $(libfoo_la_OBJECTS) | grep '^bar\.lo$$'
+test2: $(foo_OBJECTS) $(libfoo_la_OBJECTS)
+ test -f foo.$(OBJEXT)
+ test -f bar.lo
+check-local: test1 test2
+END
+
+echo 'int main (void) { return 0; }' > foo.x_
+echo 'int bar (void) { return 0; }' > bar.x_
+
+command_ok_ "libtoolize" libtoolize
+command_ok_ "aclocal" $ACLOCAL
+command_ok_ "autoconf" $AUTOCONF
+command_ok_ "automake" $AUTOMAKE -a
+command_ok_ "configure" ./configure
+command_ok_ "make test0" env OBJEXT=foo $MAKE -e test0
+command_ok_ "make test1" $MAKE test1
+
+directive=''; make_can_chain_suffix_rules || directive=TODO
+
+for target in test2 all distcheck; do
+ command_ok_ "make $target" \
+ -D "$directive" -r "suffix rules not chained" \
+ $MAKE $target
+done
+
+:
diff --git a/tests/suffix8.test b/tests/suffix8.test
deleted file mode 100755
index 27ac884..0000000
--- a/tests/suffix8.test
+++ /dev/null
@@ -1,86 +0,0 @@
-#! /bin/sh
-# Copyright (C) 2002, 2003, 2010, 2011 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-# Test to make sure Automake supports multiple derivations for the
-# same suffix.
-# From PR/37.
-
-required='cc libtoolize'
-. ./defs || Exit 1
-
-cat >>configure.in <<'END'
-AM_PROG_LIBTOOL
-AC_OUTPUT
-END
-
-cat >Makefile.am << 'END'
-# $(LINK) is not defined automatically by Automake, since the *_SOURCES
-# variables don't contain any known extension (.c, .cc, .f ...),
-# So we need this hack.
-LINK = :
-
-bin_PROGRAMS = foo
-lib_LTLIBRARIES = libfoo.la
-
-foo_SOURCES = foo.x_
-libfoo_la_SOURCES = bar.x_
-
-# The elaborate cp commands below account for VPATH issues on
-# weaker make implementations (e.g. IRIX 6.5).
-.x_.y_:
- cp `test -f '$<' || echo $(srcdir)/`$< $@
-.y_.o:
- cp `test -f '$<' || echo $(srcdir)/`$< $@
-.y_.z_:
- cp `test -f '$<' || echo $(srcdir)/`$< $@
-.z_.lo:
- cp `test -f '$<' || echo $(srcdir)/`$< $@
-
-# Some make implementations don't remove intermediate files
-# automatically, thus causing "make distcheck" to fail if
-# this is not added.
-MOSTLYCLEANFILES = *.y_ *.z_
-
-.PHONY: test0 test1 test2
-test0:
- echo $(foo_OBJECTS) | grep '^foo\.foo$$'
- echo $(libfoo_la_OBJECTS) | grep '^bar\.lo$$'
-test1:
- echo $(foo_OBJECTS) | grep '^foo\.$(OBJEXT)$$'
- echo $(libfoo_la_OBJECTS) | grep '^bar\.lo$$'
-test2: $(foo_OBJECTS) $(libfoo_la_OBJECTS)
- test -f foo.$(OBJEXT)
- test -f bar.lo
-check-local: test1 test2
-END
-
-echo 'int main(void) { return 0; }' > foo.x_
-echo 'int bar(void) { return 0; }' > bar.x_
-
-libtoolize
-$ACLOCAL
-$AUTOCONF
-$AUTOMAKE -a
-
-./configure
-
-OBJEXT=foo $MAKE -e test0
-$MAKE test1
-$MAKE test2
-$MAKE all
-$MAKE distcheck
-
-:
diff --git a/tests/tags2.test b/tests/tags2.test
index aba350b..60e6f46 100755
--- a/tests/tags2.test
+++ b/tests/tags2.test
@@ -36,7 +36,7 @@ cat >>Makefile.am << 'END'
bin_PROGRAMS = bar
END
-AUTOMAKE_run 0
+AUTOMAKE_run
grep 'define.*TAGS_DEPENDENCIES.*without' stderr && Exit 1
:
diff --git a/tests/tap-autonumber.test b/tests/tap-autonumber.test
new file mode 100755
index 0000000..12dec11
--- /dev/null
+++ b/tests/tap-autonumber.test
@@ -0,0 +1,75 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - unnumbered tests are OK, as long as their final total number
+# agrees with the plan
+# - test results without number get automatically numbered in the
+# console progress output
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..14
+ok
+ok foo
+ok - foo2
+not ok
+not ok bar
+not ok - bar2
+; See that we can intermingle different kind of results without
+; messing up the autonumbering
+ok
+ok # TODO
+not ok # TODO who cares?
+ok
+not ok
+ok muuu # SKIP
+not ok
+ok
+END
+
+TESTS=all.test $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=14 pass=6 fail=5 xpass=1 xfail=1 skip=1 error=0
+
+cat > exp <<'END'
+PASS: all.test 1
+PASS: all.test 2 foo
+PASS: all.test 3 - foo2
+FAIL: all.test 4
+FAIL: all.test 5 bar
+FAIL: all.test 6 - bar2
+PASS: all.test 7
+XPASS: all.test 8 # TODO
+XFAIL: all.test 9 # TODO who cares?
+PASS: all.test 10
+FAIL: all.test 11
+SKIP: all.test 12 muuu # SKIP
+FAIL: all.test 13
+PASS: all.test 14
+END
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-bad-prog.tap b/tests/tap-bad-prog.tap
new file mode 100755
index 0000000..8ba153f
--- /dev/null
+++ b/tests/tap-bad-prog.tap
@@ -0,0 +1,83 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - missing, unreadable, or not-executable test scripts cause proper
+# error reports
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+plan_ 5
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TESTS = none.test noread.test noexec.test
+none.test:
+END
+
+cat > noexec.test <<'END'
+#!/bin/sh
+echo 1..1
+ok 1
+END
+
+cp noexec.test noread.test
+chmod a-r noread.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+if $MAKE check >stdout; then r='not ok'; else r='ok'; fi
+cat stdout
+result_ "$r" '"make check" returns non-zero exit status'
+
+# FIXME: maybe grep for stricter error messages in the next checks?
+
+command_ok_ "non-existent test is reported" \
+ grep '^ERROR: none\.test' stdout
+
+desc="non-executable test is reported"
+if ./noexec.test; then
+ skip_ -r "any file is executable" "$desc"
+else
+ command_ok_ "$desc" -- grep '^ERROR: noexec\.test' stdout
+fi
+
+desc="non-readable test is reported"
+if test -r noread.test; then
+ skip_ -r "any file is readable" "$desc"
+else
+ command_ok_ "$desc" -- grep '^ERROR: noread\.test' stdout
+fi
+
+# Check that no spurious test results is reported. This is lower-priority
+# (and in fact the check currently fails.
+command_ok_ 'no spurious results' -D TODO -r 'still get "missing plan"' \
+ count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3
+
+:
diff --git a/tests/tap-bailout.test b/tests/tap-bailout.test
new file mode 100755
index 0000000..8354b27
--- /dev/null
+++ b/tests/tap-bailout.test
@@ -0,0 +1,138 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP test protocol support:
+# - "Bail out!" magic
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+: > exp
+
+#------------------------------------------------------------------
+
+# Bailout without explanation.
+
+cat > a.test <<END
+1..4
+ok 1
+not ok 2
+Bail out!
+not ok 3
+ok 4 # SKIP
+END
+
+cat >> exp <<END
+PASS: a.test 1
+FAIL: a.test 2
+ERROR: a.test - Bail out!
+END
+
+# pass += 1, fail +=1, error += 1
+
+#------------------------------------------------------------------
+
+# Bailout with explanation.
+
+cat > b.test <<END
+1..7
+ok 1 # SKIP
+ok 2 # TODO
+not ok 3 # TODO
+Bail out! We're out of disk space.
+ok 4
+not ok 5
+not ok 6 # TODO
+ok 7 # TODO
+END
+
+cat >> exp <<END
+SKIP: b.test 1 # SKIP
+XPASS: b.test 2 # TODO
+XFAIL: b.test 3 # TODO
+ERROR: b.test - Bail out! We're out of disk space.
+END
+
+# skip += 1, xpass += 1, xfail += 1, error += 1
+
+#------------------------------------------------------------------
+
+# Bail out before the test plan.
+
+cat > c.test <<END
+ok 1
+ok 2
+Bail out! BOOOH!
+1..2
+END
+
+cat >> exp <<END
+PASS: c.test 1
+PASS: c.test 2
+ERROR: c.test - Bail out! BOOOH!
+END
+
+# pass += 2, error += 1
+
+#------------------------------------------------------------------
+
+# Bailout on the first line.
+
+cat > d.test <<END
+Bail out! mktemp -d: Permission denied
+ok 1
+END
+
+echo 'ERROR: d.test - Bail out! mktemp -d: Permission denied' >> exp
+
+# error += 1
+
+#------------------------------------------------------------------
+
+# TAP input comprised only of a bailout directive.
+
+cat > e.test <<END
+Bail out!
+END
+
+echo "ERROR: e.test - Bail out!" >> exp
+
+# error += 1
+
+#------------------------------------------------------------------
+
+# Doing the sums above, we have:
+test_counts='total=12 pass=3 fail=1 xpass=1 xfail=1 skip=1 error=5'
+
+TESTS='a.test b.test c.test d.test e.test' $MAKE -e check >stdout \
+ && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results $test_counts
+
+LC_ALL=C sort exp > t
+mv -f t exp
+
+# We need the sort below to account for parallel make usage.
+grep ': [abcde]\.test' stdout | LC_ALL=C sort > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-basic.test b/tests/tap-basic.test
new file mode 100755
index 0000000..26a749d
--- /dev/null
+++ b/tests/tap-basic.test
@@ -0,0 +1,177 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP support:
+# - LOG_COMPILER support;
+# - basic support for TODO and SKIP directives, and "Bail out!" magic;
+# - testsuite progress output on console;
+# - runtime overriding of TESTS and TEST_LOGS;
+# - correct counts of test results (both in summary and in progress
+# output on console).
+# Note that some of the features checked here are checked in other
+# test cases too, usually in a more thorough and detailed way.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+## Defining LOG_COMPILER should work and not intefere with the
+## tap-driver script.
+TEST_LOG_COMPILER = cat
+TESTS = success.test
+
+ok.test:
+ echo '1..3' > address@hidden
+ echo 'ok 1' >> address@hidden
+ echo 'not ok 2 # TODO' >>address@hidden
+ echo 'ok 3 # SKIP' >>address@hidden
+ cat address@hidden ;: For debugging.
+ mv -f address@hidden $@
+END
+
+cat > success.test << 'END'
+1..20
+ok 1
+ok 2 two
+ok 3 - three
+ok 4 four four
+not ok 5
+not ok 6 six
+not ok 7 - seven
+not ok 8 eight eight
+ok 9 # TODO
+ok 10 ten # TODO
+ok 11 - eleven # TODO
+ok 12 twelve twelve # TODO
+not ok 13 # TODO
+not ok 14 fourteen # TODO
+not ok 15 - fifteen # TODO
+not ok 16 sixteen sixteen # TODO
+ok 17 # SKIP
+ok 18 eighteen # SKIP
+ok 19 - nineteen # SKIP
+ok 20 twenty twenty # SKIP
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# Basilar usage and testsuite progress output.
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=20 pass=4 fail=4 xpass=4 xfail=4 skip=4 error=0
+
+test -f success.log
+test -f test-suite.log
+
+cat > exp << 'END'
+PASS: success.test 1
+PASS: success.test 2 two
+PASS: success.test 3 - three
+PASS: success.test 4 four four
+FAIL: success.test 5
+FAIL: success.test 6 six
+FAIL: success.test 7 - seven
+FAIL: success.test 8 eight eight
+XPASS: success.test 9 # TODO
+XPASS: success.test 10 ten # TODO
+XPASS: success.test 11 - eleven # TODO
+XPASS: success.test 12 twelve twelve # TODO
+XFAIL: success.test 13 # TODO
+XFAIL: success.test 14 fourteen # TODO
+XFAIL: success.test 15 - fifteen # TODO
+XFAIL: success.test 16 sixteen sixteen # TODO
+SKIP: success.test 17 # SKIP
+SKIP: success.test 18 eighteen # SKIP
+SKIP: success.test 19 - nineteen # SKIP
+SKIP: success.test 20 twenty twenty # SKIP
+END
+
+$FGREP ': success.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+# Override TESTS from the command line.
+
+rm -f *.log *.test
+
+cat > bail.test <<'END'
+1..1
+Bail out!
+ok 1
+END
+
+TESTS=bail.test $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
+
+test ! -f success.log
+test -f bail.log
+test -f test-suite.log
+
+grep '^ERROR: bail\.test - Bail out!' stdout
+grep '^PASS:' stdout && Exit 1
+test `$FGREP -c ': bail.test' stdout` -eq 1
+$FGREP 'success.test' stdout && Exit 1
+
+# Override TEST_LOGS from the command line, making it point to a test
+# (ok.test) that have to be generated at make time.
+
+rm -f *.log *.test
+
+TEST_LOGS=ok.log $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=1 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+test -f ok.test
+test -f ok.log
+test ! -f success.log
+test ! -f bail.log
+test -f test-suite.log
+
+$EGREP '(bail|success)\.test' stdout && Exit 1
+
+cat > exp << 'END'
+PASS: ok.test 1
+XFAIL: ok.test 2 # TODO
+SKIP: ok.test 3 # SKIP
+END
+
+$FGREP ': ok.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-color.test b/tests/tap-color.test
new file mode 100755
index 0000000..cd0329b
--- /dev/null
+++ b/tests/tap-color.test
@@ -0,0 +1,167 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - colorization of TAP results and diagnostic messages
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+TERM=ansi; export TERM
+
+esc=''
+# Escape `[' for grep, below.
+red="$esc\[0;31m"
+grn="$esc\[0;32m"
+lgn="$esc\[1;32m"
+blu="$esc\[1;34m"
+mgn="$esc\[0;35m"
+std="$esc\[m"
+
+# Check that grep can parse nonprinting characters.
+# BSD 'grep' works from a pipe, but not a seekable file.
+# GNU or BSD 'grep -a' works on files, but is not portable.
+case `echo "$std" | grep .` in
+ "$std") ;;
+ *) echo "$me: grep can't parse nonprinting characters" >&2; Exit 77;;
+esac
+
+cat > Makefile.am << 'END'
+AUTOMAKE_OPTIONS = color-tests
+AM_TEST_LOG_DRIVER_FLAGS = --comments
+TEST_LOG_COMPILER = cat
+TESTS = all.test skip.test bail.test badplan.test noplan.test \
+ few.test many.test order.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test << 'END'
+1..5
+ok 1 - foo
+# Hi! I shouldn't be colorized!
+not ok 2 - bar # TODO td
+ok 3 - baz # SKIP sk
+not ok 4 - quux
+ok 5 - zardoz # TODO
+END
+
+cat > skip.test << 'END'
+1..0 # SKIP whole script
+END
+
+cat > bail.test << 'END'
+1..1
+ok 1
+Bail out!
+END
+
+cat > badplan.test << 'END'
+1..2
+ok 1
+1..2
+ok 2
+END
+
+cat > noplan.test << 'END'
+ok 1
+END
+
+cat > few.test << 'END'
+1..2
+ok 1
+END
+
+cat > many.test << 'END'
+1..1
+ok 1
+ok 2
+END
+
+cat > order.test << 'END'
+1..1
+ok 5
+END
+
+AM_COLOR_TESTS=always $MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+test_color ()
+{
+ # Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "^${grn}PASS${std}: all\.test 1 - foo$"
+ cat stdout | grep "^${lgn}XFAIL${std}: all\.test 2 - bar # TODO td$"
+ cat stdout | grep "^${blu}SKIP${std}: all\.test 3 - baz # SKIP sk$"
+ cat stdout | grep "^${red}FAIL${std}: all\.test 4 - quux$"
+ cat stdout | grep "^${red}XPASS${std}: all\.test 5 - zardoz # TODO$"
+ cat stdout | grep "^${blu}SKIP${std}: skip\.test - whole script$"
+ cat stdout | grep "^${grn}PASS${std}: bail\.test 1$"
+ cat stdout | grep "^${mgn}ERROR${std}: bail\.test - Bail out!$"
+ cat stdout | grep "^${mgn}ERROR${std}: badplan\.test - multiple test plans$"
+ cat stdout | grep "^${mgn}ERROR${std}: badplan\.test 2 # AFTER LATE PLAN$"
+ cat stdout | grep "^${mgn}ERROR${std}: noplan\.test - missing test plan$"
+ cat stdout | grep "^${mgn}ERROR${std}: few.test - too few tests run
(expected 2, got 1)$"
+ cat stdout | grep "^${mgn}ERROR${std}: many.test - too many tests run
(expected 1, got 2)$"
+ cat stdout | grep "^${mgn}ERROR${std}: many.test 2 # UNPLANNED$"
+ cat stdout | grep "^${mgn}ERROR${std}: order.test 5 # OUT-OF-ORDER
(expecting 1)$"
+ # Diagnostic messages shouldn't be colorized.
+ cat stdout | grep "^# all\.test: Hi! I shouldn't be colorized!$"
+ :
+}
+
+test_no_color ()
+{
+ # With make implementations that, like Solaris make, in case of errors
+ # print the whole failing recipe on standard output, we should content
+ # ourselves with a laxer check, to avoid false positives.
+ # Keep this in sync with lib/am/check.am:$(am__color_tests).
+ if $FGREP '= Xalways || test -t 1 ' stdout; then
+ # Extra verbose make, resort to laxer checks.
+ # But we also want to check that the testsuite summary is not unduly
+ # colorized.
+ (
+ set +e # In case some grepped regex below isn't matched.
+ # Not a useless use of cat; see above comments about grep.
+ cat stdout | grep "TOTAL.*:"
+ cat stdout | grep "PASS.*:"
+ cat stdout | grep "FAIL.*:"
+ cat stdout | grep "SKIP.*:"
+ cat stdout | grep "XFAIL.*:"
+ cat stdout | grep "XPASS.*:"
+ cat stdout | grep "ERROR.*:"
+ cat stdout | grep "^#"
+ cat stdout | grep 'test.*expected'
+ cat stdout | grep 'test.*not run'
+ cat stdout | grep '===='
+ cat stdout | grep '[Ss]ee .*test-suite\.log'
+ cat stdout | grep '[Tt]estsuite summary'
+ ) | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ else
+ cat stdout | grep "$esc" && Exit 1
+ : For shells with broken 'set -e'
+ fi
+}
+
+AM_COLOR_TESTS=always $MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+test_color
+
+$MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+test_no_color
+
+:
diff --git a/tests/tap-common-setup.test b/tests/tap-common-setup.test
new file mode 100755
index 0000000..de8ff2c
--- /dev/null
+++ b/tests/tap-common-setup.test
@@ -0,0 +1,47 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Auxiliary test to set up common data used by many tests on TAP support.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in << END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TEST_LOG_COMPILER = cat
+TESTS = all.test
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+rm -rf autom4te*.cache
+
+# So that the data files we've created won't be removed at exit.
+keep_testdirs=yes
+
+:
diff --git a/tests/tap-deps.test b/tests/tap-deps.test
new file mode 100755
index 0000000..b86d235
--- /dev/null
+++ b/tests/tap-deps.test
@@ -0,0 +1,91 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP test protocol support:
+# - dependencies between test scripts
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+# The tests are *deliberately* listed in inversed order here.
+TESTS = c.test b.test a.test
+b.log: a.log
+c.log: b.log
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > a.test << 'END'
+#!/bin/sh
+echo 1..2
+echo ok 1
+sleep '3' # FIXME: quotes to please maintainer-check
+echo ok 2
+: > a.run
+END
+
+cat > b.test << 'END'
+#!/bin/sh
+echo 1..2
+if test -f a.run; then
+ echo ok 1
+else
+ echo not ok 1
+fi
+sleep '3' # FIXME: quotes to please maintainer-check
+echo ok 2
+: > b.run
+END
+
+cat > c.test << 'END'
+#!/bin/sh
+echo 1..1
+test -f b.run || { echo 'Bail out!'; exit 1; }
+echo ok 1
+rm -f a.run b.run
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=5 pass=5 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+cat > exp << 'END'
+PASS: a.test 1
+PASS: a.test 2
+PASS: b.test 1
+PASS: b.test 2
+PASS: c.test 1
+END
+
+grep ': [abc]\.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+# TODO: it would be nice to also redo the checks forcing parallel make...
+
+:
diff --git a/tests/tap-diagnostic-custom.test b/tests/tap-diagnostic-custom.test
new file mode 100755
index 0000000..d965229
--- /dev/null
+++ b/tests/tap-diagnostic-custom.test
@@ -0,0 +1,96 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - option '--diagnostic-string' to customize the string introducing
+# TAP diagnostics
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+my_log_driver = $(PERL) $(srcdir)/tap-driver
+my_log_compiler = cat
+TEST_EXTENSIONS =
+TESTS =
+END
+
+: > later.mk
+
+# Quoting our comments below is an hack required to keep
+# comments near the things they refer to.
+i=0
+for string in \
+'## A letter' \
+ a \
+'## A number' \
+ 1023 \
+'## A non-alphabetic character' \
+ @ \
+'## Some metacharacters (we need to repeat the "$" for make)' \
+ '^>;&*"|$$' \
+'## A whitespace character' \
+ " " \
+'## A tab character' \
+ "$tab" \
+'## A string with more whitespace' \
+ " ${tab}a b${tab} c" \
+'## Note the we do not have the empty string here. We prefer to' \
+'## leave its behaviour in this context undefined for the moment.'
+do
+ case $string in '##'*) continue;; esac
+ i=`expr $i + 1`
+ unindent >> Makefile.am << END
+ TEST_EXTENSIONS += .t$i
+ TESTS += foo$i.t$i
+ T${i}_LOG_COMPILER = \$(my_log_compiler)
+ T${i}_LOG_DRIVER = \$(my_log_driver)
+ AM_T${i}_LOG_DRIVER_FLAGS = \
+ --comments \
+ --diagnostic-string '$string'
+END
+ unindent > foo$i.t$i <<END
+ 1..1
+ ok 1
+ $string blah blah $i
+END
+ echo "AM_T${i}_LOG_DRIVER_FLAGS = --no-comments" >> later.mk
+done
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=$i pass=$i fail=0 xpass=0 xfail=0 skip=0 error=0
+
+cat later.mk >> Makefile
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+$FGREP 'blah blah' stdout && Exit 1
+
+:
diff --git a/tests/tap-diagnostic.test b/tests/tap-diagnostic.test
new file mode 100755
index 0000000..3c7da79
--- /dev/null
+++ b/tests/tap-diagnostic.test
@@ -0,0 +1,124 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - diagnostic messages (TAP lines with leading "#")
+# - flags '--comments' and '--no-comments' of the TAP test driver
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+metacharacters=\''"\$!&()[]<>#;^?*'
+
+cat > all.test <<END
+1..4
+# Hi! I'm a comment.
+# Tests begin.
+ok 1
+not ok 2 - foo # TODO
+ok 3 - bar # SKIP
+# Tests end.
+ok - zardoz
+# Shell metacharacters here: $metacharacters
+.# Leading characters before "#", not a TAP diagnostic line.
+x # Leading characters before "#", not a TAP diagnostic line.
+ # Leading whitespace before "#", not a TAP diagnostic line.
+${tab}# Leading whitespace before "#", not a TAP diagnostic line.
+ ${tab} # Leading whitespace before "#", not a TAP diagnostic line.
+END
+
+cat > exp <<END
+# all.test: Hi! I'm a comment.
+# all.test: Tests begin.
+PASS: all.test 1
+XFAIL: all.test 2 - foo # TODO
+SKIP: all.test 3 - bar # SKIP
+# all.test: Tests end.
+PASS: all.test 4 - zardoz
+# all.test: Shell metacharacters here: $metacharacters
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+$EGREP -i "#.*all\\.test|a comment|(Tests|Shell) " stdout && Exit 1
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+echo 'AM_TEST_LOG_DRIVER_FLAGS = --comments' >> Makefile
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+$FGREP ' all.test' stdout > got
+cat exp
+cat got
+diff exp got
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+TEST_LOG_DRIVER_FLAGS="--no-comments" $MAKE -e check >stdout \
+ || { cat stdout; Exit 1; }
+cat stdout
+$EGREP -i "#.*all\\.test|a comment|(Tests|Shell) " stdout && Exit 1
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+# The "#"-prepended lines here shouldn't be parsed as test results.
+cat > all.test <<END
+1..1
+ok
+# ok
+#ok
+# not ok
+#not ok
+# Bail out!
+#Bail out!
+# SKIP
+#SKIP
+# TODO
+#TODO
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+# Diagnostic without leading whitespace, or with extra leading whitespace,
+# is ok. Be laxer in the grepping checks, to allow for whitespace
+# normalization by the TAP driver.
+
+ws="[ $tab]"
+ws0p="${ws}*"
+ws1p="${ws}${ws0p}"
+
+cat > all.test <<END
+1..1
+ok 1
+#foo
+#bar${tab}
+# zardoz ${tab}
+# ${tab} ${tab}${tab}foo bar${tab}baz ${tab}
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+grep "^# all.test:${ws0p}foo$" stdout
+grep "^# all.test:${ws0p}bar${ws0p}$" stdout
+grep "^# all.test:${ws1p}zardoz${ws0p}$" stdout
+grep "^# all.test:${ws1p}foo bar${tab}baz${ws0p}$" stdout
+
+test `grep -c '^# all\.test:' stdout` -eq 4
+
+:
diff --git a/tests/tap-doc.test b/tests/tap-doc.test
new file mode 100755
index 0000000..2920d4e
--- /dev/null
+++ b/tests/tap-doc.test
@@ -0,0 +1,92 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that an example given in the documentation really works.
+# See section "Simple Tests" subsection "Script-based Testsuites".
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_PROG_CC
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.sh zardoz.tap bar.sh mu.tap
+TEST_EXTENSIONS = .sh .tap
+TAP_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+END
+
+cat > foo.sh <<'END'
+#!/bin/sh
+exit 0
+END
+
+cat > bar.sh <<'END'
+#!/bin/sh
+exit 77
+END
+
+cat > zardoz.tap << 'END'
+#!/bin/sh
+echo 1..4
+echo 'ok 1 - Daemon started'
+echo 'ok 2 - Daemon responding'
+echo 'ok 3 - Daemon uses /proc # SKIP /proc is not mounted'
+echo 'ok 4 - Daemon stopped'
+END
+
+cat > mu.tap << 'END'
+#!/bin/sh
+echo 1..2
+echo 'ok'
+echo 'not ok # TODO frobnication not yet implemented'
+END
+
+chmod a+x *.sh *.tap
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+cat > exp <<'END'
+PASS: foo.sh
+PASS: zardoz.tap 1 - Daemon started
+PASS: zardoz.tap 2 - Daemon responding
+SKIP: zardoz.tap 3 - Daemon uses /proc # SKIP /proc is not mounted
+PASS: zardoz.tap 4 - Daemon stopped
+SKIP: bar.sh
+PASS: mu.tap 1
+XFAIL: mu.tap 2 # TODO frobnication not yet implemented
+END
+
+sed -n '/^PASS: foo\.sh/,/^XFAIL: mu\.tap/p' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-doc2.test b/tests/tap-doc2.test
new file mode 100755
index 0000000..45d09ca
--- /dev/null
+++ b/tests/tap-doc2.test
@@ -0,0 +1,135 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that an example given in the documentation really works.
+# See section "Using the TAP test protocol", subsection "Use TAP
+# with the Automake test harness".
+
+am_create_testdir=empty
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am <<'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/build-aux/tap-driver
+TESTS = foo.test bar.test baz.test
+EXTRA_DIST = $(TESTS)
+END
+
+cat > configure.ac <<'END'
+AC_INIT([GNU Try Tap], [1.0], address@hidden)
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE([foreign parallel-tests -Wall -Werror])
+AC_CONFIG_FILES([Makefile])
+AC_REQUIRE_AUX_FILE([tap-driver])
+AC_PATH_PROG([PERL], [perl])
+test -n "$PERL" || AC_MSG_ERROR([perl not found])
+$PERL -MTAP::Parser -e 1 || AC_MSG_ERROR([TAP::Parser not found])
+AC_OUTPUT
+END
+
+cat > foo.test <<'END'
+#!/bin/sh
+echo 1..4 # Number of tests to be executed.
+echo 'ok 1 - Swallows fly'
+echo 'not ok 2 - Caterpillars fly # TODO metamorphosis in progress'
+echo 'ok 3 - Pigs fly # SKIP not enough acid'
+echo '# I just love word plays...'
+echo 'ok 4 - Flies fly too :-)'
+END
+
+cat > bar.test <<'END'
+#!/bin/sh
+echo 1..3
+echo 'not ok 1 - Bummer, this test has failed.'
+echo 'ok 2 - This passed though.'
+echo 'Bail out! Ennui kicking in, sorry...'
+echo 'ok 3 - This will not be seen.'
+END
+
+cat > baz.test <<'END'
+#!/bin/sh
+echo 1..1
+echo ok 1
+# Exit with error, even if all the test case has been successful.
+exit 7
+END
+
+chmod a+x *.test
+
+mkdir build-aux
+cp "$top_testsrcdir"/lib/tap-driver build-aux
+
+(export AUTOMAKE ACLOCAL AUTOCONF && $AUTORECONF -vi) || Exit 1
+
+./configure --help # Sanity check.
+./configure || skip_ "configure failed"
+
+case $MAKE in *\ -j*) skip_ "can't work easily with concurrent make";; esac
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+cat > exp <<'END'
+PASS: foo.test 1 - Swallows fly
+XFAIL: foo.test 2 - Caterpillars fly # TODO metamorphosis in progress
+SKIP: foo.test 3 - Pigs fly # SKIP not enough acid
+PASS: foo.test 4 - Flies fly too :-)
+FAIL: bar.test 1 - Bummer, this test has failed.
+PASS: bar.test 2 - This passed though.
+ERROR: bar.test - Bail out! Ennui kicking in, sorry...
+PASS: baz.test 1
+ERROR: baz.test - exited with status 7
+END
+
+sed -n '/^PASS: foo\.test/,/^ERROR: baz\.test/p' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+grep '^Please report to address@hidden' stdout
+
+env \
+ TESTS='foo.test baz.test' \
+ TEST_LOG_DRIVER_FLAGS='--comments --ignore-exit' \
+ $MAKE -e check >stdout || { cat stdout; Exit 1; }
+
+cat > exp <<'END'
+PASS: foo.test 1 - Swallows fly
+XFAIL: foo.test 2 - Caterpillars fly # TODO metamorphosis in progress
+SKIP: foo.test 3 - Pigs fly # SKIP not enough acid
+# foo.test: I just love word plays...
+PASS: foo.test 4 - Flies fly too :-)
+PASS: baz.test 1
+END
+
+sed -n '/^PASS: foo\.test/,/^PASS: baz\.test/p' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+# Sanity check the distribution.
+cat > bar.test <<'END'
+#!/bin/sh
+echo 1..1
+echo ok 1
+END
+echo AM_TEST_LOG_DRIVER_FLAGS = --ignore-exit >> Makefile.in
+./config.status Makefile
+$MAKE distcheck
+
+:
diff --git a/tests/tap-empty-diagnostic.test b/tests/tap-empty-diagnostic.test
new file mode 100755
index 0000000..2dcc880
--- /dev/null
+++ b/tests/tap-empty-diagnostic.test
@@ -0,0 +1,47 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - empty diagnostic messages are discarder
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+sed 's/\$$//' > all.test <<END
+1..1$
+ok 1$
+#$
+# $
+#${tab}$
+# ${tab} $tab${tab}$
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+grep '^PASS:.*all\.test' stdout # Sanity check.
+grep '#.*all\.test' stdout && Exit 1
+grep "all\.test[ $tab]*:[ $tab]*$" stdout && Exit 1
+
+:
diff --git a/tests/tap-empty.test b/tests/tap-empty.test
new file mode 100755
index 0000000..557b7c7
--- /dev/null
+++ b/tests/tap-empty.test
@@ -0,0 +1,55 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: the following situations should be flagged as errors:
+# - empty TAP input
+# - blank TAP input
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Empty TAP input.
+: > empty.test
+
+# Blank TAP input (one empty line).
+echo > blank.test
+
+# Blank TAP input (one whitespace-only line).
+cat > white.test <<END
+ ${tab}
+END
+
+# Blank TAP input (few blank and whitespace-only lines).
+cat > white2.test <<END
+
+
+ ${tab} ${tab}${tab}
+
+${tab}
+
+END
+
+for input in empty blank white white2; do
+ cp $input.test all.test
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
+ grep '^ERROR: all\.test - missing test plan$' stdout
+done
+
+:
diff --git a/tests/tap-escape-directive.test b/tests/tap-escape-directive.test
new file mode 100755
index 0000000..2f8cb73
--- /dev/null
+++ b/tests/tap-escape-directive.test
@@ -0,0 +1,45 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - "escape" TODO and SKIP directives (by escaping the "#" character)
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..2
+ok \# TODO
+ok \# SKIP
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+grep '^PASS: all\.test 1 .*TODO' stdout
+grep '^PASS: all\.test 2 .*SKIP' stdout
+
+:
diff --git a/tests/tap-exit.test b/tests/tap-exit.test
new file mode 100755
index 0000000..381adff
--- /dev/null
+++ b/tests/tap-exit.test
@@ -0,0 +1,54 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - an exit status != 0 of a test script causes an hard error
+# - the `--ignore-exit' option causes the TAP test driver to ignore
+# exit statuses of the test scripts.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+echo TESTS = > Makefile.am
+for st in 1 2 77 99; do
+ unindent > exit${st}.test <<END
+ #!/bin/sh
+ echo 1..1
+ echo ok 1
+ exit $st
+END
+ echo TESTS += exit${st}.test >> Makefile.am
+done
+
+chmod a+x *.test
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=8 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=4
+
+grep '^ERROR: exit1\.test - exited with status 1$' stdout
+grep '^ERROR: exit2\.test - exited with status 2$' stdout
+grep '^ERROR: exit77\.test - exited with status 77$' stdout
+grep '^ERROR: exit99\.test - exited with status 99$' stdout
+
+echo TEST_LOG_DRIVER_FLAGS = --ignore-exit >> Makefile
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=4 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+:
diff --git a/tests/tap-fancy.test b/tests/tap-fancy.test
new file mode 100755
index 0000000..3348db7
--- /dev/null
+++ b/tests/tap-fancy.test
@@ -0,0 +1,133 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: some unusual forms for valid TAP input.
+# See also related test 'tap-fancy2.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+#
+# From manpage Test::Harness::TAP(3):
+#
+# ``Lines written to standard output matching /^(not )?ok\b/ must be
+# interpreted as test lines. All other lines must not be considered
+# test output.''
+#
+# Unfortunately, the exact format of TODO and SKIP directives is not as
+# clearly described in that manpage; but a simple reverse-engineering of
+# the prove(1) utility shows that it is probably given by the perl regex
+# /#\s*(TODO|SKIP)\b/.
+#
+
+cat > all.test <<END
+1..21
+
+ok? a question
+not ok? a question
+
+ok+plus
+not ok+plus
+
+ok-minus
+not ok-minus
+
+ok#55
+not ok#55
+
+ok${tab} ${tab}9
+ok ${tab}${tab} 10
+
+not ok${tab} ${tab}11
+not ok ${tab}${tab} 12
+
+ok# SKIP
+ok${tab}#SKIP--who cares?
+ok?#SKIP!
+ok!#SKIP?
+
+not ok# TODO
+not ok${tab}#TODO--who cares?
+not ok?#TODO!
+not ok!#TODO?
+
+ok~#TODO
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=21 pass=6 fail=6 xfail=4 xpass=1 skip=4 error=0
+
+#
+# "Weird" characters support.
+#
+
+# The "#" character might cause confusion w.r.t. TAP directives (TODO,
+# SKIP), so we don't attempt to use it.
+weirdchars=\''"$!&()[]<>;^?*/@%=,.:'
+
+cat > all.test <<END
+1..6
+ok $weirdchars
+not ok $weirdchars
+ok $weirdchars # TODO
+not ok $weirdchars # TODO
+ok $weirdchars # SKIP
+Bail out! $weirdchars
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=6 pass=1 fail=1 xfail=1 xpass=1 skip=1 error=1
+$FGREP "PASS: all.test 1 $weirdchars" stdout
+$FGREP "FAIL: all.test 2 $weirdchars" stdout
+$FGREP "XPASS: all.test 3 $weirdchars" stdout
+$FGREP "XFAIL: all.test 4 $weirdchars" stdout
+$FGREP "SKIP: all.test 5 $weirdchars" stdout
+$FGREP "ERROR: all.test - Bail out! $weirdchars" stdout
+
+#
+# Trailing backslashes does not confuse the parser.
+#
+
+bs='\'
+
+cat > all.test <<END
+1..6
+ok $bs
+not ok $bs
+ok # TODO $bs
+not ok # TODO $bs
+ok # SKIP $bs
+Bail out! $bs
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=6 pass=1 fail=1 xfail=1 xpass=1 skip=1 error=1
+grep '^PASS: all\.test 1 \\$' stdout
+grep '^FAIL: all\.test 2 \\$' stdout
+grep '^XPASS: all\.test 3 # TODO \\$' stdout
+grep '^XFAIL: all\.test 4 # TODO \\$' stdout
+grep '^SKIP: all\.test 5 # SKIP \\$' stdout
+grep '^ERROR: all\.test - Bail out! \\$' stdout
+
+:
diff --git a/tests/tap-fancy2.test b/tests/tap-fancy2.test
new file mode 100755
index 0000000..ddd64ea
--- /dev/null
+++ b/tests/tap-fancy2.test
@@ -0,0 +1,144 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: more unusual forms for valid TAP input.
+# See also related test 'tap-fancy.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+#
+# From manpage Test::Harness::TAP(3):
+#
+# ``Lines written to standard output matching /^(not )?ok\b/ must be
+# interpreted as test lines. All other lines must not be considered
+# test output.''
+#
+# Unfortunately, the exact format of TODO and SKIP directives is not as
+# clearly described in that manpage; but a simple reverse-engineering of
+# the prove(1) utility shows that it is probably given by the perl regex
+# /#\s*(TODO|SKIP)\b/.
+#
+
+# To avoid problems with backslashes in echo arguments.
+xecho () { printf '%s\n' "$*"; }
+
+# There are 34 values for $str ...
+for str in \
+ \' \
+ '"' \
+ '`' \
+ '#' \
+ '$' \
+ '!' \
+ '\' \
+ '/' \
+ '&' \
+ '%' \
+ '(' \
+ ')' \
+ '|' \
+ '^' \
+ '~' \
+ '?' \
+ '*' \
+ '+' \
+ '-' \
+ ',' \
+ ':' \
+ ';' \
+ '=' \
+ '<' \
+ '>' \
+ '@' \
+ '[' \
+ ']' \
+ '{' \
+ '}' \
+ '\\' \
+ '...' \
+ '?[a-zA-Z0-9]*' \
+ '*.*' \
+; do
+ # ... each of them add 1 pass, 1 fail, ...
+ xecho "ok${str}"
+ xecho "not ok${str}"
+ # ... and (generally) 4 skips, 4 xfails, and 4 xpasses ...
+ for settings in \
+ 'result="ok" directive=SKIP' \
+ 'result="not ok" directive=TODO' \
+ 'result="ok" directive=TODO' \
+ ; do
+ eval "$settings"
+ xecho "${result}# ${directive}${str}"
+ # ... but 6 skips, 6 xpasses and 6 xfails are to be removed, since
+ # they might not work with $str = '#' or $str = '\' ...
+ if test x"$str" != x'#' && test x"$str" != x'\'; then
+ xecho "${result}${str}#${directive}"
+ xecho "${result}${str}# ${tab}${tab} ${directive}"
+ xecho "${result}${str}#${directive}${str}"
+ fi
+ done
+done > all.test
+
+# Sanity check against a previous use of unportable usages of backslashes
+# with the "echo" builtin.
+if grep '[^\\]\\#' all.test; then
+ framework_failure_ "writing backslashes in all.test"
+fi
+
+# ... so that we finally have:
+pass=34
+fail=34
+xfail=130 # = 4 * 34 - 6
+xpass=130 # = 4 * 34 - 6
+skip=130 # = 4 * 34 - 6
+error=0
+total=`expr $pass + $fail + $xfail + $xpass + $skip`
+
+# Even nastier! But accordingly to the specifics, it should still work.
+for result in 'ok' 'not ok'; do
+ echo "${result}{[(<#${tab}TODO>)]}" >> all.test
+done
+echo "ok{[(<#${tab}SKIP>)]}" >> all.test
+
+# We have to update some test counts.
+xfail=`expr $xfail + 1`
+xpass=`expr $xpass + 1`
+skip=`expr $skip + 1`
+total=`expr $total + 3`
+
+# And add the test plan!
+echo 1..$total >> all.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+$EGREP '^(PASS|FAIL|SKIP).*#.*TODO' stdout && Exit 1
+$EGREP '^X?(PASS|FAIL).*#.*SKIP' stdout && Exit 1
+
+count_test_results total=$total pass=$pass fail=$fail skip=$skip \
+ xpass=$xpass xfail=$xfail error=$error
+
+:
diff --git a/tests/tap-functions.sh b/tests/tap-functions.sh
new file mode 100644
index 0000000..603b53c
--- /dev/null
+++ b/tests/tap-functions.sh
@@ -0,0 +1,232 @@
+# -*- shell-script -*-
+#
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Helper functions used by TAP-producing tests of the Automake testsuite.
+
+#
+# IMPORTANT: All the functions defined in this file can *not* be used
+# from within a subshell, unless explicitly noted otherwise.
+#
+
+# The count of the TAP test results seen so far.
+tap_count_=0
+# The count of skipped tests.
+tap_skip_count_=0
+# The count of tests that experienced an expected failure.
+tap_xfail_count_=0
+# The count of tests with unexpected outcomes (i.e., failed and xpassed).
+tap_bad_count_=0
+
+# The first "test -n" tries to avoid extra forks when possible.
+if test -n "${ZSH_VERSION}${BASH_VERSION}" \
+ || (eval 'test $((1 + 1)) = 2') >/dev/null 2>&1
+then
+ # Outer use of 'eval' needed to protect dumber shells from parsing
+ # errors.
+ eval 'incr_ () { eval "$1=\$((\${$1} + 1))"; }'
+else
+ incr_ () { eval "$1=\`expr \${$1} + 1\`"; }
+fi
+
+# not COMMAND [ARGS...]
+# ---------------------
+# Run the given command and invert its exit status.
+not ()
+{
+ if "$@"; then return 1; else return 0; fi
+}
+
+# plan_ [unknown|later|lazy|now|NUMBER-OF-PLANNED-TESTS]
+# ------------------------------------------------------
+# Print a TAP plan for the given number of tests. This must be called
+# before reporting any test result. If called with the special argument
+# "unknown" or "later", it will do nothing, expecting the calling script
+# to declare the plan later. If called with the special argument "lazy"
+# or "now", it will print a TAP plan that accounts for the number of tests
+# seen so far.
+plan_ ()
+{
+ if test $# -eq 0; then
+ bailout_ "plan_: missing argument"
+ elif test $# -ge 2; then
+ bailout_ "plan_: too many arguments"
+ elif test x"$1" = x"unknown" || test x"$1" = x"later"; then
+ : No-op.
+ elif test x"$1" = x"lazy" || test x"$1" = x"now"; then
+ echo "1..$tap_count_" # Number of test results seen so far.
+ have_tap_plan_=yes
+ elif test $1 -ge 0; then
+ echo "1..$1"
+ have_tap_plan_=yes
+ else
+ bailout_ "plan_: invalid argument '$1'"
+ fi
+}
+have_tap_plan_=no # Avoid interferences from the environment.
+
+# diag_ [EXPLANATION]
+# ------------------
+# Report the given text as TAP diagnostic. Assumes the string denoting
+# TAP diagnostic lines is stored in the `$diag_string_' variable; this is
+# done to allow better interplay with TAP drivers that allow such a string
+# to be configured.
+diag_ ()
+{
+ test $# -eq 0 || echo "$diag_string_ $*"
+}
+
+# Used by the `diag_' function above. User-overridable.
+diag_string_="#"
+
+# warn_ [EXPLANATION]
+# ------------------
+# Give a warning (using TAP diagnostic).
+warn_ ()
+{
+ diag_ "WARNING:" ${1-"(unknown warning)"} ${1+"$@"}
+}
+
+# result_ RESULT [-D DIRECTIVE] [-r REASON] [--] [DESCRIPTION...]
+# ---------------------------------------------------------------
+# Report a test case with the given RESULT (valid values are "ok" and
+# "not ok") and the given DESCRIPTION (if any). If DIRECTIVE is given
+# and non-empty (valid values being "TODO" and "SKIP"), it will be
+# reported too, with the REASON (if given) appended.
+result_ ()
+{
+ set +x # Don't pollute the log files.
+ test $# -gt 0 || bailout_ "result_: missing argument"
+ tap_result_=$1; shift
+ case $tap_result_ in
+ "ok"|"not ok") ;;
+ *) bailout_ "result_: invalid result '$tap_result'" ;;
+ esac
+ tap_directive_= tap_reason_=
+ while test $# -gt 0; do
+ case $1 in
+ -D|--directive) tap_directive_=$2; shift;;
+ -r|--reason) tap_reason_=$2; shift;;
+ --) shift; break;;
+ -*) bailout_ "result_: invalid option '$1'";;
+ *) break;;
+ esac
+ shift
+ done
+ case $tap_directive_ in
+ ""|TODO|SKIP) ;;
+ *) bailout_ "result_: invalid directive '$directive_'" ;;
+ esac
+ incr_ tap_count_
+ case $tap_result_,$tap_directive_ in
+ ok,) ;; # Passed.
+ not\ ok,TODO) incr_ tap_xfail_count_;; # Expected failure.
+ not\ ok,*|ok,TODO) incr_ tap_bad_count_ ;; # Failed or xpassed.
+ ok,SKIP) incr_ tap_skip_count_ ;; # Skipped.
+ *) bailout_ "internal error in 'result_'";; # Can't happen.
+ esac
+ tap_text_="$tap_result_ $tap_count_"
+ if test x"$*" != x; then
+ tap_text_="$tap_text_ - $*"
+ fi
+ if test x"$tap_directive_" != x; then
+ tap_text_="$tap_text_ # $tap_directive_"${tap_reason_:+" $tap_reason_"}
+ fi
+ printf '%s\n' "$tap_text_"
+ set -x # Restore shell xtraces.
+}
+
+# Shorthands for common usages of `result_'.
+ok_ () { result_ 'ok' ${1+"$@"}; }
+not_ok_ () { result_ 'not ok' ${1+"$@"}; }
+skip_ () { result_ 'ok' -D SKIP ${1+"$@"}; }
+
+# skip_row_ COUNT [-r REASON] [--] [DESCRIPTION...]
+# -------------------------------------------------
+# Report a COUNT of skipped test, with the given reason and descriptions
+# (if any). Useful to avoid cascade failures in case a fair number of
+# tests depend on an earlier one that failed.
+skip_row_ ()
+{
+ skip_count_=$1; shift
+ for i_ in `seq $skip_count_`; do skip_ ${1+"$@"}; done
+}
+
+# skip_all_ [REASON ...]
+# ----------------------
+# Skip all the tests in a test script. Must be used before calling `plan_'
+# or reporting any test result. Can't be used from within a subshell.
+skip_all_ ()
+{
+ echo "1..0 # SKIP" ${1+"$@"}
+ have_tap_plan_=yes
+ Exit 0
+}
+
+# bailout_ [REASON ...]
+# ---------------------
+# Stop the execution of the current test suite right now, due to an
+# unrecoverable error. Can be called at any point, but cannot be used
+# from within a subshell.
+bailout_ ()
+{
+ echo 'Bail out!' ${1+"$@"}
+ Exit 99
+}
+
+# fatal_ [REASON ...]
+# -------------------
+# Same as `bailout_'; for compatibility with `plain-functions.sh'.
+fatal_ ()
+{
+ bailout_ ${1+"$@"}
+}
+
+# framework_failure_ [REASON ...]
+# -------------------------------
+# Stop the execution of the current test suite right now, due to an
+# unrecoverable error in the set-up of the test case. Can be called
+# at any point, but cannot be used from within a subshell.
+framework_failure_ ()
+{
+ bailout_ "set-up failure"${1+": $*"}
+}
+
+# command_ok_ TEST-DESCRIPTION [OPTIONS..] [--] CMD [ARGS...]
+# -----------------------------------------------------------
+# Helper subroutine for when a TAP result must be determined by the
+# outcome of a command.
+command_ok_ ()
+{
+ tap_directive_= tap_reason_=
+ test $# -gt 0 || bailout_ "command_ok_: missing argument"
+ tap_description_=$1; shift
+ while test $# -gt 0; do
+ case $1 in
+ -D|--directive) tap_directive_=$2; shift;;
+ -r|--reason) tap_reason_=$2; shift;;
+ --) shift; break;;
+ -*) bailout_ "command_ok_: invalid option '$1'";;
+ *) break;;
+ esac
+ shift
+ done
+ tap_result_="ok"; "$@" || tap_result_="not ok"
+ result_ "$tap_result_" -D "$tap_directive_" -r "$tap_reason_" \
+ -- "$tap_description_"
+}
+
+:
diff --git a/tests/tap-global-log.test b/tests/tap-global-log.test
new file mode 100755
index 0000000..d343581
--- /dev/null
+++ b/tests/tap-global-log.test
@@ -0,0 +1,122 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - which log files get copied in the global log?
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > ok.test << 'END'
+1..5
+ok 1
+ok 2
+ok 3
+not seen in global log
+ok 4
+ok 5
+END
+
+cat > top << 'END'
+1..6
+ok 1
+ok 2
+ok 3
+END
+
+cat > bot << 'END'
+ok 5
+ok 6
+END
+
+cat top - bot > skip.test << 'END'
+ok # SKIP
+::skip::
+END
+
+cat top - bot > todo.test << 'END'
+not ok # TODO
+::todo::
+END
+
+cat top - bot > fail.test << 'END'
+not ok
+::fail::
+END
+
+cat top - bot > xpass.test << 'END'
+ok # TODO
+::xpass::
+END
+
+cat top - bot > bail.test << 'END'
+::bail::
+Bail out!
+END
+
+cat top - bot > error.test << 'END'
+::error::
+1..7
+END
+
+# Created with "dd if=/dev/urandom count=1 | base64-encode"
+cat > hodgepodge <<'END'
+1+0 records in
+1+0 records out
+512 bytes (512 B) copied, 0.000241092 s, 2.1 MB/s
+gdivnV4VhL4DOzhE3zULJuun3PwqqQqMdATVcZbIQkNgyRvNBoqqHMBQEs7QsjDbp2nK+Szz
+EcelGyvXmHrW7yImaom6Yrg95k31VCmp/pGDRnTDwDrcOPJiv9jDReEmTAQkPuqLO+mFNly+
+DDHM9fNbzGumstsQ3wq3DOXz1pCV3JXwhjeaHSboeEbmr55bX0XHLSKaecctA0rXDXEyZWZ/
+ODlawSrAXzw0H7a+xBwjnNXZ3zYiwk3x+WQrPqNjb+qXiLLTxAKzx2/KnaFhxkPlte5jPRNB
+FciDolL+H/10LsCdSzLOrGnY2zH6vL2JMZfxjnb73zWFcdWWE01LTD7wpN5O1MP3+N47lcVe
+lWbkD04cJvhwxLElYSO24B743GG5EyGYt9SeZRE6xbgwq3fVOS8KqjwGxwi4adSBTtw0CV8W
+S/6n8dck1vBvjA+qpk0zMSYSqc3+jzW9UiGTmTEIwfw80p/lGNsfjcNBJ86nFkWUnHmrsi8k
+Dv57sK70mTg239g08f5Uvdga/5UreMBSgB0hUj5sbq57r7B1fsVr0Kag468la8zKy3ZEZ0gX
+++sbaJ9WGHhnKvjooeH+4Y6HwAFsdINde++FlCmp4EuNKKEEuXbSKLaOTy3+6pJ2DYdvRCL+
+frZwxH4hcrw8qh+8IakB02viewZS/qT57v4=
+END
+
+exec 5>misc.test
+echo 'ok # SKIP' >&5
+cat hodgepodge >&5
+echo 'not ok # TODO' >&5
+echo 'ok' >&5
+exec 5>&-
+
+cat > skipall.test << 'END'
+1..0 # SKIP all
+END
+
+# We don't care about the exit status of "make check" here.
+TESTS="`echo *.test`" $MAKE -e check || :
+cat test-suite.log
+
+grep ':.*ok|not seen' test-suite.log && Exit 1
+
+for s in skip todo fail xpass bail error; do
+ $FGREP "::$s::" test-suite.log
+done
+
+grep '^1\.\.0 # SKIP all$' test-suite.log
+
+case `cat test-suite.log` in
+ *"`cat hodgepodge`"*) ;;
+ *) Exit 1;;
+esac
+
+:
diff --git a/tests/tap-global-result.test b/tests/tap-global-result.test
new file mode 100755
index 0000000..32e2401
--- /dev/null
+++ b/tests/tap-global-result.test
@@ -0,0 +1,120 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - which global test result derives from different test results
+# mixed in a single script?
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > ok.test <<END
+1..3
+ok 1
+not ok 2 # TODO
+ok 3 # SKIP
+END
+
+cat > skip.test <<'END'
+1..3
+ok 1 # SKIP
+ok 2 # SKIP
+ok 3 # SKIP
+END
+
+cat > skipall.test <<'END'
+1..0 # SKIP
+not ok 1
+END
+
+cat > fail.test <<'END'
+1..1
+not ok 1
+END
+
+(sed '1s/.*/1..4/' ok.test && echo 'not ok 4') > fail2.test
+
+cat > xpass.test <<'END'
+1..1
+ok 1 # TODO
+END
+
+(sed '1s/.*/1..4/' ok.test && echo 'ok 4 # TODO') > xpass2.test
+
+echo 'Bail out!' > error.test
+(cat ok.test && echo 'Bail out!') > error2.test
+
+cat > hodgepodge.test <<'END'
+1..2
+not ok 1
+ok 2 # TODO
+Bail out!
+END
+
+cat > hodgepodge-all.test <<'END'
+1..4
+ok 1
+ok 2 # SKIP
+not ok 2 # TODO
+not ok 3
+ok 4 # TODO
+Bail out!
+END
+
+# TODO: add scripts with TAP errors (multiple plans, out-of-order
+# tests, etc).
+
+tests=`echo *.test` # Also required later.
+
+TESTS="$tests" $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+# Dirty tricks required here.
+for tst in ok skip skipall fail fail2 xpass xpass2 error error2 \
+ hodgepodge hodgepodge-all; do
+ echo :copy-in-global-log: yes >> $tst.trs
+done
+
+rm -f test-suite.log
+TESTS="$tests" $MAKE -e test-suite.log && Exit 1
+cat test-suite.log
+
+have_rst_section ()
+{
+ eqeq=`echo "$1" | sed 's/./=/g'`
+ # Assume $1 contains no RE metacharacters.
+ sed -n "/^$1$/,/^$eqeq$/p" test-suite.log > got
+ (echo "$1" && echo "$eqeq") > exp
+ cat exp
+ cat got
+ diff exp got
+}
+
+have_rst_section 'PASS: ok'
+have_rst_section 'SKIP: skip'
+have_rst_section 'SKIP: skipall'
+have_rst_section 'FAIL: fail'
+have_rst_section 'FAIL: fail2'
+have_rst_section 'FAIL: xpass'
+have_rst_section 'FAIL: xpass2'
+have_rst_section 'ERROR: error'
+have_rst_section 'ERROR: error2'
+have_rst_section 'ERROR: hodgepodge'
+have_rst_section 'ERROR: hodgepodge-all'
+
+:
diff --git a/tests/tap-html.test b/tests/tap-html.test
new file mode 100755
index 0000000..b5a1beb
--- /dev/null
+++ b/tests/tap-html.test
@@ -0,0 +1,80 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - "check-html" and "recheck-html" targets
+# - reStructuredText -> HTML conversion
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_COMPILER = cat
+TESTS = ok.test ko.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > ok.test << 'END'
+1..3
+ok 1 - expected success
+not ok 2 - expectred failure # TODO
+ok 3 - skipped test # SKIP
+END
+
+cat > ko.test << 'END'
+1..2
+not ok 1 - unexpected failure
+ok 2 - spurious success # TODO
+Bail out! hard error
+END
+
+no_rst2html_error ()
+{
+ $EGREP '(test-suite\.log:[0-9]|WARNING|SEVERE)' output && Exit 1
+ :
+}
+
+# Overriding TESTS should work with check-html.
+TESTS=ok.test $MAKE -e check-html >output 2>&1 || { cat output; Exit 1; }
+cat output
+no_rst2html_error
+test -f test-suite.html
+
+mv output stdout
+count_test_results total=3 pass=1 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+: > older
+$sleep
+$MAKE check-html >output 2>&1 && { cat output; Exit 1; }
+cat output
+no_rst2html_error
+# test-suite.html should be remake if if it was up-to-date.
+test -f test-suite.html
+is_newest test-suite.html older
+
+mv output stdout
+count_test_results total=6 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=1
+
+rm -f ok.test
+$MAKE recheck-html >output 2>&1 && { cat output; Exit 1; }
+cat output
+no_rst2html_error
+
+mv output stdout
+count_test_results total=3 pass=0 fail=1 xpass=1 xfail=0 skip=0 error=1
+
+:
diff --git a/tests/tap-log.test b/tests/tap-log.test
new file mode 100755
index 0000000..eecb2a2
--- /dev/null
+++ b/tests/tap-log.test
@@ -0,0 +1,164 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - log file creation
+# - log file removal
+# - stdout and stderr of a script go in its log file
+# - TEST_SUITE_LOG redefinition, at either automake or make time
+# - VERBOSE environment variable support
+# Keep in sync with 'test-log.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TESTS = pass.test skip.test xfail.test fail.test xpass.test error.test
+TEST_SUITE_LOG = global.log
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Custom markers, for use in grepping checks.
+cmarker=::: # comment marker
+pmarker=%%% # plain maker
+
+cat > pass.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker pass $pmarker" >&2
+echo "# $cmarker pass $cmarker" >&2
+echo "ok 1"
+END
+
+cat > skip.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker skip $pmarker"
+echo "# $cmarker skip $cmarker"
+echo "ok 1 # SKIP"
+END
+
+cat > xfail.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker xfail $pmarker" >&2
+echo "# $cmarker xfail $cmarker" >&2
+echo "not ok 1 # TODO"
+END
+
+cat > fail.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker fail $pmarker"
+echo "# $cmarker fail $cmarker"
+echo "not ok 1"
+END
+
+cat > xpass.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker xpass $pmarker" >&2
+echo "# $cmarker xpass $cmarker" >&2
+echo "ok 1 # TODO"
+END
+
+cat > error.test <<END
+#! /bin/sh -e
+echo 1..1
+echo "$pmarker error $pmarker"
+echo "# $cmarker error $cmarker"
+echo 'Bail out!'
+END
+
+chmod a+x *.test
+
+TEST_SUITE_LOG=my.log $MAKE -e check && Exit 1
+ls -l # For debugging.
+test ! -f test-suite.log
+test ! -f global.log
+test -f my.log
+st=0
+for result in pass fail xfail xpass skip error; do
+ cat $result.log # For debugging.
+ $FGREP "$pmarker $result $pmarker" $result.log || st=1
+ $FGREP "$cmarker $result $cmarker" $result.log || st=1
+done
+test $st -eq 0 || Exit 1
+cat my.log # For debugging.
+for result in xfail fail xpass skip error; do
+ cat $result.log # For debugging.
+ $FGREP "$pmarker $result $pmarker" my.log || st=1
+ $FGREP "$cmarker $result $cmarker" my.log || st=1
+done
+test `$FGREP -c "$pmarker" my.log` -eq 5
+test `$FGREP -c "$cmarker" my.log` -eq 5
+
+# Passed test scripts shouldn't be mentioned in the global log.
+$EGREP '(^pass|[^x]pass)\.test' my.log && Exit 1
+# But failing (expectedly or not) and skipped ones should.
+$FGREP 'xfail.test' my.log
+$FGREP 'skip.test' my.log
+$FGREP 'fail.test' my.log
+$FGREP 'xpass.test' my.log
+$FGREP 'error.test' my.log
+
+touch error2.log test-suite.log global.log
+TEST_SUITE_LOG=my.log $MAKE -e mostlyclean
+ls -l # For debugging.
+test ! -f my.log
+test ! -f pass.log
+test ! -f fail.log
+test ! -f xfail.log
+test ! -f xpass.log
+test ! -f skip.log
+test ! -f error.log
+# "make mostlyclean" shouldn't remove unrelated log files.
+test -f error2.log
+test -f test-suite.log
+test -f global.log
+
+rm -f *.log
+
+VERBOSE=yes $MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+cat global.log
+test ! -f my.log
+test ! -f test-suite.log
+# Check that VERBOSE causes the global testsuite log to be
+# emitted on stdout.
+out=`cat stdout`
+log=`cat global.log`
+case $out in *"$log"*) ;; *) Exit 1;; esac
+
+touch error2.log test-suite.log my.log
+$MAKE clean
+ls -l # For debugging.
+test ! -f global.log
+test ! -f pass.log
+test ! -f fail.log
+test ! -f xfail.log
+test ! -f xpass.log
+test ! -f skip.log
+test ! -f error.log
+# "make clean" shouldn't remove unrelated log files.
+test -f error2.log
+test -f test-suite.log
+test -f my.log
+
+rm -f *.log
+
+:
diff --git a/tests/tap-merge-stdout-stderr.test
b/tests/tap-merge-stdout-stderr.test
new file mode 100755
index 0000000..787ad51
--- /dev/null
+++ b/tests/tap-merge-stdout-stderr.test
@@ -0,0 +1,73 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - The Automake TAP driver has an option that instruct it to read TAP
+# input also from the stderr of the test command, not only its stdout.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat > Makefile.am << 'END'
+AM_TEST_LOG_DRIVER_FLAGS = --comments --merge
+TESTS = all.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+#!/bin/sh
+echo 1..4
+echo ok 1 >&2
+echo ok 2
+echo "not ok 3 # TODO" >&2
+echo "ok 4 # SKIP"
+echo "# foo foo foo" >&2
+END
+chmod a+x all.test
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=1 skip=1 error=0
+grep '^# all\.test: foo foo foo' stdout
+
+cat > all.test <<END
+#!/bin/sh
+echo 1..1
+echo ok 1
+echo 'Bail out!' >&2
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1
+
+# See that the option `--no-merge' can override the effect of `--merge'.
+
+TEST_LOG_DRIVER_FLAGS=--no-merge $MAKE -e check >stdout \
+ || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+
+
+:
diff --git a/tests/tap-message-0.test b/tests/tap-message-0.test
new file mode 100755
index 0000000..bce333f
--- /dev/null
+++ b/tests/tap-message-0.test
@@ -0,0 +1,66 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - having "0" as a test description (or TODO or SKIP message) should
+# be supported
+# Note that a bug in some versions of TAP::Parser causes this not to be
+# generally true!
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test << 'END'
+1..10
+ok 1 0
+ok - 0
+not ok 3 0 # TODO
+not ok - 0 # TODO
+ok 5 0 # SKIP
+ok - 0 # SKIP
+not ok 7 # TODO 0
+ok 8 # SKIP 0
+not ok 9 0 # TODO 0
+ok 10 0 # SKIP 0
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=10 pass=2 fail=0 xpass=0 xfail=4 skip=4 error=0
+
+cat > exp << 'END'
+PASS: all.test 1 0
+PASS: all.test 2 - 0
+XFAIL: all.test 3 0 # TODO
+XFAIL: all.test 4 - 0 # TODO
+SKIP: all.test 5 0 # SKIP
+SKIP: all.test 6 - 0 # SKIP
+XFAIL: all.test 7 # TODO 0
+SKIP: all.test 8 # SKIP 0
+XFAIL: all.test 9 0 # TODO 0
+SKIP: all.test 10 0 # SKIP 0
+END
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-more.test b/tests/tap-more.test
new file mode 100755
index 0000000..7d0bf86
--- /dev/null
+++ b/tests/tap-more.test
@@ -0,0 +1,164 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# More on TAP support:
+# - more than one TAP-generating test script in $(TESTS)
+# - VPATH builds
+# - use with parallel make (if supported)
+# - basic use of diagnostic messages (lines beginning with "#")
+# - flags for TAP driver defined through AC_SUBST in configure.ac
+# - messages generated by the testsuite harness reference the
+# correct test script(s)
+# - "make distcheck" works
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_SUBST([AM_TEST_LOG_DRIVER_FLAGS], ['--comments'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TESTS = 1.test 2.test 3.test
+EXTRA_DIST = $(TESTS) tap-driver
+END
+
+cat > 1.test <<'END'
+#! /bin/sh
+echo 1..2
+echo ok 1 - mu
+if test -f not-skip; then
+ echo "not ok 2 zardoz"
+else
+ echo "ok 2 zardoz # SKIP"
+fi
+END
+
+cat > 2.test <<'END'
+#! /bin/sh
+echo 1..3
+echo "ok"
+echo "not ok # TODO not implemented"
+echo "ok 3"
+END
+
+cat > 3.test <<END
+#! /bin/sh
+echo 1..1
+echo ok - blah blah blah
+echo '# Some diagnostic'
+if test -f bail-out; then
+ echo 'Bail out! Kernel Panic'
+else
+ :
+fi
+END
+
+chmod a+x [123].test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+# Try a VPATH and by default serial build first, and then an in-tree
+# and by default parallel build.
+for try in 0 1; do
+
+ if test $try -eq 0; then
+ # VPATH serial build.
+ mkdir build
+ cd build
+ srcdir=..
+ run_make=$MAKE
+ elif test $try -eq 1; then
+ # In-tree parallel build.
+ srcdir=.
+ case $MAKE in
+ *\ -j*)
+ # Degree of parallelism already specified by the user: do
+ # not override it.
+ run_make=$MAKE
+ ;;
+ *)
+ # Some make implementations (e.g., HP-UX) don't grok `-j',
+ # some require no space between `-j' and the number of jobs
+ # (e.g., older GNU make versions), and some *do* require a
+ # space between `-j' and the number of jobs (e.g., Solaris
+ # dmake). We need a runtime test to see what works.
+ echo 'all:' > Makefile
+ for run_make in "$MAKE -j3" "$MAKE -j 3" "$MAKE"; do
+ $run_make && break
+ done
+ rm -f Makefile
+ ;;
+ esac
+ else
+ fatal_ "internal error, invalid value of '$try' for \$try"
+ fi
+
+ $srcdir/configure
+ ls -l # For debugging.
+
+ # Success.
+
+ $run_make check >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=6 pass=4 fail=0 xpass=0 xfail=1 skip=1 error=0
+ grep '^PASS: 1\.test 1 - mu$' stdout
+ grep '^SKIP: 1\.test 2 zardoz # SKIP$' stdout
+ test `$FGREP -c '1.test' stdout` -eq 2
+ grep '^PASS: 2\.test 1$' stdout
+ grep '^XFAIL: 2\.test 2 # TODO not implemented$' stdout
+ grep '^PASS: 2\.test 3$' stdout
+ test `$FGREP -c '2.test' stdout` -eq 3
+ grep '^PASS: 3\.test 1 - blah blah blah$' stdout
+ grep '^# 3\.test: Some diagnostic$' stdout
+ test `$FGREP -c '3.test' stdout` -eq 2
+
+ # Failure.
+
+ # Use `echo' here, since Solaris 10 /bin/sh would try to optimize
+ # a `:' away after the first iteration, even if it is redirected.
+ echo dummy > not-skip
+ echo dummy > bail-out
+ $run_make check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=7 pass=4 fail=1 xpass=0 xfail=1 skip=0 error=1
+ grep '^PASS: 1\.test 1 - mu$' stdout
+ grep '^FAIL: 1\.test 2 zardoz$' stdout
+ test `$FGREP -c '1.test' stdout` -eq 2
+ grep '^PASS: 2\.test 1$' stdout
+ grep '^XFAIL: 2\.test 2 # TODO not implemented$' stdout
+ grep '^PASS: 2\.test 3$' stdout
+ test `$FGREP -c '2.test' stdout` -eq 3
+ grep '^PASS: 3\.test 1 - blah blah blah$' stdout
+ grep '^# 3\.test: Some diagnostic$' stdout
+ grep '^ERROR: 3\.test - Bail out! Kernel Panic$' stdout
+ test `$FGREP -c '3.test' stdout` -eq 3
+
+ cd $srcdir
+
+done
+
+$MAKE distcheck
+
+:
diff --git a/tests/tap-more2.test b/tests/tap-more2.test
new file mode 100755
index 0000000..810f9b8
--- /dev/null
+++ b/tests/tap-more2.test
@@ -0,0 +1,93 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# More on TAP support:
+# - more LOG_COMPILER at once for TAP tests
+# - binary programs in $(TESTS)
+# - interactions with `check_*' variables
+
+required='cc native'
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_PROG_CC
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .sh .tap
+
+tap_driver = $(PERL) $(srcdir)/tap-driver
+
+LOG_DRIVER = $(tap_driver)
+SH_LOG_DRIVER = $(tap_driver)
+TAP_LOG_DRIVER = $(tap_driver)
+
+TAP_LOG_COMPILER = cat
+SH_LOG_COMPILER = $(SHELL)
+LOG_COMPILER =
+
+EXTRA_DIST = baz.tap
+
+check_SCRIPTS = bar.sh
+bar.sh: Makefile
+ echo '#!/bin/sh' > address@hidden
+ echo 'echo 1..1' >> address@hidden
+ echo 'echo "not ok 1 # TODO"' >> address@hidden
+ chmod a-w address@hidden && mv -f address@hidden $@
+CLEANFILES = bar.sh
+
+check_PROGRAMS = foo-test
+foo_test_SOURCES = foo.c
+
+TESTS = foo-test $(check_SCRIPTS) baz.tap
+
+EXTRA_DIST += tap-driver
+END
+
+cat > foo.c <<'END'
+#include <stdio.h>
+int main (void)
+{
+ printf ("1..1\n");
+ printf ("ok 1\n");
+ return 0;
+}
+END
+
+cat > baz.tap << 'END'
+1..1
+ok 1 # SKIP
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+for target in check distcheck; do
+ $MAKE $target >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=3 pass=1 fail=0 xpass=0 xfail=1 skip=1 error=0
+done
+
+:
diff --git a/tests/tap-no-disable-hard-error.test
b/tests/tap-no-disable-hard-error.test
new file mode 100755
index 0000000..678e184
--- /dev/null
+++ b/tests/tap-no-disable-hard-error.test
@@ -0,0 +1,50 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - "Bail out!" magic and TAP parse errors are not disabled nor turned
+# into simple failures by the definition DISABLE_HARD_ERRORS.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+DISABLE_HARD_ERRORS = yes
+TEST_LOG_COMPILER = cat
+TESTS = bail.test few.test noplan.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > bail.test <<END
+1..1
+Bail out!
+END
+
+cat > few.test <<END
+1..1
+END
+
+cat > noplan.test <<END
+# nothing here
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=3
+
+:
diff --git a/tests/tap-no-merge-stdout-stderr.test
b/tests/tap-no-merge-stdout-stderr.test
new file mode 100755
index 0000000..25a06fd
--- /dev/null
+++ b/tests/tap-no-merge-stdout-stderr.test
@@ -0,0 +1,53 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - By default, TAP input is only from the stdout (and not the stderr)
+# of the test command.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER_FLAGS = --comments
+TESTS = all.test
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+#!/bin/sh
+echo 1..2
+echo 'not ok' >&2
+echo 'not ok 1 # TODO' >&2
+echo 'ok 1'
+echo '# foo foo foo'
+echo '# bar bar bar' >&2
+echo 'ok 44 # TODO' >&2
+echo 'Bail out!' >&2
+echo 'ok 2 # SKIP'
+END
+
+chmod a+x all.test
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=1 fail=0 xpass=0 xfail=0 skip=1 error=0
+$FGREP 'foo foo foo' stdout
+$FGREP 'bar bar bar' stdout && Exit 1
+
+:
diff --git a/tests/tap-no-spurious-summary.test
b/tests/tap-no-spurious-summary.test
new file mode 100755
index 0000000..bc7fd51
--- /dev/null
+++ b/tests/tap-no-spurious-summary.test
@@ -0,0 +1,47 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - `:test-results:' directives in test scripts' output doesn't
+# originate spurious results in the testsuite summary
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..1
+:test-result: PASS
+:test-result: FAIL
+:test-result: XPASS
+:test-result: XFAIL
+:test-result: SKIP
+:test-result: ERROR
+:test-result: UNKNOWN
+ok 1
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+for result in PASS FAIL XPASS XFAIL SKIP ERROR UNKNOWN; do
+ grep "^ *:test-result: $result$" all.log
+done
+
+:
diff --git a/tests/tap-no-spurious.test b/tests/tap-no-spurious.test
new file mode 100755
index 0000000..91b115f
--- /dev/null
+++ b/tests/tap-no-spurious.test
@@ -0,0 +1,102 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - don't spuriously recognize lines that are "almost" TAP lines as
+# real TAP lines
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo 1..5 > all.test
+
+# The only recognized directives are "TODO" and "SKIP".
+# So the following should count as passed tests.
+cat >> all.test <<END
+ok 1 # XFAIL
+ok 2 # SKIPPED
+ok 3 # TO DO
+ok 4 # TODOALL
+ok 5 # FIXME
+END
+
+# According to documentation of Test::Harness::TAP(3):
+#
+# ``Lines written to standard output matching /^(not )?ok\b/
+# must be interpreted as test lines. All other lines must
+# not be considered test output.''
+
+cat >> all.test <<END
+ ok
+ ok 1
+${tab}ok
+${tab}ok 1
+ok1
+ok23
+ not ok
+ not ok 1
+${tab}not ok
+${tab}not ok 1
+not ok1
+not ok23
+notok
+notok 1
+not${tab}ok
+not${tab}ok 1
+not ok
+not ok 1
+no ok
+no ok 1
+# ok
+# not ok
+# ok 1
+# not ok 1
+#ok
+#not ok
+#ok 1
+#not ok 1
+END
+
+# The prove(1) utility doesn't bail out on these, so our driver
+# shouldn't either.
+cat >> all.test <<'END'
+bailout
+bailout!
+bail out
+bail out!
+Bailout
+Bailout!
+Bail out
+ Bail out
+ Bail out!
+#Bail out!
+# Bail out!
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=5 pass=5 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+:
diff --git a/tests/tap-not-ok-skip.test b/tests/tap-not-ok-skip.test
new file mode 100755
index 0000000..f154d23
--- /dev/null
+++ b/tests/tap-not-ok-skip.test
@@ -0,0 +1,39 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - a "not ok # SKIP" line should count as a failure, for consistency
+# with the prove(1) utility.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..4
+not ok # SKIP
+not ok 2 # SKIP
+not ok - foo # SKIP
+not ok 4 - bar # SKIP
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=4 pass=0 fail=4 skip=0 xpass=0 xfail=0 error=0
+
+:
diff --git a/tests/tap-numeric-description.test
b/tests/tap-numeric-description.test
new file mode 100755
index 0000000..ea01038
--- /dev/null
+++ b/tests/tap-numeric-description.test
@@ -0,0 +1,69 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP result lines whose description is a number.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Some random numbers to be used as test names. The definitions below are
+# selected so that $x<n> != <n> for every n >= 1. We can't use positional
+# parameters for this unfortunately, since Solaris 10 /bin/sh doesn't allow
+# the dereferencing of parameters from ${10} onwards.
+x1=5 x2=7 x3=45 x4=11 x5=7 x6=3 x7=6 x8=9 x9=1000 x10=1
+
+cat > all.test <<END
+1..10
+ok 1 ${x1}
+ok - ${x2}
+not ok 3 ${x3}
+not ok - ${x4}
+ok 5 ${x5} # SKIP
+ok - ${x6} # SKIP
+not ok 7 ${x7} # TODO
+not ok - ${x8} # TODO
+ok 9 ${x9} # TODO
+ok - ${x10} # TODO
+END
+
+cat > exp <<END
+PASS: all.test 1 ${x1}
+PASS: all.test 2 - ${x2}
+FAIL: all.test 3 ${x3}
+FAIL: all.test 4 - ${x4}
+SKIP: all.test 5 ${x5} # SKIP
+SKIP: all.test 6 - ${x6} # SKIP
+XFAIL: all.test 7 ${x7} # TODO
+XFAIL: all.test 8 - ${x8} # TODO
+XPASS: all.test 9 ${x9} # TODO
+XPASS: all.test 10 - ${x10} # TODO
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=10 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=0
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+
+:
diff --git a/tests/tap-out-of-order.test b/tests/tap-out-of-order.test
new file mode 100755
index 0000000..f8f69c7
--- /dev/null
+++ b/tests/tap-out-of-order.test
@@ -0,0 +1,88 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - out-of-order test results
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > a.test <<END
+1..3
+ok
+not ok 2 # TODO
+ok 71
+END
+
+cat > b.test <<END
+1..5
+ok 1
+ok 2
+ok 4 foo # SKIP mu
+not ok 2 bar # TODO um
+ok 5
+END
+
+cat > c.test <<END
+1..4
+ok 1
+ok foo
+ok 4 - zardoz
+not ok 3
+END
+
+# This is documented as an "test out of sequence" error in the
+# TAP::Parser documentation. Keep it in sync with the example
+# there.
+cat > d.test <<'END'
+1..5
+ok 1
+ok 2
+ok 15
+ok 16
+ok 17
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+
+TESTS='a.test b.test c.test d.test' $MAKE -e check >stdout \
+ && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=17 pass=8 fail=0 xpass=0 xfail=1 skip=0 error=8
+
+test `$FGREP -c ': a.test' stdout` -eq 3
+test `$FGREP -c ': b.test' stdout` -eq 5
+test `$FGREP -c ': c.test' stdout` -eq 4
+test `$FGREP -c ': d.test' stdout` -eq 5
+
+grep '^ERROR: a\.test 71 # OUT-OF-ORDER (expecting 3)$' stdout
+grep '^ERROR: b\.test 4 foo # OUT-OF-ORDER (expecting 3)$' stdout
+grep '^ERROR: b\.test 2 bar # OUT-OF-ORDER (expecting 4)$' stdout
+grep '^ERROR: c\.test 4 - zardoz # OUT-OF-ORDER (expecting 3)$' stdout
+grep '^ERROR: c\.test 3 # OUT-OF-ORDER (expecting 4)$' stdout
+grep '^ERROR: d\.test 15 # OUT-OF-ORDER (expecting 3)$' stdout
+grep '^ERROR: d\.test 16 # OUT-OF-ORDER (expecting 4)$' stdout
+grep '^ERROR: d\.test 17 # OUT-OF-ORDER (expecting 5)$' stdout
+
+:
diff --git a/tests/tap-passthrough-exit.test b/tests/tap-passthrough-exit.test
new file mode 100755
index 0000000..68d7071
--- /dev/null
+++ b/tests/tap-passthrough-exit.test
@@ -0,0 +1,59 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - non-success exit status of a test script is reported in the
+# log file
+# See also related test 'tap-passthrough.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_COMPILER = $(SHELL)
+TESTS =
+END
+
+exit_statuses='1 2 77 99'
+
+for e in $exit_statuses; do
+ unindent > exit-$e.test <<END
+ #!/bin/sh
+ echo 1..1
+ echo ok 1
+ exit $e
+END
+ echo TESTS += exit-$e.test >> Makefile.am
+done
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+st=0
+$MAKE check || st=$?
+for e in $exit_statuses; do cat exit-$e.log; done
+cat test-suite.log
+test $st -gt 0 || Exit 1
+
+for e in $exit_statuses; do
+ for log in exit-$e.log test-suite.log; do
+ grep "^ERROR: exit-$e\\.test - exited with status $e$" $log
+ done
+done
+
+env TEST_LOG_DRIVER_FLAGS='--ignore-exit' $MAKE -e check
+$FGREP ".test - exited with status" *.log && Exit 1
+
+:
diff --git a/tests/tap-passthrough.test b/tests/tap-passthrough.test
new file mode 100755
index 0000000..6406548
--- /dev/null
+++ b/tests/tap-passthrough.test
@@ -0,0 +1,166 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - all input (valid TAP lines, invalid TAP lines, non-TAP lines)
+# are passed through in the log file
+# - TAP errors are reported in the log file too
+# See also related test 'tap-passthrough-exit.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+weirdchars=\''"address@hidden&()[]<>#;,:.^?*/'
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+#
+# Only successful tests.
+#
+
+# The whitespace in this test might be normalized in the testsuite
+# progress output, but should be copied verbatim in the log files.
+cat > ok.test <<END
+1..6
+TAP plan in the previous line.
+ok${tab}
+ok 2
+ok - foo
+ok 4 - x
+ This is not a TAP line, but should still be copied in the log file!
+# some diagnostic${tab}
+not ok # TODO low priority
+ok # SKIP who cares?
+$weirdchars
+END
+
+TESTS=ok.test $MAKE -e check || { cat ok.log; Exit 1; }
+cat ok.log
+
+for rx in \
+ '1\.\.6' \
+ 'TAP plan in the previous line\.' \
+ "ok${tab}" \
+ 'ok 2' \
+ 'ok - foo' \
+ 'ok 4 - x' \
+ ' This is not a TAP line, but should still be copied in the log file!' \
+ "# some diagnostic${tab}" \
+ 'not ok # TODO low priority' \
+ 'ok # SKIP who cares?' \
+; do
+ grep "^$rx$" ok.log
+done
+$FGREP "$weirdchars" ok.log
+
+#
+# Mixed failing/successful tests.
+#
+
+cat > tiny.test <<END
+1..1
+ok
+END
+
+cat > ok.test <<END
+1..1
+ok
+only one success here
+END
+
+cat > ko.test <<END
+1..5
+foo foo foo
+ok${tab}
+ok 2
+not ok - foo
+not ok 4 - x
+# diagnostic ko
+ bar${tab}bar${tab}bar
+ok # TODO dunno
+$weirdchars
+END
+
+cat > bail.test <<END
+Bail out! Test is taking too long!
+END
+
+cat > skip.test <<END
+1..0 # Skipped: WWW::Mechanize not installed
+END
+
+cat > err.test <<END
+1..3
+ok 1
+Invalid test count
+ok 23
+Misplaced plan
+1..13
+ok
+Extra test
+ok
+Last line
+END
+
+st=0
+env TESTS='tiny.test ok.test ko.test bail.test skip.test err.test' \
+ $MAKE -e check || st=$?
+cat tiny.log
+cat ok.log
+cat ko.log
+cat bail.log
+cat skip.log
+cat err.log
+test $st -gt 0 || Exit 1
+
+grep '^1\.\.1$' tiny.log
+grep '^ok$' tiny.log
+grep '^only one success here$' ok.log
+
+for rx in \
+ '1\.\.5' \
+ 'foo foo foo' \
+ "ok${tab}" \
+ 'ok 2' \
+ 'not ok - foo' \
+ 'not ok 4 - x' \
+ '# diagnostic ko' \
+ " bar${tab}bar${tab}bar" \
+ 'ok # TODO dunno' \
+; do
+ grep "^$rx$" ko.log
+done
+$FGREP "$weirdchars" ko.log
+
+grep '^Bail out! Test is taking too long!$' bail.log
+grep '^1\.\.0 # Skipped: WWW::Mechanize not installed$' skip.log
+
+for rx in \
+ '^1\.\.3$' \
+ '^Invalid test count$' \
+ '^ok 23$' \
+ '^Misplaced plan$' \
+ '^1\.\.13$' \
+ '^ERROR:.* multiple test plans' \
+ '^Extra test$' \
+ '^Last line$' \
+ '^ERROR:.* [tT]oo many tests run.*expected 3, got 4' \
+ '^ERROR:.* err\.test 23 .*OUT[ -]OF[ -]ORDER.*expecting 2' \
+; do
+ grep "$rx" err.log
+done
+
+:
diff --git a/tests/tap-plan-corner.test b/tests/tap-plan-corner.test
new file mode 100755
index 0000000..eaebcf2
--- /dev/null
+++ b/tests/tap-plan-corner.test
@@ -0,0 +1,124 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - some corner cases for TAP plan
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > leading-repeated.test <<END
+1..1
+1..1
+ok 1
+END
+
+cat > trailing-repeated.test <<END
+ok 1
+1..1
+1..1
+END
+
+for pos in leading trailing; do
+ TESTS="$pos-repeated.test" $MAKE -e check >stdout \
+ && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1
+ grep "^ERROR: $pos-repeated\\.test - multiple test plans$" stdout
+done
+
+cat > leading-repeated.test <<END
+1..2
+ok 1
+1..2
+ok 2
+END
+
+cat > trailing-repeated.test <<END
+ok 1
+1..2
+ok 2
+1..2
+END
+
+env TESTS="leading-repeated.test trailing-repeated.test" \
+ $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=6 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=4
+for pos in leading trailing; do
+ grep "^ERROR: $pos-repeated\\.test - multiple test plans$" stdout
+ grep "^ERROR: $pos-repeated\\.test 2 # AFTER LATE PLAN$" stdout
+done
+
+cat > all.test <<END
+1..5
+ok 1
+ok 2
+1..5
+ok 3
+1..5
+ok 4
+1..5
+ok 5
+END
+
+$MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=8 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=6
+
+cat > exp <<'END'
+PASS: all.test 1
+PASS: all.test 2
+ERROR: all.test - multiple test plans
+ERROR: all.test 3 # AFTER LATE PLAN
+ERROR: all.test - multiple test plans
+ERROR: all.test 4 # AFTER LATE PLAN
+ERROR: all.test - multiple test plans
+ERROR: all.test 5 # AFTER LATE PLAN
+END
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+sed -e 1d all.test > t
+mv -f t all.test
+
+$MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=7 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=5
+
+cat > exp <<'END'
+PASS: all.test 1
+PASS: all.test 2
+ERROR: all.test 3 # AFTER LATE PLAN
+ERROR: all.test - multiple test plans
+ERROR: all.test 4 # AFTER LATE PLAN
+ERROR: all.test - multiple test plans
+ERROR: all.test 5 # AFTER LATE PLAN
+END
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-plan-errors.test b/tests/tap-plan-errors.test
new file mode 100755
index 0000000..95cc640
--- /dev/null
+++ b/tests/tap-plan-errors.test
@@ -0,0 +1,93 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: the following situations should be flagged as errors:
+# - unmatched test plan (too few tests run)
+# - multiple test plans
+# - missing test plan
+# - misplaced test plan (tests run after a late plan)
+# Checks about unplanned tests are performed in 'tap-unplanned.test'.
+# More checks about corner-cases in TAP plans are performed in
+# 'tap-plan-corner.test' and 'tap-plan-corner2.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+my_check ()
+{
+ cat > all.test
+ test -n "$err" || fatal_ "\$err not set before calling my_check"
+ cat all.test # For debugging.
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results "$@"
+ grep "^ERROR: all\\.test $err$" stdout
+ unset err
+}
+
+err='- too few tests run (expected 2, got 1)'
+my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+1..2
+ok 1
+END
+
+err='- too few tests run (expected 12, got 3)'
+my_check total=4 pass=2 fail=0 xpass=0 xfail=1 skip=0 error=1 <<END
+ok 1
+ok 2
+not ok 3 # TODO
+1..12
+END
+
+err='- too few tests run (expected 1, got 0)'
+my_check total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+1..1
+END
+
+err='2 # AFTER LATE PLAN'
+my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+ok 1
+1..2
+ok 2
+END
+
+err='5 # AFTER LATE PLAN'
+my_check total=5 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+ok 1
+ok 2
+ok 3
+ok 4
+1..5
+ok 5
+END
+
+# The two test plans here are deliberately equal.
+err='- multiple test plans'
+my_check total=3 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+1..2
+ok 1
+ok 2
+1..2
+END
+
+err='- missing test plan'
+my_check total=2 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=1 <<END
+ok 1
+END
+
+:
diff --git a/tests/tap-plan-middle.test b/tests/tap-plan-middle.test
new file mode 100755
index 0000000..88bb647
--- /dev/null
+++ b/tests/tap-plan-middle.test
@@ -0,0 +1,83 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - test plan preceding and/or following non-result TAP lines
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > top1.test <<END
+non-TAP line, ignored
+1..1
+ok 1
+END
+
+cat > top2.test <<END
+some
+non-TAP
+lines
+are
+ignored
+# and a TAP comment won't cause problems either
+1..2
+ok 1
+ok 2
+END
+
+# Try with a blank line too, just to be sure.
+cat > top3.test <<END
+
+1..1
+ok 1
+END
+
+cat > bot1.test <<END
+ok 1 # SKIP
+1..1
+bla blah blah ...
+END
+
+cat > bot2.test <<END
+ok 1
+ok 2
+not ok 3 # TODO
+1..3
address@hidden (a cursing comment :-)
+END
+
+# Try with a blank line too, just to be sure.
+cat > bot3.test <<END
+ok 1
+not ok 2 # TODO
+ok 3 # SKIP
+ok 4 # SKIP
+1..4
+
+END
+
+tests=`echo *.test`
+
+for tap_flags in "" "--comments"; do
+ env TEST_LOG_DRIVER_FLAGS="$tap_flags" TESTS="$tests" \
+ $MAKE -e check >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=12 pass=7 xfail=2 skip=3 fail=0 xpass=0 error=0
+done
+
+:
diff --git a/tests/tap-plan.test b/tests/tap-plan.test
new file mode 100755
index 0000000..e2fda35
--- /dev/null
+++ b/tests/tap-plan.test
@@ -0,0 +1,52 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - test scripts with the test plan at the beginning
+# - test scripts with the test plan at the end
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > top.test <<END
+1..3
+ok 1
+ok 2 # SKIP
+ok 3
+# a trailing comment
+END
+
+cat > bot.test <<END
+ok 1
+# a comment
+ok 2
+not ok 3 # TODO
+# another comment
+ok
+1..4
+END
+
+for tap_flags in "" "--comments"; do
+ env TEST_LOG_DRIVER_FLAGS="$tap_flags" TESTS='top.test bot.test' \
+ $MAKE -e check >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=7 pass=5 xfail=1 skip=1 fail=0 xpass=0 error=0
+done
+
+
+:
diff --git a/tests/tap-realtime.test b/tests/tap-realtime.test
new file mode 100755
index 0000000..4a38f14
--- /dev/null
+++ b/tests/tap-realtime.test
@@ -0,0 +1,101 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - testsuite progress on console should happen mostly "in real time";
+# i.e., it's not acceptable for the driver to wait the end of the
+# script to start displaying results from it.
+# FIXME: this test uses expect(1) to ensure line buffering from make and
+# children, and is pretty hacky and complex; is there a better way to
+# accomplish the checks done here?
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >expect-check <<'END'
+eval spawn $env(SHELL) -c ":"
+expect eof
+END
+expect -f expect-check || {
+ echo "$me: failed to find a working expect program" >&2
+ Exit 77
+}
+rm -f expect-check
+
+cat > Makefile.am << 'END'
+TESTS = all.test
+END
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+#! /bin/sh
+echo 1..3
+
+# FIXME: creative quoting to placate maintainer-check
+sleep="sleep "3
+
+echo ok 1 - foo
+$sleep
+test -f ok-1 || { echo 'Bail out!'; exit 1; }
+
+echo ok 2 - bar
+$sleep
+test -f ok-2 || { echo 'Bail out!'; exit 1; }
+
+echo ok 3 - baz
+$sleep
+test -f ok-3 || { echo 'Bail out!'; exit 1; }
+
+: > all-is-well
+END
+
+chmod a+x all.test
+
+cat > expect-make <<'END'
+eval spawn $env(MAKE) check
+expect {
+ "PASS: all.test 1 - foo" {
+ open "ok-1" "w"
+ exp_continue
+ }
+ "PASS: all.test 2 - bar" {
+ open "ok-2" "w"
+ exp_continue
+ }
+ "PASS: all.test 3 - baz" {
+ open "ok-3" "w"
+ exp_continue
+ }
+ "Testsuite summary" {
+ exit 0
+ }
+ timeout {
+ puts "expect timed out"
+ exit 1
+ }
+ default {
+ puts "expect error"
+ exit 1
+ }
+}
+END
+
+# Expect should simulate a tty as stdout, which should ensure a
+# line-buffered output.
+MAKE=$MAKE expect -f expect-make
+test -f all-is-well
+
+:
diff --git a/tests/tap-recheck-logs.test b/tests/tap-recheck-logs.test
new file mode 100755
index 0000000..6e1b646
--- /dev/null
+++ b/tests/tap-recheck-logs.test
@@ -0,0 +1,119 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - RECHECK_LOGS
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_COMPILER = cat
+TESTS = foo.test bar.test baz.test
+baz.log: zardoz
+END
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+: > zardoz
+
+cat > foo.test <<'END'
+1..2
+ok 1
+ok 2
+END
+
+cat > bar.test <<'END'
+1..1
+not ok 1
+END
+
+cat > baz.test <<'END'
+1..1
+Bail out!
+END
+
+# Even the tests that are not re-run should contribute to the testsuite
+# summary when obtained by "make check RECHECK_LOGS=".
+grep_summary ()
+{
+ grep '^# TOTAL: *4$' stdout
+ grep '^# PASS: *2$' stdout
+ grep '^# XPASS: *0$' stdout
+ grep '^# FAIL: *1$' stdout
+ grep '^# XFAIL: *0$' stdout
+ grep '^# SKIP: *0$' stdout
+ grep '^# ERROR: *1$' stdout
+}
+
+$MAKE -e check && Exit 1
+test -f foo.log
+test -f bar.log
+test -f baz.log
+
+rm -f foo.log bar.log
+
+env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+test -f foo.log
+test -f bar.log
+grep '^PASS: foo\.test 1$' stdout
+grep '^PASS: foo\.test 2$' stdout
+grep '^FAIL: bar\.test 1$' stdout
+grep 'baz\.test' stdout && Exit 1
+grep_summary
+
+$sleep
+touch foo.test
+# We re-run only a successful test, but the tests that failed in the
+# previous run should still be taken into account, and cause an overall
+# failure.
+env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^PASS: foo\.test 1$' stdout
+grep '^PASS: foo\.test 2$' stdout
+grep 'ba[rz]\.test' stdout && Exit 1
+is_newest foo.log foo.test
+grep_summary
+
+$sleep
+touch zardoz
+env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^ERROR: baz\.test' stdout
+$EGREP '(foo|bar)\.test' stdout && Exit 1
+is_newest baz.log zardoz
+grep_summary
+
+# Now, explicitly retry with all test logs already updated, and ensure
+# that the summary is still displayed.
+env RECHECK_LOGS= $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+$EGREP '(foo|bar|baz)\.test' stdout && Exit 1
+grep_summary
+
+# The following should re-run foo.test (and only foo.test), even if its
+# log file is up-to-date.
+: > older
+env RECHECK_LOGS=foo.log $MAKE -e check > stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^PASS: foo\.test 1$' stdout
+grep '^PASS: foo\.test 2$' stdout
+grep 'ba[rz]\.test' stdout && Exit 1
+is_newest foo.log older
+grep_summary
+
+:
diff --git a/tests/tap-recheck.test b/tests/tap-recheck.test
new file mode 100755
index 0000000..14b02ad
--- /dev/null
+++ b/tests/tap-recheck.test
@@ -0,0 +1,217 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test the 'recheck' target for TAP test protocol.
+# Keep in sync with 'test-driver-custom-multitest-recheck.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat >> configure.in <<END
+AC_SUBST([PERL], ['$PERL'])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TESTS = a.test b.test c.test d.test
+END
+
+cat > a.test << 'END'
+#! /bin/sh
+echo 1..2
+echo ok 1
+echo ok 2
+: > a.run
+END
+
+cat > b.test << 'END'
+#! /bin/sh
+: > b.run
+echo 1..2
+echo ok 1
+if test -f b.ok; then
+ echo ok 2
+else
+ echo 'Bail out!'
+fi
+END
+
+cat > c.test << 'END'
+#! /bin/sh
+echo 1..3
+if test -f c.pass; then
+ echo 'ok - c is ok :-)'
+else
+ echo 'not ok - c is ko :-('
+fi
+if test -f c.xfail; then
+ echo 'not ok 2 # TODO'
+else
+ echo 'ok 2 # TODO'
+fi
+echo 'not ok 3 - blah blah # TODO need better diagnostic'
+: > c.run
+END
+
+cat > d.test << 'END'
+#! /bin/sh
+test -f ./d.count && . ./d.count
+echo 1..${test_count-2}
+echo ok 1 '# SKIP: who cares ...'
+(. ./d.extra) || echo 'not ok 2 - d.extra failed'
+: > d.run
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+do_recheck ()
+{
+ case $* in
+ --fail) on_bad_rc='&&';;
+ --pass) on_bad_rc='||';;
+ *) fatal_ "invalid usage of function 'do_recheck'";;
+ esac
+ rm -f *.run
+ eval "\$MAKE recheck >stdout $on_bad_rc { cat stdout; ls -l; Exit 1; }; :"
+ cat stdout; ls -l
+}
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ : A "make recheck" in a clean tree should run no tests.
+ do_recheck --pass
+ cat test-suite.log
+ test ! -r a.run
+ test ! -r a.log
+ test ! -r b.run
+ test ! -r b.log
+ test ! -r c.run
+ test ! -r c.log
+ test ! -r d.run
+ test ! -r d.log
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+ : Run the tests for the first time.
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ ls -l
+ # All the test scripts should have run.
+ test -f a.run
+ test -f b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
+
+ : Let us make b.test pass.
+ echo OK > b.ok
+ do_recheck --fail
+ # a.test has been successful the first time, so no need to re-run it.
+ # Similar considerations apply to similar checks, below.
+ test ! -r a.run
+ test -f b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
+
+ : Let us make the first part of c.test pass.
+ echo OK > c.pass
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
+
+ : Let us make also the second part of c.test pass.
+ echo KO > c.xfail
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
+
+ : Nothing changed, so only d.test should be run.
+ for i in 1 2; do
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
+ done
+
+ : Let us make d.test run more testcases, and experience _more_ failures.
+ echo 'test_count=9' > d.count
+ unindent > d.extra <<'END'
+ echo 'ok # SKIP s'
+ echo 'not ok - f 1'
+ echo 'ok - p 1'
+ echo 'not ok - f 2'
+ echo 'ok # TODO xp'
+ echo 'not ok - f 3'
+ echo 'not ok - f 4'
+ echo 'ok - p 2'
+ echo 'ok' # Extra test.
+ echo 'Bail out!'
+END
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
+
+ : Let us finally make d.test pass.
+ echo 'test_count=1' > d.count
+ echo : > d.extra
+ do_recheck --pass
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
+
+ : All tests have been successful or skipped, nothing should be re-run.
+ do_recheck --pass
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test ! -r d.run
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/tap-setup.sh b/tests/tap-setup.sh
new file mode 100755
index 0000000..0d1a1e3
--- /dev/null
+++ b/tests/tap-setup.sh
@@ -0,0 +1,48 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Auxiliary shell script that copies the common data and files used by
+# many tests on TAP support into the current directory. It should be
+# sourced by client test scripts, and assumes the auxiliary test
+# `tap-common-setup.test' has been run beforehand (it will error out
+# if this is not the case).
+
+# Check that we are running from a proper directory: last thing we want
+# is to overwrite some random user files.
+test -f ../defs-static && test -f ../defs \
+ || fatal_ "running from a wrong directory"
+
+test ! -f Makefile.am || mv Makefile.am Makefile.am~ \
+ || fatal_ "failed to save Makefile.am"
+
+test -d ../tap-common-setup.dir && cp -Rp ../tap-common-setup.dir/* . \
+ || fatal_ "couldn't get precomputed data files"
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+if test -f Makefile.am~; then
+ mv -f Makefile.am~ Makefile.am \
+ || fatal_ "failed to restore Makefile.am"
+ echo 'TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver' >> Makefile.am \
+ || fatal_ "failed to update Makefile.am"
+ $AUTOMAKE Makefile \
+ || fatal_ "failed to remake Makefile.in"
+ ./config.status Makefile \
+ || fatal_ "failed to remake Makefile"
+fi
+
+:
diff --git a/tests/tap-signal.test b/tests/tap-signal.test
new file mode 100755
index 0000000..8ceeddd
--- /dev/null
+++ b/tests/tap-signal.test
@@ -0,0 +1,90 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - a test script terminated by a signal causes an hard error
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+echo TESTS = > Makefile.am
+for sig in 1 2 13 15; do
+ unindent > signal-$sig.test <<END
+ #!/bin/sh
+ echo 1..1
+ echo ok 1
+ kill -$sig \$\$
+ echo "Bail out! \$0 not killed?"
+END
+ echo TESTS += signal-$sig.test >> Makefile.am
+done
+
+chmod a+x *.test
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Solaris /bin/sh, when killed with a SIGTERM or SIGINT signal, can end up
+# exiting with exit status 208, instead of leaving the correct wide exit
+# status to the parent. See:
+# <http://dbaspot.com/shell/396118-bourne-shell-exit-code-term.html>
+# We need to detect and work around this incompatibility.
+
+if /bin/sh -c 'kill -2 $$'; then
+ fatal_ "/bin/sh cannot kill itself"
+elif test $? -eq 208; then
+ have_solaris_bug=yes
+else
+ have_solaris_bug=no
+fi
+
+signal_caught ()
+{
+ numeric=$1
+ symbolic=$2
+ sig_re="((SIG)?$symbolic|$numeric)"
+ tst_re="signal-$numeric\\.test"
+ if $EGREP "^ERROR: $tst_re - terminated by signal $sig_re$" stdout; then
+ return 0
+ elif test $have_solaris_bug = yes; then
+ case $symbolic in
+ INT|TERM) $EGREP "^ERROR: $tst_re - exited with status 208$" stdout;;
+ *) return 1;;
+ esac
+ else
+ return 1
+ fi
+}
+
+all_signals_caught ()
+{
+ # This are the only signals that are portably trappable.
+ signal_caught 1 HUP
+ signal_caught 2 INT
+ signal_caught 13 PIPE
+ signal_caught 15 TERM
+}
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=8 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=4
+all_signals_caught
+
+echo 'TEST_LOG_DRIVER_FLAGS = --ignore-exit' >> Makefile
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=4 pass=4 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+:
diff --git a/tests/tap-skip-whole-badexit.test
b/tests/tap-skip-whole-badexit.test
new file mode 100755
index 0000000..76ba692
--- /dev/null
+++ b/tests/tap-skip-whole-badexit.test
@@ -0,0 +1,52 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - an exit status != 0 of a test script causes an hard error, even if
+# the last line of output is a "SKIP plan" (e.g., "1..0 # SKIP").
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+echo TESTS = one.test two.test > Makefile.am
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > one.test <<'END'
+#!/bin/sh
+echo '1..0 # SKIP'
+exit 1
+END
+
+cat > two.test <<'END'
+#!/bin/sh
+echo '1..0'
+exit 22
+END
+
+chmod a+x one.test two.test
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+# The 'prove' utility reports both the skip and the non-zero exit status,
+# so we do the same.
+count_test_results total=4 pass=0 fail=0 xpass=0 xfail=0 skip=2 error=2
+
+grep '^ERROR: one\.test - exited with status 1$' stdout
+grep '^ERROR: two\.test - exited with status 22$' stdout
+
+:
diff --git a/tests/tap-skip-whole-lastline.test
b/tests/tap-skip-whole-lastline.test
new file mode 100755
index 0000000..25697e0
--- /dev/null
+++ b/tests/tap-skip-whole-lastline.test
@@ -0,0 +1,59 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - the special plan format to skip all the tests in a script can also
+# be used on the last line, even if preceded by non-TAP output, TAP
+# diagnostic, and even non-failing TAP test results.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo AM_TEST_LOG_DRIVER_FLAGS = --comments >> Makefile
+
+cat > foo.test <<END
+some non-TAP text, will be copied in the global log
+1..0 # SKIP from the last line
+END
+
+cat > bar.test <<END
+# some TAP diagnostic, will go to console
+1..0
+END
+
+cat > baz.test <<END
+ok 1 - seen also if the whole test is skipped!
+1..0 # SKIP
+END
+
+st=0
+TESTS='foo.test bar.test baz.test' $MAKE -e check >stdout || st=$?
+cat stdout
+cat test-suite.log
+test $st -eq 0
+
+grep '^SKIP: foo\.test .* from the last line$' stdout
+grep '^SKIP: bar\.test$' stdout
+grep '^SKIP: baz\.test$' stdout
+grep '^# bar\.test: some TAP diagnostic, will go to console$' stdout
+grep '^some non-TAP text, will be copied in the global log$' test-suite.log
+grep '^PASS: baz\.test 1 - seen also if the whole test is skipped!$' stdout
+test `grep -c ': .*\.test' stdout` -eq 4
+count_test_results total=4 pass=1 fail=0 xpass=0 xfail=0 skip=3 error=0
+
+:
diff --git a/tests/tap-skip-whole-whitespace.test
b/tests/tap-skip-whole-whitespace.test
new file mode 100755
index 0000000..bf83241
--- /dev/null
+++ b/tests/tap-skip-whole-whitespace.test
@@ -0,0 +1,48 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - normalization of whitespace in console testsuite progress associated
+# with a SKIP directive in the TAP plan
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > foo.test <<END
+1..0${tab}${tab} #${tab}SKIP ${tab}Strip leading & trailing ${tab}${tab}
+END
+
+cat > bar.test <<END
+1..0 #SKIP Preserve ${tab} middle${tab}${tab}space
+END
+
+cat > baz.test <<END
+1..0 # SKIP${tab} Strip${tab}external preserve ${tab}middle ${tab}${sp}
+END
+
+TESTS='foo.test bar.test baz.test' $MAKE -e check > stdout \
+ || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=0 fail=0 error=0 xpass=0 xfail=0 skip=3
+
+grep "SKIP: foo\\.test - Strip leading & trailing$" stdout
+grep "SKIP: bar\\.test - Preserve ${tab} middle${tab}${tab}space" stdout
+grep "SKIP: baz\\.test - Strip${tab}external preserve ${tab}middle" stdout
+
+:
diff --git a/tests/tap-skip-whole.test b/tests/tap-skip-whole.test
new file mode 100755
index 0000000..6b14be4
--- /dev/null
+++ b/tests/tap-skip-whole.test
@@ -0,0 +1,83 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Basic TAP test protocol support:
+# - special plan format to skip all the tests in a script
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+weirdchars=\''"$!&()[]<>#;^?*/@%=,.:'
+
+cat > foo.test <<END
+1..0
+not ok 1
+not ok 2
+END
+
+cat > bar.test <<END
+1..0$tab $tab
+ok 1
+Bail out! some random failure
+END
+
+# It is undefined whether the comment after the plan below should
+# count as an explanation; still, the test should be skipped.
+cat > baz.test <<END
+1..0 # WWW::Mechanize not installed
+ok 1
+END
+
+cat > wget.test <<END
+1..0 # SKIP wget(1) not installed
+not ok 1 # TODO
+ok 2
+ok 3 # SKIP
+END
+
+cat > curl.test <<END
+1..0 # skip: Can't connect to gnu.org!
+not ok 1
+ok 2 # TODO
+END
+
+cat > mu.test <<END
+1..0 # Skip $weirdchars
+# Various TAP errors in here shouldn't be seen, as the test is skipped.
+1..2
+ok 1
+ok 22
+not ok 333
+1..7
+END
+
+env TESTS='foo.test bar.test baz.test wget.test curl.test mu.test' \
+ $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=6 pass=0 fail=0 xpass=0 xfail=0 skip=6 error=0
+
+grep '^SKIP: foo\.test$' stdout
+grep '^SKIP: bar\.test$' stdout
+grep '^SKIP: baz\.test' stdout # Deliberately laxer, see above for why.
+grep '^SKIP: wget\.test .* wget(1) not installed$' stdout
+grep '^SKIP: curl\.test .* Can'\''t connect to gnu\.org!$' stdout
+grep '^SKIP: mu\.test' stdout | $FGREP "$weirdchars" stdout
+test `grep -c ': .*\.test' stdout` -eq 6
+
+:
diff --git a/tests/tap-summary-aux.sh b/tests/tap-summary-aux.sh
new file mode 100755
index 0000000..a5cad53
--- /dev/null
+++ b/tests/tap-summary-aux.sh
@@ -0,0 +1,361 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Auxiliary script for tests on TAP support: checking testsuite summary.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+br='============================================================================'
+
+case $use_colors in
+ yes|no) ;;
+ *) fatal_ "invalid \$use_colors value '$use_colors'"
+esac
+
+cp "$top_testsrcdir"/lib/tap-driver . \
+ || fatal_ "failed to fetch auxiliary script tap-driver"
+
+cat > configure.in <<END
+AC_INIT([GNU AutoTAP], [5.12], address@hidden)
+AM_INIT_AUTOMAKE([parallel-tests])
+AC_SUBST([PERL], ['$PERL'])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(PERL) $(srcdir)/tap-driver
+TEST_LOG_COMPILER = cat
+TESTS = all.test
+END
+
+# The following shell variables are influential for this function:
+# - expect_failure
+# - use_colors
+do_check ()
+{
+ case $#,$1 in
+ 1,--pass) expect_failure=no;;
+ 1,--fail) expect_failure=yes;;
+ *) fatal_ "invalid usage of 'do_check'";;
+ esac
+ shift
+ cat > summary.exp
+ cat all.test
+ st=0
+ if test $use_colors = yes; then
+ make_cmd="env AM_COLOR_TESTS=always $MAKE -e"
+ else
+ make_cmd=$MAKE
+ fi
+ $make_cmd check > stdout || st=$?
+ cat stdout
+ if test $expect_failure = yes; then
+ test $st -gt 0 || Exit 1
+ else
+ test $st -eq 0 || Exit 1
+ fi
+ $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
+ || fatal_ "cannot extract testsuite summary"
+ cat summary.exp
+ cat summary.got
+ if test $use_colors = yes; then
+ # Use cmp, not diff, because the files might contain binary data.
+ compare=cmp
+ else
+ compare=diff
+ fi
+ $compare summary.exp summary.got || Exit 1
+}
+
+if test $use_colors = yes; then
+ red='[0;31m'
+ grn='[0;32m'
+ lgn='[1;32m'
+ blu='[1;34m'
+ mgn='[0;35m'
+ brg='[1m'
+ std='[m'
+ echo AUTOMAKE_OPTIONS = color-tests >> Makefile.am
+else
+ red= grn= lgn= blu= mgn= brg= std=
+fi
+
+ success_header="\
+${grn}${br}${std}
+${grn}Testsuite summary for GNU AutoTAP 5.12${std}
+${grn}${br}${std}"
+
+ success_footer=${grn}${br}${std}
+
+ failure_header="\
+${red}${br}${std}
+${red}Testsuite summary for GNU AutoTAP 5.12${std}
+${red}${br}${std}"
+
+ failure_footer="\
+${red}${br}${std}
+${red}See ./test-suite.log${std}
+${red}Please report to address@hidden
+${red}${br}${std}"
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+./configure
+
+# 1 pass.
+{ echo 1..1 && echo ok; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 skip.
+{ echo 1..1 && echo 'ok # SKIP'; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 xfail.
+{ echo 1..1 && echo 'not ok # TODO'; } > all.test
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 fail.
+{ echo 1..1 && echo not ok; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 xpass.
+{ echo 1..1 && echo 'ok # TODO'; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+# 1 hard error.
+{ echo 1..1 && echo 'Bail out!'; } > all.test
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# 3 non-failing results.
+cat > all.test <<END
+1..3
+ok
+not ok # TODO
+ok # SKIP
+END
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 pass, 1 skip, 1 fail.
+cat > all.test <<END
+1..3
+ok
+ok # SKIP
+not ok
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 pass, 1 xfail, 1 xpass.
+cat > all.test <<END
+1..3
+ok
+ok # TODO
+not ok # TODO
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+# 1 skip, 1 xfail, 1 error.
+cat > all.test <<END
+1..3
+ok # SKIP
+not ok # TODO
+Bail out!
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# 1 of each kind
+cat > all.test <<END
+1..6
+ok
+not ok
+ok # TODO
+not ok # TODO
+ok # SKIP
+Bail out!
+END
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 6${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+${red}# FAIL: 1${std}
+${red}# XPASS: 1${std}
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+# Prepare some common data for later.
+for i in 0 1 2 3 4 5 6 7 8 9; do
+ for j in 0 1 2 3 4 5 6 7 8 9; do
+ echo "ok"
+ echo "not ok # TODO"
+ echo "ok # SKIP"
+ done
+done > tap
+
+# Lots of non-failures (300 per kind).
+(cat tap && cat tap && cat tap) > all.test
+test `wc -l <all.test` -eq 900 || Exit 99 # Sanity check.
+echo 1..900 >> all.test # Test plan.
+do_check --pass <<END
+$success_header
+${brg}# TOTAL: 900${std}
+${grn}# PASS: 300${std}
+${blu}# SKIP: 300${std}
+${lgn}# XFAIL: 300${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+# 1 failure and lots of non-failures means failure.
+(cat tap && echo "not ok" && cat tap) > all.test
+test `wc -l <all.test` -eq 601 || Exit 99 # Sanity check.
+echo 1..601 >> all.test # Test plan.
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 601${std}
+${grn}# PASS: 200${std}
+${blu}# SKIP: 200${std}
+${lgn}# XFAIL: 200${std}
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# 1 error and lots of non-failures means failure.
+(cat tap && sed 30q tap && echo 'Bail out!') > all.test
+test `wc -l <all.test` -eq 331 || Exit 99 # Sanity check.
+echo 1..331 >> all.test # Test plan.
+do_check --fail <<END
+$failure_header
+${brg}# TOTAL: 331${std}
+${grn}# PASS: 110${std}
+${blu}# SKIP: 110${std}
+${lgn}# XFAIL: 110${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+:
diff --git a/tests/tap-summary-color.test b/tests/tap-summary-color.test
new file mode 100755
index 0000000..2dacb54
--- /dev/null
+++ b/tests/tap-summary-color.test
@@ -0,0 +1,23 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - colorized testsuite summary
+
+. ./defs-static
+
+use_colors=yes
+. "$testsrcdir"/tap-summary-aux.sh
diff --git a/tests/tap-summary.test b/tests/tap-summary.test
new file mode 100755
index 0000000..b49aba7
--- /dev/null
+++ b/tests/tap-summary.test
@@ -0,0 +1,23 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - colorized testsuite summary
+
+. ./defs-static
+
+use_colors=no
+. "$testsrcdir"/tap-summary-aux.sh
diff --git a/tests/tap-todo-skip-together.test
b/tests/tap-todo-skip-together.test
new file mode 100755
index 0000000..14b8ed5
--- /dev/null
+++ b/tests/tap-todo-skip-together.test
@@ -0,0 +1,39 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - TODO and SKIP directives on the same line: the first one wins
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+1..2
+ok 1 # SKIP TODO
+not ok 2 # TODO SKIP
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+grep '^SKIP: all\.test 1 ' stdout
+grep '^XFAIL: all\.test 2 ' stdout
+
+:
diff --git a/tests/tap-todo-skip-whitespace.test
b/tests/tap-todo-skip-whitespace.test
new file mode 100755
index 0000000..695e98c
--- /dev/null
+++ b/tests/tap-todo-skip-whitespace.test
@@ -0,0 +1,100 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - normalization of whitespace in console testsuite progress associated
+# with TODO and SKIP directives
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > stub.tap <<END
+1 # TODO
+2 # TODO foo?
+3 # TODO: bar!
+4 aa # TODO
+5 bb # TODO fnord 5
+6 cc # TODO:${tab}fnord 6
+7 - x # TODO
+8 - y # TODO fnord $tab 8
+9 - z # TODO: fnord 9 $tab
+10# TODO x0
+11$tab# TODO x1
+12 $tab$tab # TODO x2
+13 asd# TODO x3
+14 sad$tab# TODO x4
+15 das$tab$tab # TODO x5
+END
+
+cat > stub.exp <<END
+1 # TODO
+2 # TODO foo?
+3 # TODO: bar!
+4 aa # TODO
+5 bb # TODO fnord 5
+6 cc # TODO:${tab}fnord 6
+7 - x # TODO
+8 - y # TODO fnord $tab 8
+9 - z # TODO: fnord 9
+10 # TODO x0
+11 # TODO x1
+12 # TODO x2
+13 asd # TODO x3
+14 sad # TODO x4
+15 das # TODO x5
+END
+
+plan=1..15
+
+my_make_check ()
+{
+ xpass=0 xfail=0 skip=0
+ case $1 in
+ xpass|xfail|skip) eval $1=15;;
+ *) fatal_ "bad argument '$1' for my_make_check";;
+ esac
+ cat all.test
+ $MAKE check >stdout || : # Don't care about the exit status in this test.
+ cat stdout
+ count_test_results total=15 pass=0 fail=0 error=0 \
+ xpass=$xpass xfail=$xfail skip=$skip
+ # Don't be too strict w.r.t. possible normalization of "TODO: foo" into
+ # "TODO : foo" (as is done by, e.g., the 'TAP::Parser' perl module).
+ LC_ALL=C grep '^[A-Z][A-Z]*:' stdout \
+ | sed -e 's/# \(TODO\|SKIP\) *:/# \1:/' > got
+ cat exp
+ cat got
+ diff exp got
+}
+
+# For "TODO" directives leading to XPASS results.
+(echo $plan && sed -e 's/^/ok /' stub.tap) > all.test
+sed -e 's/^/XPASS: all.test /' stub.exp > exp
+my_make_check xpass
+
+# For "TODO" directives leading to XFAIL results.
+(echo $plan && sed -e 's/^/not ok /' stub.tap) > all.test
+sed -e 's/^/XFAIL: all.test /' stub.exp > exp
+my_make_check xfail
+
+# For "SKIP" directives.
+(echo $plan && sed -e 's/^/ok /' -e 's/TODO/SKIP/' stub.tap) > all.test
+sed -e 's/TODO/SKIP/' -e 's/^/SKIP: all.test /' stub.exp > exp
+my_make_check skip
+
+:
diff --git a/tests/tap-todo-skip.test b/tests/tap-todo-skip.test
new file mode 100755
index 0000000..d00dddd
--- /dev/null
+++ b/tests/tap-todo-skip.test
@@ -0,0 +1,91 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - TODO and SKIP directives are case-insensitive
+# - TODO and SKIP directives can be followed optionally by a colon ":"
+# and by an optional explanation.
+# - our driver isn't fooled into recognizing TODO and SKIP directives
+# spuriously
+# - the reasons for TODO and SKIP, if present, are nicely printed in
+# the testsuite progress output
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# ----------------------------------------------------- #
+# Check all possible combinations of: #
+# - uppercase/lowercase #
+# - with/without colon character ":" #
+# - with/without explanatory message #
+# in TODO and SKIP directives. #
+# ----------------------------------------------------- #
+
+# There are 2 * 2^6 + 2 * 2^6 = 256 tests.
+echo 1..256 > all.test
+
+# These nested loops are clearer without indentation.
+
+for c1 in t T; do
+for c2 in o O; do
+for c3 in d D; do
+for c4 in o O; do
+for ex in '' ':' ' foo' ': foo'; do
+ echo "not ok # $c1$c2$c3$c4$ex"
+ echo "not ok# $c1$c2$c3$c4$ex"
+done; done; done; done; done >> all.test
+
+for c1 in s S; do
+for c2 in k K; do
+for c3 in i I; do
+for c4 in p P; do
+for ex in '' ':' ' foo' ': foo'; do
+ echo "ok # $c1$c2$c3$c4$ex"
+ echo "ok# $c1$c2$c3$c4$ex"
+done; done; done; done; done >> all.test
+
+cat all.test # For debugging.
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=256 pass=0 fail=0 xpass=0 xfail=128 skip=128 error=0
+
+# -------------------------------------------------------- #
+# TODO ans SKIP directives aren't recognized spuriously. #
+# -------------------------------------------------------- #
+
+cat > all.test <<'END'
+1..9
+ok TODO
+ok - TODO
+ok 3 TODO
+ok 4 - TODO
+ok SKIP
+ok - SKIP
+ok 7 SKIP
+ok 8 - SKIP
+ok 9
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=9 pass=9 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+:
diff --git a/tests/tap-unplanned.test b/tests/tap-unplanned.test
new file mode 100755
index 0000000..177e36e
--- /dev/null
+++ b/tests/tap-unplanned.test
@@ -0,0 +1,159 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - unplanned tests are properly reported as errors
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<END
+1..1
+ok 1
+ok 2
+END
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=3 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=2
+grep '^ERROR: all\.test - too many tests run (expected 1, got 2)$' stdout
+grep '^ERROR: all\.test 2 # UNPLANNED$' stdout
+
+cat > all.test <<END
+1..2
+ok 1
+ok 2
+ok 3
+END
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=4 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=2
+grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout
+grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
+
+# Interaction with XFAIL_TESTS.
+cat > all.test <<END
+1..2
+not ok 1
+ok 2 # SKIP
+ok 3
+not ok 4
+ok 5 # SKIP
+END
+XFAIL_TESTS=all.test $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=6 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=4
+grep '^ERROR: all\.test - too many tests run (expected 2, got 5)$' stdout
+grep '^ERROR: all\.test 3 # UNPLANNED$' stdout
+grep '^ERROR: all\.test 4 # UNPLANNED$' stdout
+grep '^ERROR: all\.test 5 # UNPLANNED$' stdout
+
+cat > all.test <<END
+1..1
+ok 1
+
+ok
+ok 3
+ok foo
+ok 5 - bar bar
+
+not ok
+not ok 7
+not ok foo
+not ok 9 - bar bar
+
+ok # TODO
+ok 11 # TODO
+ok foo # TODO
+ok 13 - bar bar # TODO
+
+not ok # TODO
+not ok 15 # TODO
+not ok foo # TODO
+not ok 17 - bar bar # TODO
+
+ok # SKIP
+ok 19 # SKIP
+ok foo # SKIP
+ok 21 - bar bar # SKIP
+END
+
+cat > t <<END
+
+2
+3
+4 foo
+5 - bar bar
+
+6
+7
+8 foo
+9 - bar bar
+
+10
+11
+12 foo
+13 - bar bar
+
+14
+15
+16 foo
+17 - bar bar
+
+18
+19
+20 foo
+21 - bar bar
+
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=22 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=21
+
+echo 'PASS: all.test 1' > exp
+sed -e '/^$/d' -e 's/.*/ERROR: all.test & # UNPLANNED/' t >> exp
+echo 'ERROR: all.test - too many tests run (expected 1, got 21)' >> exp
+
+$FGREP ': all.test' stdout > got
+
+cat exp
+cat got
+diff exp got
+
+# Note that, if the TAP input has a trailing plan, it is not possible
+# to flag unplanned tests as such, since we do not know they're unplanned
+# until the plan is reached; still, we should give at least an error
+# message about the unmatched number of tests once we've got the plan.
+
+for x in 'ok' 'ok 3' 'not ok' 'not ok # TODO' 'ok # TODO' 'ok # SKIP'; do
+ unindent > all.test <<END
+ ok 1
+ ok 2 # SKIP
+ $x
+ 1..2
+END
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ test `$FGREP -c ': all.test' stdout` -eq 4
+ $EGREP '^PASS: all\.test 1($| )' stdout
+ $EGREP '^SKIP: all\.test 2($| )' stdout
+ $EGREP ': all\.test 3($| )' stdout
+ grep '^ERROR: all\.test - too many tests run (expected 2, got 3)$' stdout
+done
+
+:
diff --git a/tests/tap-whitespace-normalization.test
b/tests/tap-whitespace-normalization.test
new file mode 100755
index 0000000..a902438
--- /dev/null
+++ b/tests/tap-whitespace-normalization.test
@@ -0,0 +1,165 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support: whitespace normalization (or lack thereof) in the testsuite
+# progress output on console. We keep all these checks here in a single
+# script so that a potential cosmetic change in the output format won't
+# force us to tweak dozens of other tests (hopefully).
+# See also related test 'tap-todo-skip-whitespace.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat > Makefile.am << 'END'
+TEST_LOG_COMPILER = cat
+TESTS =
+END
+
+: > exp
+
+spaces_a=${sp}${tab}${tab}${sp}${sp}${tab}
+spaces_b=${tab}${tab}${sp}${tab}${sp}${sp}${sp}
+
+#-----------------------------------------------------------------------
+
+echo TESTS += numbers.test >> Makefile.am
+
+cat > numbers.test <<END
+1..6
+ok${spaces_a}1
+ok 2${spaces_b}
+ok${spaces_a}3${spaces_b}
+not ok${spaces_b}4
+not ok 5${spaces_a}
+not ok${spaces_b}6${spaces_a}
+END
+
+cat >> exp <<END
+PASS: numbers.test 1
+PASS: numbers.test 2
+PASS: numbers.test 3
+FAIL: numbers.test 4
+FAIL: numbers.test 5
+FAIL: numbers.test 6
+END
+
+#-----------------------------------------------------------------------
+
+echo TESTS += description.test >> Makefile.am
+
+cat > description.test <<END
+1..8
+ok${spaces_a}+foo
+ok +bar${spaces_b}
+ok${spaces_a}+baz${spaces_b}
+not ok${spaces_b}-foo
+not ok -bar${spaces_a}
+not ok${spaces_b}-baz${spaces_a}
+ok u${spaces_b}v${spaces_a}w${sp}
+not ok${spaces_a}x${spaces_a}y${tab}z${tab}
+END
+
+cat >> exp <<END
+PASS: description.test 1 +foo
+PASS: description.test 2 +bar
+PASS: description.test 3 +baz
+FAIL: description.test 4 -foo
+FAIL: description.test 5 -bar
+FAIL: description.test 6 -baz
+PASS: description.test 7 u${spaces_b}v${spaces_a}w
+FAIL: description.test 8 x${spaces_a}y${tab}z
+END
+
+#-----------------------------------------------------------------------
+
+# "Bail out!" magic.
+
+echo TESTS += bailout.test >> Makefile.am
+
+cat > bailout.test <<END
+1..1
+Bail out!${tab}${sp}${sp}${tab}We're out of disk space.
+ok 1
+END
+
+cat >> exp <<END
+ERROR: bailout.test - Bail out! We're out of disk space.
+END
+
+echo TESTS += bailout2.test >> Makefile.am
+
+cat > bailout2.test <<END
+1..1
+Bail out!foo${tab}${sp}
+ok 1
+END
+
+cat >> exp <<END
+ERROR: bailout2.test - Bail out! foo
+END
+
+#-----------------------------------------------------------------------
+
+# Diagnostic lines.
+
+echo AM_TEST_LOG_DRIVER_FLAGS = --comments >> Makefile.am
+echo TESTS += cmnt.test >> Makefile.am
+
+cat > cmnt.test <<END
+1..1
+ok 1
+#Leading whitespace gets added
+# ${tab}${tab} ${tab}Extra leading whitespace is stripped
+# Trailing whitespace is stripped ${tab} ${tab}${tab}
+# Middle${tab}whitespace is${tab} ${tab}${tab} kept
+# ${tab} And only${tab}middle ${tab}whitespace ${tab}${tab} ${tab}
+END
+
+cat >> exp <<END
+PASS: cmnt.test 1
+# cmnt.test: Leading whitespace gets added
+# cmnt.test: Extra leading whitespace is stripped
+# cmnt.test: Trailing whitespace is stripped
+# cmnt.test: Middle${tab}whitespace is${tab} ${tab}${tab} kept
+# cmnt.test: And only${tab}middle ${tab}whitespace
+END
+
+#-----------------------------------------------------------------------
+
+# TODO: we should have more checks here ... (but let's not over-do FTM).
+
+#-----------------------------------------------------------------------
+
+chmod a+x *.test
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+# Don't care about exit status or number of test results, they should be
+# checked for in many other tests.
+$MAKE check >stdout || :
+cat stdout
+
+LC_ALL=C sort exp > t
+mv -f t exp
+
+# We need the sort below to account for parallel make usage.
+LC_ALL=C grep '[a-z0-9][a-z0-9]*\.test' stdout | LC_ALL=C sort > got
+
+cat exp
+cat got
+diff exp got
+
+:
diff --git a/tests/tap-with-and-without-number.test
b/tests/tap-with-and-without-number.test
new file mode 100755
index 0000000..e989cc8
--- /dev/null
+++ b/tests/tap-with-and-without-number.test
@@ -0,0 +1,50 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+# - numbered and unnumbered test results can coexist
+# - tests without explicit number get automatically numbered in the
+# testsuite progress output on console
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test <<'END'
+1..7
+ok 1 foo
+ok 2 - foo2
+ok - bar
+not ok baz # TODO
+not ok 5 - asd # TODO
+ok 6 # SKIP
+ok zardoz
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+count_test_results total=7 pass=4 fail=0 xpass=0 xfail=2 skip=1 error=0
+
+grep '^PASS: all\.test 1 foo$' stdout
+grep '^PASS: all\.test 2 - foo2$' stdout
+grep '^PASS: all\.test 3 - bar$' stdout
+grep '^XFAIL: all\.test 4 baz # TODO$' stdout
+grep '^XFAIL: all\.test 5 - asd # TODO$' stdout
+grep '^SKIP: all\.test 6 # SKIP$' stdout
+grep '^PASS: all\.test 7 zardoz$' stdout
+
+:
diff --git a/tests/tap-xfail-tests.test b/tests/tap-xfail-tests.test
new file mode 100755
index 0000000..22733c8
--- /dev/null
+++ b/tests/tap-xfail-tests.test
@@ -0,0 +1,68 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# TAP and $(XFAIL_TESTS): test results without directives are turned from
+# PASS to XPASS and from FAIL to XFAIL; other results are unchanged.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo 'XFAIL_TESTS = $(TESTS)' >> Makefile
+
+cat > all.test <<END
+1..6
+ok 1
+not ok 2
+ok 3 # TODO
+not ok 4 # TODO
+ok 5 # SKIP
+# The next should count as a failure (and thus will be
+# rendered into an expected failure by XFAIL_TESTS).
+not ok 6 # SKIP
+Bail out!
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=7 pass=0 fail=0 xpass=2 xfail=3 skip=1 error=1
+
+grep '^XPASS: all\.test 1$' stdout
+grep '^XFAIL: all\.test 2$' stdout
+grep '^XPASS: all\.test 3 # TODO' stdout
+grep '^XFAIL: all\.test 4 # TODO' stdout
+grep '^SKIP: all\.test 5 # SKIP' stdout
+grep '^XFAIL: all\.test 6 # SKIP' stdout
+grep '^ERROR: all\.test - Bail out!' stdout
+
+# Check that the exit status of the testsuite is indeed 0 when we
+# would expect success.
+
+cat > all.test <<END
+1..3
+not ok 1
+ok 2 # SKIP
+not ok 3 # TODO
+END
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=0 fail=0 xpass=0 xfail=2 skip=1 error=0
+
+:
diff --git a/tests/test-driver-create-log-dir.test
b/tests/test-driver-create-log-dir.test
new file mode 100755
index 0000000..c554793
--- /dev/null
+++ b/tests/test-driver-create-log-dir.test
@@ -0,0 +1,83 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: if a log file has a directory component (as in
+# e.g., `sub/foo.log'), the Automake test harness must ensure that
+# directory exists before calling any custom test driver.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+# The extra `.' components below are meant.
+TESTS = sub/foo sub/subsub/bar.test ././sub1/./baz
+$(TESTS):
+LOG_DRIVER = $(srcdir)/checkdir-driver
+TEST_LOG_DRIVER = $(LOG_DRIVER)
+EXTRA_DIST = checkdir-driver
+
+check-local: $(TEST_SUITE_LOG)
+ test -d sub
+ test -d sub1
+ test -d sub/subsub
+ test -f sub/foo.log
+ test -f sub/subsub/bar.log
+ test -f sub1/baz.log
+ test -f sub/foo.trs
+ test -f sub/subsub/bar.trs
+ test -f sub1/baz.trs
+END
+
+cat > checkdir-driver <<'END'
+#! /bin/sh
+set -e; set -u
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name|--expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+echo "log: $log_file" # For debugging.
+echo "trs: $trs_file" # Ditto.
+case $log_file in */*);; *) exit 1;; esac
+dir_log=`expr "$log_file" : '\(.*\)/[^/]*'`
+dir_trs=`expr "$trs_file" : '\(.*\)/[^/]*'`
+echo "dir_log: $dir_log" # For debugging.
+echo "dir_trs: $dir_trs" # Likewise.
+test x"$dir_trs" = x"$dir_log" || exit 1
+test -d "$dir_log" || exit 1
+echo dummy1 > "$log_file"
+echo dummy2 > "$trs_file"
+END
+chmod a+x checkdir-driver
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+$MAKE check
+$MAKE distcheck
+
+:
diff --git a/tests/test-driver-custom-html.test
b/tests/test-driver-custom-html.test
new file mode 100755
index 0000000..5663ee6
--- /dev/null
+++ b/tests/test-driver-custom-html.test
@@ -0,0 +1,104 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers features:
+# - check-html
+# - recheck-html
+# Keep this in sync with sister test `parallel-tests2.test'.
+
+parallel_tests=yes
+required=rst2html
+. ./defs || Exit 1
+
+cp "$testsrcdir"/trivial-test-driver . \
+ || fatal_ "failed to fetch auxiliary script trivial-test-driver"
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) ./trivial-test-driver
+TEST_SUITE_LOG = mylog.log
+TESTS = foo.test bar.test baz.test
+check_SCRIPTS = bla
+bla:
+ echo bla > $@
+CLEANFILES = bla
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+if test -f bla; the
+ echo "PASS: this is $0"
+else
+ echo "FAIL: this is $0"
+fi
+END
+
+cat > bar.test <<'END'
+#! /bin/sh
+echo "FAIL: this is $0"
+END
+
+cat > baz.test <<'END'
+#! /bin/sh
+echo "FAIL: this is $0"
+END
+
+chmod a+x foo.test bar.test baz.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check-html && Exit 1
+test -f mylog.html
+# check-html should cause check_SCRIPTS to be created.
+test -f bla
+
+# "make clean" should remove HTML files.
+$MAKE clean
+test ! -f mylog.html
+test ! -f bla
+
+# Always create the HTML output, even if there were no failures.
+rm -f mylog.html
+env TESTS=foo.test $MAKE -e check-html
+test -f mylog.html
+
+# Create summarizing HTML output also with recheck-html.
+rm -f mylog.html
+env TESTS=foo.test $MAKE -e recheck-html
+test -f mylog.html
+
+# check-html should cause check_SCRIPTS to be created.
+$MAKE clean
+env TEST_LOGS=foo.log $MAKE -e check-html
+test -f bla
+test -f foo.log
+test -f mylog.html
+# recheck-html should cause check_SCRIPTS to be created, and should rerun
+# no tests if it appears that check has not been run.
+$MAKE clean
+env TESTS=foo.test $MAKE -e recheck-html
+test -f bla
+test ! -f foo.log
+test -f mylog.html
+
+:
diff --git a/tests/test-driver-custom-multitest-recheck.test
b/tests/test-driver-custom-multitest-recheck.test
new file mode 100755
index 0000000..8bbcc7c
--- /dev/null
+++ b/tests/test-driver-custom-multitest-recheck.test
@@ -0,0 +1,214 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: try the "recheck" functionality with test protocols
+# that allow multiple testcases in a single test script. This test not
+# only checks implementation details in Automake's custom test drivers
+# support, but also serves as a "usability test" for our APIs.
+# See also related tests `test-driver-custom-multitest-recheck2.test'
+# and `parallel-tests-recheck-override.test'.
+# Keep in sync with 'tap-recheck.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$testsrcdir"/trivial-test-driver . \
+ || fatal_ "failed to fetch auxiliary script trivial-test-driver"
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+TESTS = a.test b.test c.test d.test
+END
+
+cat > a.test << 'END'
+#! /bin/sh
+echo PASS: aa
+echo PASS: AA
+: > a.run
+END
+
+cat > b.test << 'END'
+#! /bin/sh
+echo PASS:
+if test -f b.ok; then
+ echo PASS:
+else
+ echo ERROR:
+fi
+: > b.run
+END
+
+cat > c.test << 'END'
+#! /bin/sh
+if test -f c.pass; then
+ echo PASS: c0
+else
+ echo FAIL: c0
+fi
+if test -f c.xfail; then
+ echo XFAIL: c1
+else
+ echo XPASS: c1
+fi
+echo XFAIL: c2
+: > c.run
+END
+
+cat > d.test << 'END'
+#! /bin/sh
+echo SKIP: who cares ...
+(. ./d.extra) || echo FAIL: d.extra failed
+: > d.run
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+do_recheck ()
+{
+ case $* in
+ --fail) on_bad_rc='&&';;
+ --pass) on_bad_rc='||';;
+ *) fatal_ "invalid usage of function 'do_recheck'";;
+ esac
+ rm -f *.run
+ eval "\$MAKE recheck >stdout $on_bad_rc { cat stdout; ls -l; Exit 1; }; :"
+ cat stdout; ls -l
+}
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ : A "make recheck" in a clean tree should run no tests.
+ do_recheck --pass
+ cat test-suite.log
+ test ! -r a.run
+ test ! -r a.log
+ test ! -r b.run
+ test ! -r b.log
+ test ! -r c.run
+ test ! -r c.log
+ test ! -r d.run
+ test ! -r d.log
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+ : Run the tests for the first time.
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ ls -l
+ # All the test scripts should have run.
+ test -f a.run
+ test -f b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=9 pass=3 fail=2 xpass=1 xfail=1 skip=1 error=1
+
+ : Let us make b.test pass.
+ echo OK > b.ok
+ do_recheck --fail
+ # a.test has been successful the first time, so no need to re-run it.
+ # Similar considerations apply to similar checks, below.
+ test ! -r a.run
+ test -f b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=7 pass=2 fail=2 xpass=1 xfail=1 skip=1 error=0
+
+ : Let us make the first part of c.test pass.
+ echo OK > c.pass
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=5 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=0
+
+ : Let us make also the second part of c.test pass.
+ echo KO > c.xfail
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+ test -f d.run
+ count_test_results total=5 pass=1 fail=1 xpass=0 xfail=2 skip=1 error=0
+
+ : Nothing changed, so only d.test should be run.
+ for i in 1 2; do
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
+ done
+
+ : Let us make d.test run more testcases, and experience _more_ failures.
+ unindent > d.extra <<'END'
+ echo SKIP: s
+ echo FAIL: f 1
+ echo PASS: p 1
+ echo FAIL: f 2
+ echo XPASS: xp
+ echo FAIL: f 3
+ echo FAIL: f 4
+ echo ERROR: e 1
+ echo PASS: p 2
+ echo ERROR: e 2
+END
+ do_recheck --fail
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=11 pass=2 fail=4 xpass=1 xfail=0 skip=2 error=2
+
+ : Let us finally make d.test pass.
+ echo : > d.extra
+ do_recheck --pass
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test -f d.run
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
+
+ : All tests have been successful or skipped, nothing should be re-run.
+ do_recheck --pass
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ test ! -r d.run
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/test-driver-custom-multitest-recheck2.test
b/tests/test-driver-custom-multitest-recheck2.test
new file mode 100755
index 0000000..0d254dc
--- /dev/null
+++ b/tests/test-driver-custom-multitest-recheck2.test
@@ -0,0 +1,188 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: try the "recheck" functionality with test protocols
+# that allow multiple testcases in a single test script. In particular,
+# check that this still works when we override $(TESTS) and $(TEST_LOGS)
+# at make runtime.
+# See also related tests `test-driver-custom-multitest-recheck.test' and
+# `parallel-tests-recheck-override.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$testsrcdir"/trivial-test-driver . \
+ || fatal_ "failed to fetch auxiliary script trivial-test-driver"
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+TESTS = a.test b.test c.test
+END
+
+cat > a.test << 'END'
+#! /bin/sh
+echo PASS: 1
+echo PASS: 2
+: > a.run
+END
+
+cat > b.test << 'END'
+#! /bin/sh
+echo SKIP: b0
+if test -f b.ok; then
+ echo XFAIL: b1
+else
+ echo FAIL: b2
+fi
+: > b.run
+END
+
+cat > c.test << 'END'
+#! /bin/sh
+if test -f c.err; then
+ echo ERROR: xxx
+elif test -f c.ok; then
+ echo PASS: ok
+else
+ echo XPASS: xp
+fi
+: > c.run
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ : Run the tests for the first time.
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ # All the test scripts should have run.
+ test -f a.run
+ test -f b.run
+ test -f c.run
+ count_test_results total=5 pass=2 fail=1 xpass=1 xfail=0 skip=1 error=0
+
+ rm -f *.run
+
+ : An empty '$(TESTS)' or '$(TEST_LOGS)' means that no test should be run.
+ for var in TESTS TEST_LOGS; do
+ env "$var=" $MAKE -e recheck >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ done
+ unset var
+
+ : a.test was sucessfull the first time, no need to re-run it.
+ env TESTS=a.test $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+
+ : b.test failed, it should be re-run. And make it pass this time.
+ echo OK > b.ok
+ TEST_LOGS=b.log $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ test ! -r a.run
+ test -f b.run
+ test ! -r c.run
+ count_test_results total=2 pass=0 fail=0 xpass=0 xfail=1 skip=1 error=0
+
+ rm -f *.run
+
+ : No need to re-run a.test or b.test anymore.
+ TEST_LOGS=b.log $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+ TESTS='a.test b.test' $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+
+ : No need to re-run a.test anymore, but c.test should be rerun,
+ : as it contained an XPASS. And this time, make it fail with
+ : an hard error.
+ # Use `echo' here, since Solaris 10 /bin/sh would try to optimize
+ # a `:' away after the first iteration, even if it is redirected.
+ echo dummy > c.err
+ env TEST_LOGS='a.log c.log' $MAKE -e recheck >stdout \
+ && { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+
+ rm -f *.run *.err
+
+ : c.test contained and hard error the last time, so it should be re-run.
+ : This time, make it pass
+ # Use `echo', not `:'; see comments above for why.
+ echo dummy > c.ok
+ env TESTS='c.test a.test' $MAKE -e recheck >stdout \
+ || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+ test ! -r a.run
+ test ! -r b.run
+ test -f c.run
+
+ rm -f *.run *.err *.ok
+
+ : Nothing should be rerun anymore, as all tests have been eventually
+ : succesful.
+ $MAKE recheck >stdout || { cat stdout; Exit 1; }
+ cat stdout
+ count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+ test ! -r a.run
+ test ! -r b.run
+ test ! -r c.run
+
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/test-driver-custom-multitest.test
b/tests/test-driver-custom-multitest.test
new file mode 100755
index 0000000..4fb023c
--- /dev/null
+++ b/tests/test-driver-custom-multitest.test
@@ -0,0 +1,172 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: check that we can easily support test protocols
+# that allow multiple testcases in a single test script. This test not
+# only checks implementation details in Automake's custom test drivers
+# support, but also serves as a "usability test" for our APIs.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cp "$testsrcdir"/trivial-test-driver . \
+ || fatal_ "failed to fetch auxiliary script trivial-test-driver"
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .t
+T_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+
+TESTS = \
+ pass.t \
+ fail.t \
+ fail2.t \
+ pass-fail.t \
+ pass4-skip.t \
+ pass3-skip2-xfail.t \
+ pass-xpass-fail-xfail-skip-error.t
+END
+
+cat > pass.t << 'END'
+echo %% pass %%
+echo PASS: pass
+END
+
+cat > fail.t << 'END'
+echo %% fail %%
+echo FAIL: fail
+END
+
+cat > fail2.t << 'END'
+echo %% fail2 %%
+echo FAIL: stdout >&1
+echo FAIL: stderr >&2
+echo :PASS: this should be ignored
+END
+
+cat > pass-fail.t << 'END'
+echo %% pass-fail %%
+echo 'FAIL: this fails :-('
+echo 'some randome message'
+echo 'some randome warning' >&2
+echo 'PASS: this passes :-)'
+echo 'INFO: blah'
+echo 'WARNING: blah blah' >&2
+END
+
+cat > pass4-skip.t << 'END'
+echo %% pass4-skip %%
+echo PASS: on stdout >&1
+echo PASS: on stderr >&2
+echo PASS: 3
+echo PASS: 4
+echo SKIP: 1
+echo this FAIL: should be ignored
+echo FAIL as should this
+exit 99
+END
+
+cat > pass3-skip2-xfail.t << 'END'
+echo %% pass4-skip2-xfail %%
+echo 'PASS: -v'
+echo 'PASS: --verbose'
+echo 'SKIP: Oops, unsupported system.'
+echo 'PASS: -#-#-#-'
+cp || echo "SKIP: cp cannot read users' mind" >&2
+mv || echo "XFAIL: mv cannot read users' mind yet"
+exit 127
+END
+
+cat > pass-xpass-fail-xfail-skip-error.t << 'END'
+echo PASS:
+echo FAIL:
+echo XFAIL:
+echo XPASS:
+echo SKIP:
+echo ERROR:
+echo %% pass-xpass-fail-xfail-skip-error %%
+END
+
+chmod a+x *.t
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+for vpath in : false; do
+ if $vpath; then
+ mkdir build
+ cd build
+ srcdir=..
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ $MAKE check >stdout && { cat stdout; cat test-suite.log; Exit 1; }
+ cat stdout
+ cat test-suite.log
+ # Couple of sanity checks. These might need to be updated if the
+ # `trivial-test-driver' script is changed.
+ $FGREP INVALID.NAME stdout test-suite.log && Exit 1
+ test -f BAD.LOG && Exit 1
+ test -f BAD.TRS && Exit 1
+ # These log files must all have been created by the testsuite.
+ cat pass.log
+ cat fail.log
+ cat fail2.log
+ cat pass-fail.log
+ cat pass4-skip.log
+ cat pass3-skip2-xfail.log
+ cat pass-xpass-fail-xfail-skip-error.log
+
+ count_test_results total=23 pass=10 fail=5 skip=4 xfail=2 xpass=1 error=1
+
+ tst=pass-xpass-fail-xfail-skip-error
+ grep "^PASS: $tst\.t, testcase 1" stdout
+ grep "^FAIL: $tst\.t, testcase 2" stdout
+ grep "^XFAIL: $tst\.t, testcase 3" stdout
+ grep "^XPASS: $tst\.t, testcase 4" stdout
+ grep "^SKIP: $tst\.t, testcase 5" stdout
+ grep "^ERROR: $tst\.t, testcase 6" stdout
+
+ # Check that the content of, and only of, the test logs with at least
+ # one failing test case has been copied into `test-suite.log'. Note
+ # that test logs containing skipped or failed test cases are *not*
+ # copied into `test-suite.log' -- a behaviour that deliberately differs
+ # from the one of the built-in Automake test drivers.
+ grep '%%' test-suite.log # For debugging.
+ grep '%% fail %%' test-suite.log
+ grep '%% fail2 %%' test-suite.log
+ grep '%% pass-fail %%' test-suite.log
+ grep '%% pass-xpass-fail-xfail-skip-error %%' test-suite.log
+ test `grep -c '%% ' test-suite.log` -eq 4
+
+ TESTS='pass.t pass3-skip2-xfail.t' $MAKE -e check >stdout \
+ || { cat stdout; cat test-suite.log; Exit 1; }
+ cat test-suite.log
+ cat stdout
+ count_test_results total=7 pass=4 fail=0 skip=2 xfail=1 xpass=0 error=0
+
+ cd $srcdir
+
+done
+
+:
diff --git a/tests/test-driver-custom-no-extra-driver.test
b/tests/test-driver-custom-no-extra-driver.test
new file mode 100755
index 0000000..28c9a19
--- /dev/null
+++ b/tests/test-driver-custom-no-extra-driver.test
@@ -0,0 +1,63 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that auxiliary script 'test-driver' doesn't get needlessly
+# installed or referenced if it's not used, i.e., if the user has
+# defined his own `*LOG_DRIVER' variables.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<'END'
+AC_PROG_CC
+AC_CONFIG_FILES([sub1/Makefile sub2/Makefile])
+AC_SUBST([X_LOG_DRIVER], [none])
+AC_OUTPUT
+END
+
+mkdir sub1 sub2
+
+cat > Makefile.am <<'END'
+SUBDIRS = sub1 sub2
+LOG_DRIVER =
+TEST_LOG_DRIVER = :
+TESTS = foo bar.test
+END
+
+cat > sub1/Makefile.am <<'END'
+TEST_EXTENSIONS = .x .sh .pl
+SH_LOG_DRIVER = dummy1
+PL_LOG_DRIVER = dummy2
+TESTS = a.pl b.sh c.x
+END
+
+cat > sub2/Makefile.am <<'END'
+TEST_EXTENSIONS = .bar
+LOG_DRIVER = x
+BAR_LOG_DRIVER = y
+TESTS = 1 2.bar 3.test 4.t 5.tt $(check_PROGRAMS)
+check_PROGRAMS = p1 p2$(EXEEXT) p3.bar p4.suf
+END
+
+$ACLOCAL
+
+for opts in '' '--add-missing' '-a -c'; do
+ $AUTOMAKE $opts
+ $FGREP test-driver Makefile.in sub[12]/Makefile.in && Exit 1
+ find . | $FGREP test-driver && Exit 1
+done
+
+:
diff --git a/tests/test-driver-custom-no-html.test
b/tests/test-driver-custom-no-html.test
new file mode 100755
index 0000000..2c86026
--- /dev/null
+++ b/tests/test-driver-custom-no-html.test
@@ -0,0 +1,67 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that custom test drivers does not need to produce sensible
+# reStructuredText output in the test logs. This might be legitimate
+# for drivers that are not interested to support the .log -> HTML
+# conversion offered by Automake.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = ./no-rst
+TESTS = foo.test
+END
+
+: > foo.test
+
+cat > no-rst <<'END'
+#! /bin/sh
+echo ':test-result: SKIP' > foo.trs
+echo ':copy-in-global-log: yes' >> foo.trs
+# The genereted log file is deliberately syntactically invalid
+# reStructuredText.
+cat > foo.log <<'EOF'
+SKIP: FooBar
+=============
+
+--------------
+ dummy title
+EOF
+END
+chmod a+x no-rst
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+VERBOSE=yes $MAKE check
+cat foo.log
+cat test-suite.log
+$FGREP 'dummy title' test-suite.log
+
+# Sanity check: trying to produce HTML output should fail.
+$MAKE check-html >output 2>&1 && { cat output; Exit 1; }
+cat output
+$FGREP SEVERE output
+
+:
diff --git a/tests/test-driver-custom-xfail-tests.test
b/tests/test-driver-custom-xfail-tests.test
new file mode 100755
index 0000000..e68e04b
--- /dev/null
+++ b/tests/test-driver-custom-xfail-tests.test
@@ -0,0 +1,166 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: "abstract" XFAIL_TESTS support.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<'END'
+AC_SUBST([nihil], [])
+AC_SUBST([ac_xfail_tests], ['x5.test x6$(test_suffix)'])
+AC_CONFIG_FILES([sub1/Makefile sub2/Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+SUBDIRS = . sub1 sub2
+TEST_LOG_DRIVER = $(srcdir)/td
+TESTS = pass.test xfail.test
+XFAIL_TESTS = xfail.test
+END
+
+mkdir sub1 sub2
+
+cat > sub1/Makefile.am <<END
+empty =
+
+TEST_LOG_DRIVER = \$(top_srcdir)/td
+
+# XFAIL_TESTS should gracefully handle TAB characters, and multiple
+# whitespaces.
+XFAIL_TESTS =\$(empty)${tab}x1.test x2.test${tab}x3.test${tab}\
+x4.test ${tab} x5.test x6.test${tab}\$(empty)
+
+TESTS = pass.test x1.test x2.test x3.test x4.test x5.test x6.test
+END
+
+cat > sub2/Makefile.am <<'END'
+AUTOMAKE_OPTIONS = -Wno-portability-recursive
+
+TEST_LOG_DRIVER = $(srcdir)/../td
+
+# XFAIL_TESTS should gracefully AC_SUBST @substitution@ and
+# make variables indirections.
+an_xfail_test = x1.test
+test_suffix = .test
+v0 = x3.test
+v1 = v
+v2 = 0
+XFAIL_TESTS = $(an_xfail_test) x2.test @nihil@ x3${test_suffix}
+XFAIL_TESTS += $($(v1)$(v2)) x4.test @ac_xfail_tests@
+
+TESTS = pass.test x1.test x2.test x3.test x4.test x5.test x6.test
+END
+
+cat > pass.test <<'END'
+#!/bin/sh
+exit 0
+END
+
+cat > xfail.test <<'END'
+#!/bin/sh
+exit 1
+END
+
+chmod a+x pass.test xfail.test
+
+cp pass.test sub1/pass.test
+cp pass.test sub2/pass.test
+
+for i in 1 2 3 4 5 6; do
+ cp xfail.test sub1/x$i.test
+ cp xfail.test sub2/x$i.test
+done
+
+cat > td <<'END'
+#! /bin/sh
+set -e; set -u
+test_name=INVALID
+log_file=/dev/null
+trs_file=/dev/null
+expect_failure=no
+while test $# -gt 0; do
+ case $1 in
+ --test-name) test_name=$2; shift;;
+ --expect-failure) expect_failure=$2; shift;;
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ # Ignored.
+ --color-tests) shift;;
+ --enable-hard-errors) shift;;
+ # Explicitly terminate option list.
+ --) shift; break;;
+ # Shouldn't happen
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+st=0
+"$@" || st=$?
+case $st,$expect_failure in
+ 0,no)
+ echo "PASS: $test_name" | tee "$log_file"
+ echo ":test-result: PASS" > "$trs_file"
+ ;;
+ 1,no)
+ echo "FAIL: $test_name" | tee "$log_file"
+ echo ":test-result: FAIL" > "$trs_file"
+ ;;
+ 0,yes)
+ echo "XPASS: $test_name" | tee "$log_file"
+ echo ":test-result: XPASS" > "$trs_file"
+ ;;
+ 1,yes)
+ echo "XFAIL: $test_name" | tee "$log_file"
+ echo ":test-result: XFAIL" > "$trs_file"
+ ;;
+ *)
+ echo "INTERNAL ERROR" >&2
+ exit 99
+ ;;
+esac
+END
+chmod a+x td
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+$MAKE check >stdout || { cat stdout; Exit 1; }
+cat stdout
+test `grep -c '^PASS:' stdout` -eq 3
+test `grep -c '^XFAIL:' stdout` -eq 13
+
+for dir in sub1 sub2; do
+ cd $dir
+ cp pass.test x1.test
+ cp x2.test pass.test
+ $MAKE check >stdout && { cat stdout; Exit 1; }
+ cat stdout
+ test "`cat pass.trs`" = ":test-result: FAIL"
+ test "`cat x1.trs`" = ":test-result: XPASS"
+ test "`cat x2.trs`" = ":test-result: XFAIL"
+ grep '^FAIL: pass\.test$' stdout
+ grep '^XPASS: x1\.test$' stdout
+ grep '^XFAIL: x2\.test$' stdout
+ count_test_results total=7 pass=0 xpass=1 fail=1 xfail=5 skip=0 error=0
+ cd ..
+done
+
+:
diff --git a/tests/test-driver-custom.test b/tests/test-driver-custom.test
new file mode 100755
index 0000000..8305047
--- /dev/null
+++ b/tests/test-driver-custom.test
@@ -0,0 +1,144 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: per-extension test drivers.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+# Automake shouldn't match the '/test' in 'sub/test' as '.test' suffix.
+TESTS = 1.chk 2.test 3 4.c.chk 5.suf sub/test
+
+TEST_EXTENSIONS = .chk .test
+
+CHK_LOG_DRIVER = ./chk-wrapper
+TEST_LOG_DRIVER = $(SHELL) $(srcdir)/test-wrapper
+LOG_DRIVER = noext-wrapper
+
+AM_CHK_LOG_DRIVER_FLAGS = --am-chk
+CHK_LOG_DRIVER_FLAGS = --chk
+AM_TEST_LOG_DRIVER_FLAGS = -am-test
+TEST_LOG_DRIVER_FLAGS = -test
+AM_LOG_DRIVER_FLAGS = am
+LOG_DRIVER_FLAGS = _
+END
+
+mkdir sub bin
+PATH=`pwd`/bin$PATH_SEPARATOR$PATH; export PATH
+
+cat > wrapper.skel <<'END'
+#! /bin/sh
+set -e
+
+me=`echo "$0" | sed 's,^.*/,,'`
+if test -z "$me"; then
+ echo "$0: cannot determine program name" >&2
+ exit 99
+fi
+
+am_log_wflags='@am_log_wflags@'
+log_wflags='@log_wflags@'
+
+test_name=INVALID
+log_file=BAD.log
+trs_file=BAD.trs
+extra_opts=
+while test $# -gt 0; do
+ case $1 in
+ --test-name) test_name=$2; shift;;
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ # Ignored.
+ --expect-failure) shift;;
+ --color-tests) shift;;
+ --enable-hard-errors) shift;;
+ # Remembered in the same order they're passed in.
+ $am_log_wflags|$log_wflags) extra_opts="$extra_opts $1";;
+ # Explicitly terminate option list.
+ --) shift; break;;
+ # Shouldn't happen
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+
+echo "$me" "$test_name" $extra_opts > "$log_file"
+: > "$trs_file"
+
+exec "$@"
+exit 127
+END
+
+sed -e 's|@am_log_wflags@|--am-chk|' \
+ -e 's|@log_wflags@|--chk|' \
+ < wrapper.skel > chk-wrapper
+
+sed -e 's|@am_log_wflags@|-am-test|' \
+ -e 's|@log_wflags@|-test|' \
+ < wrapper.skel > test-wrapper
+
+sed -e 's|@am_log_wflags@|am|' \
+ -e 's|@log_wflags@|_|' \
+ < wrapper.skel > bin/noext-wrapper
+
+# `test-wrapper' is deliberately not made executable.
+chmod a+x chk-wrapper bin/noext-wrapper
+
+# Not needed anymore.
+rm -f wrapper.skel
+
+cat > 1.chk << 'END'
+#! /bin/sh
+exit 0
+END
+chmod a+x 1.chk
+cp 1.chk 2.test
+cp 1.chk 3
+cp 1.chk 4.c.chk
+cp 1.chk 5.suf
+cp 1.chk sub/test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+$MAKE
+VERBOSE=yes $MAKE check
+ls -l . sub
+
+test ! -r BAD.log
+test ! -r BAD.trs
+
+echo 'chk-wrapper 1.chk --am-chk --chk' > 1.exp
+echo 'test-wrapper 2.test -am-test -test' > 2.exp
+echo 'noext-wrapper 3 am _' > 3.exp
+echo 'chk-wrapper 4.c.chk --am-chk --chk' > 4.c.exp
+echo 'noext-wrapper 5.suf am _' > 5.suf.exp
+echo 'noext-wrapper sub/test am _' > sub/test.exp
+
+st=0
+for x in 1 2 3 4.c 5.suf sub/test; do
+ cat $x.log
+ diff $x.exp $x.log || st=1
+done
+
+Exit $st
diff --git a/tests/test-driver-fail.test b/tests/test-driver-fail.test
new file mode 100755
index 0000000..e1d2f9c
--- /dev/null
+++ b/tests/test-driver-fail.test
@@ -0,0 +1,65 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: what happens when a test driver fails? Well,
+# "make check" should at least fail too, and the test-suite.log
+# shouldn't be created. Unfortunately, we cannot truly control also
+# the (non-)creation of individual test logs, since those are expected
+# to be created by the drivers themselves, and an ill-behaved driver
+# (like our dummy one in this test) might leave around a test log even
+# in case of internal failures.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TEST_LOG_DRIVER = ./oops
+TESTS = foo.test
+END
+
+cat > foo.test <<'END'
+#! /bin/sh
+exit 0
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# The testsuite driver does not exist.
+$MAKE check && Exit 1
+test ! -f test-suite.log
+
+# The testsuite driver exists and create the test log files, but fails.
+
+cat > oops <<'END'
+#!/bin/sh
+: > foo.log
+echo 'Oops, I fail!' >&2
+exit 1
+END
+chmod a+x oops
+
+$MAKE check && Exit 1
+test ! -f test-suite.log
+
+:
diff --git a/tests/test-driver-strip-vpath.test
b/tests/test-driver-strip-vpath.test
new file mode 100755
index 0000000..7187abb
--- /dev/null
+++ b/tests/test-driver-strip-vpath.test
@@ -0,0 +1,98 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers: check that the test name passed to the test
+# driver has any VPATH prefix stripped.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+ocwd=`pwd` || fatal_ "cannot get current working directory"
+
+mkdir src build
+mv install-sh missing configure.in src
+rm -f depcomp
+
+cd src
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+# The directories of the `bar.test' and `baz.test' tests are deliberately
+# called as the source directory, to verify that the VPATH-stripping code
+# doesn't get too easily confused.
+# The $(empty) are for eliciting VPATH rewrites on make implementations
+# that support it (e.g., Solaris make), to improve coverage.
+empty =
+TESTS = $(empty) foo.test src/bar.test ./src/baz.test $(empty)
+$(TESTS):
+TEST_LOG_DRIVER = $(srcdir)/checkstrip-driver
+EXTRA_DIST = checkstrip-driver
+END
+
+cat > checkstrip-driver <<'END'
+#! /bin/sh
+set -e
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+echo "test name: $test_name" # For debugging.
+case $test_name in
+ foo.test|./foo.test|src/ba[rz].test|./src/ba[rz].test);;
+ *) exit 1;;
+esac
+echo dummy > "$log_file"
+echo dummy > "$trs_file"
+END
+chmod a+x checkstrip-driver
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+cd ..
+
+mkdir build1
+cd build1
+../src/configure
+# "$MAKE -n" is for debugging, should highlight any VPATH rewrite.
+$MAKE -n check || :
+$MAKE check
+cd ..
+
+mkdir build2
+cd build2
+"$ocwd"/src/configure
+# "$MAKE -n" is for debugging, should highlight any VPATH rewrite.
+$MAKE -n check || :
+$MAKE check
+cd ..
+
+cd src
+./configure
+$MAKE distcheck
+
+:
diff --git a/tests/test-driver-trs-suffix-registered.test
b/tests/test-driver-trs-suffix-registered.test
new file mode 100755
index 0000000..7f6522a
--- /dev/null
+++ b/tests/test-driver-trs-suffix-registered.test
@@ -0,0 +1,58 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests support: the following are registered with `.SUFFIXES':
+# - .log
+# - .trs (used by files that store test results and metadata)
+# - .test if $(TEST_EXTENSIONS) is not defined
+# - stuff in $(TEST_EXTENSIONS) otherwise
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+: > Makefile.am
+
+cat > 1.am << 'END'
+TESTS =
+END
+
+cat > 2.am << 'END'
+TEST_EXTENSIONS = .SH .abcdef
+TESTS =
+END
+
+: > test-driver
+
+$ACLOCAL
+
+$AUTOMAKE 1
+$AUTOMAKE 2
+
+sed -e 's/$/ /' 1.in > mk.1
+sed -e 's/$/ /' 2.in > mk.2
+
+grep '^\.SUFFIXES:' mk.1
+grep '^\.SUFFIXES:' mk.2
+
+for suf in test log trs; do
+ grep "^\\.SUFFIXES:.* \\.$suf " mk.1
+done
+
+for suf in SH abcdef log trs; do
+ grep "^\\.SUFFIXES:.* \\.$suf " mk.2
+done
+
+:
diff --git a/tests/test-harness-vpath-rewrite.test
b/tests/test-harness-vpath-rewrite.test
new file mode 100755
index 0000000..57ff909
--- /dev/null
+++ b/tests/test-harness-vpath-rewrite.test
@@ -0,0 +1,59 @@
+#! /bin/sh
+# Copyright (C) 2010, 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that our concurrent test harness is not subject to spurious VPATH
+# rewrites.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+LOG_COMPILER = true
+empty =
+TESTS = x0 foo bar baz x1
+EXTRA_DIST = $(TESTS)
+END
+
+: > x0
+: > foo
+: > bar
+: > baz
+: > x1
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+mkdir build
+cd build
+../configure
+$MAKE check
+test -f test-suite.log
+test -f foo.log
+test -f foo.trs
+test -f bar.log
+test -f bar.trs
+test -f baz.log
+test -f baz.trs
+# We don't want VPATH rewrites, really.
+$MAKE check -n | $EGREP '\.\./(foo|bar|baz)( |$)' && Exit 1
+$MAKE distcheck
+
+:
diff --git a/tests/test-log.test b/tests/test-log.test
new file mode 100755
index 0000000..c25cb20
--- /dev/null
+++ b/tests/test-log.test
@@ -0,0 +1,179 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel-tests features:
+# - log file creation
+# - log file removal
+# - stdout and stderr of a test script go in its log file
+# - TEST_SUITE_LOG redefinition, at either automake or make time
+# - VERBOSE environment variable support
+# Keep in sync with 'tap-log.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<END
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = pass.test skip.test xfail.test fail.test xpass.test error.test
+XFAIL_TESTS = xpass.test xfail.test
+TEST_SUITE_LOG = global.log
+END
+
+# Custom markers, for use in grepping checks.
+cmarker=::: # comment marker
+pmarker=%%% # plain maker
+
+cat > pass.test <<END
+#! /bin/sh
+echo "$pmarker pass $pmarker" >&2
+echo "# $cmarker pass $cmarker" >&2
+exit 0
+END
+
+cat > skip.test <<END
+#! /bin/sh
+echo "$pmarker skip $pmarker"
+echo "# $cmarker skip $cmarker"
+exit 77
+END
+
+cat > xfail.test <<END
+#! /bin/sh
+echo "$pmarker xfail $pmarker" >&2
+echo "# $cmarker xfail $cmarker" >&2
+exit 1
+END
+
+cat > fail.test <<END
+#! /bin/sh
+echo 1..1
+echo "$pmarker fail $pmarker"
+echo "# $cmarker fail $cmarker"
+exit 1
+END
+
+cat > xpass.test <<END
+#! /bin/sh
+echo "$pmarker xpass $pmarker" >&2
+echo "# $cmarker xpass $cmarker" >&2
+exit 0
+END
+
+cat > error.test <<END
+#! /bin/sh
+echo "$pmarker error $pmarker"
+echo "# $cmarker error $cmarker"
+exit 99
+END
+
+chmod a+x *.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+TEST_SUITE_LOG=my.log $MAKE -e check && Exit 1
+ls -l # For debugging.
+test ! -f test-suite.log
+test ! -f global.log
+test -f my.log
+st=0
+for result in pass fail xfail xpass skip error; do
+ cat $result.log # For debugging.
+ $FGREP "$pmarker $result $pmarker" $result.log || st=1
+ $FGREP "$cmarker $result $cmarker" $result.log || st=1
+done
+test $st -eq 0 || Exit 1
+cat my.log # For debugging.
+for result in xfail fail xpass skip error; do
+ cat $result.log # For debugging.
+ $FGREP "$pmarker $result $pmarker" my.log || st=1
+ $FGREP "$cmarker $result $cmarker" my.log || st=1
+done
+test `$FGREP -c "$pmarker" my.log` -eq 5
+test `$FGREP -c "$cmarker" my.log` -eq 5
+
+have_rst_section ()
+{
+ eqeq=`echo "$1" | sed 's/./=/g'`
+ # Assume $1 contains no RE metacharacters.
+ sed -n "/^$1$/,/^$eqeq$/p" $2 > got
+ (echo "$1" && echo "$eqeq") > exp
+ cat exp
+ cat got
+ diff exp got
+}
+
+# Passed test scripts shouldn't be mentioned in the global log.
+$EGREP ':.*[^x]pass' my.log && Exit 1
+# But failing (expectedly or not) and skipped ones should.
+have_rst_section 'SKIP: skip' my.log
+have_rst_section 'FAIL: fail' my.log
+have_rst_section 'XFAIL: xfail' my.log
+have_rst_section 'XPASS: xpass' my.log
+have_rst_section 'ERROR: error' my.log
+
+touch error2.log test-suite.log global.log
+TEST_SUITE_LOG=my.log $MAKE -e mostlyclean
+ls -l # For debugging.
+test ! -f my.log
+test ! -f pass.log
+test ! -f fail.log
+test ! -f xfail.log
+test ! -f xpass.log
+test ! -f skip.log
+test ! -f error.log
+# "make mostlyclean" shouldn't remove unrelated log files.
+test -f error2.log
+test -f test-suite.log
+test -f global.log
+
+rm -f *.log
+
+VERBOSE=yes $MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+cat global.log
+test ! -f my.log
+test ! -f test-suite.log
+# Check that VERBOSE causes the global testsuite log to be
+# emitted on stdout.
+out=`cat stdout`
+log=`cat global.log`
+case $out in *"$log"*) ;; *) Exit 1;; esac
+
+touch error2.log test-suite.log my.log
+$MAKE clean
+ls -l # For debugging.
+test ! -f global.log
+test ! -f pass.log
+test ! -f fail.log
+test ! -f xfail.log
+test ! -f xpass.log
+test ! -f skip.log
+test ! -f error.log
+# "make clean" shouldn't remove unrelated log files.
+test -f error2.log
+test -f test-suite.log
+test -f my.log
+
+rm -f *.log
+
+:
diff --git a/tests/test-metadata-global-log.test
b/tests/test-metadata-global-log.test
new file mode 100755
index 0000000..a6f0999
--- /dev/null
+++ b/tests/test-metadata-global-log.test
@@ -0,0 +1,147 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Custom test drivers and parallel-tests harness: check the documented
+# semantics for deciding when the content of a test log file should be
+# copied in the global test-suite.log file. Currently, this is done
+# with the use of the reStructuredText field `:copy-in-global-log:' in
+# the associated `.trs' files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = ./passthrough-driver
+TEST_LOG_COMPILER = $(SHELL) -e
+END
+
+cat > passthrough-driver <<'END'
+#!/bin/sh
+set -e; set -u;
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+echo "$test_name: RUN"
+"$@" >$log_file 2>&1 5>$trs_file
+END
+chmod a+x passthrough-driver
+
+# The `:test-result:' and `:recheck:' fields and the first line of the
+# log file should be be irrelevant for the decision of whether a test
+# output is to be copied in the `test-suite.log'.
+
+cat > no-1.test <<END
+echo :test-result: SKIP >&5
+echo :copy-in-global-log: no >&5
+echo :test-result: FAIL >&5
+echo :test-result: XPASS >&5
+echo not seen 1
+END
+
+# In the last line, with leading and trailing whitespace in the value.
+cat > no-2.test <<END
+echo ":test-result: FAIL" >&5
+echo "not seen 2"
+echo ":recheck: yes" >&5
+echo ":copy-in-global-log:$tab $tab no $tab" >&5
+END
+
+for RES in XPASS FAIL XFAIL SKIP ERROR UNKNOWN; do
+ unindent > $RES.test <<END
+ echo :test-result: $RES >&5
+ echo :copy-in-global-log: no >&5
+ echo not seen $RES
+END
+done
+
+# In the first line, with no whitespace.
+cat > no-3.test <<END
+echo :copy-in-global-log:no >&5
+echo ":test-result: FAIL" >&5
+echo "not seen 3"
+END
+
+# Leading whitespace before the field.
+cat > no-4.test <<END
+echo ":test-result: FAIL" >&5
+echo " $tab $tab$tab :copy-in-global-log: no" >&5
+echo "not seen 4"
+END
+
+cat > yes-1.test <<END
+echo :test-result: PASS >&5
+echo :copy-in-global-log: yes >&5
+echo seen yes 1
+END
+
+# A lacking `:copy-in-global-log:' implies that the content of
+# the log file should be copied.
+cat > yes-2.test <<END
+echo :test-result: PASS >&5
+echo seen yes 2
+END
+
+# Three corner cases.
+
+cat > corn-1.test <<END
+echo seen corn 1
+echo ':copy-in-global-log:' >&5
+END
+
+cat > corn-2.test <<END
+echo seen corn 2
+echo '$tab $tab$tab' >&5
+END
+
+cat > corn-3.test <<'END'
+echo seen corn 31
+echo ':copy-in-global-log:address@hidden' >&5
+echo seen corn 32
+END
+
+echo TESTS = *.test >> Makefile.am
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# We don't care about the exit status of "make check" here, that
+# should be checked in other tests.
+$MAKE check || :
+cat test-suite.log
+grep '^seen yes 1$' test-suite.log
+grep '^seen yes 2$' test-suite.log
+grep '^seen corn 1$' test-suite.log
+grep '^seen corn 2$' test-suite.log
+grep '^seen corn 31$' test-suite.log
+grep '^seen corn 32$' test-suite.log
+$FGREP 'not seen' test-suite.log && Exit 1
+
+:
diff --git a/tests/test-metadata-global-result.test
b/tests/test-metadata-global-result.test
new file mode 100755
index 0000000..d20b249
--- /dev/null
+++ b/tests/test-metadata-global-result.test
@@ -0,0 +1,196 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Parallel testsuite harness: check APIs for the registering the
+# "global test result" in `*.trs' files, as documented in the automake
+# manual.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .test .x
+TEST_LOG_DRIVER = ./dummy-driver
+X_LOG_DRIVER = ./dummy-driver
+TESTS = foo.test zar-doz.test
+END
+
+cat > dummy-driver <<'END'
+#! /bin/sh
+set -e; set -u
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+echo logloglog > $log_file
+cp $1 $trs_file
+END
+chmod a+x dummy-driver
+
+# Do this in a subroutine to avoid quoting problem in the backticked
+# command substitution below.
+get_escaped_line()
+{
+ sed -e 's,[$^/\\\.],\\&,g' -e 1q "$@"
+}
+
+have_result ()
+{
+ cat > exp; echo >> exp; echo logloglog >> exp
+ eline=`get_escaped_line exp`
+ sed -n -e "/^$eline$/,/^logloglog$/p" test-suite.log > got
+ cat exp; cat got
+ diff exp got
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+: Basic checks.
+
+echo :global-test-result: PASS > foo.test
+echo :global-test-result: ERROR > zar-doz.x
+
+$MAKE check
+cat test-suite.log
+
+have_result <<END
+PASS: foo
+=========
+END
+
+have_result <<END
+ERROR: zar-doz
+==============
+END
+
+: Try usage documented in the manual.
+
+echo :global-test-result: PASS/SKIP > foo.test
+echo :global-test-result: ALMOST PASSED > zar-doz.x
+
+$MAKE check
+cat test-suite.log
+
+have_result <<END
+PASS/SKIP: foo
+==============
+END
+
+have_result <<END
+ALMOST PASSED: zar-doz
+======================
+END
+
+: Fields ':test-result:' does not interfere with the global test result.
+
+cat > foo.test << 'END'
+:test-result: FAIL
+:global-test-result: PASS
+:test-result: ERROR
+END
+
+cat > zar-doz.x << 'END'
+:global-test-result: FAIL
+:test-result: SKIP
+:test-result: XFAIL
+END
+
+$MAKE check && Exit 1
+cat test-suite.log
+
+have_result <<END
+PASS: foo
+=========
+END
+
+have_result <<END
+FAIL: zar-doz
+=============
+END
+
+: What happens when ':global-test-result:' is absent.
+
+cat > foo.test << 'END'
+:test-result: PASS
+:test-result: ERROR
+END
+: > zar-doz.x
+
+$MAKE check && Exit 1
+cat test-suite.log
+
+have_result <<END
+RUN: foo
+========
+END
+
+have_result <<END
+RUN: zar-doz
+============
+END
+
+# Leading and trailing whitespace gets eaten/normalized.
+
+echo ":global-test-result:SKIP${tab} ${tab}${tab}" > foo.test
+echo ":global-test-result:${tab} ${tab}XFAIL ${tab} " > zar-doz.x
+
+$MAKE check
+cat test-suite.log
+
+have_result <<END
+SKIP: foo
+=========
+END
+
+have_result <<END
+XFAIL: zar-doz
+==============
+END
+
+# Whitespaces before and after `:global-test-result:' are handled OK.
+
+echo " $tab:global-test-result:PASS" > foo.test
+echo "${tab}${tab}:global-test-result:${tab} ${tab}SKIP" > zar-doz.x
+
+$MAKE check
+cat test-suite.log
+
+have_result <<END
+PASS: foo
+=========
+END
+
+have_result <<END
+SKIP: zar-doz
+=============
+END
+
+:
diff --git a/tests/test-metadata-recheck.test b/tests/test-metadata-recheck.test
new file mode 100755
index 0000000..6a372f3
--- /dev/null
+++ b/tests/test-metadata-recheck.test
@@ -0,0 +1,163 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Test the "make recheck" semantics for custom test drivers, as documented
+# in the Automake manual.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+LOG_DRIVER = ./dummy-driver
+TEST_EXTENSIONS =
+TESTS =
+END
+
+#
+# Tests to be re-run by "make recheck"
+#
+
+: > y-1
+echo "foo bar" > y-2
+echo ":recheck:" > y-3
+echo ":recheck:yes" > y-4
+echo ":recheck: who cares" > y-5
+echo ":recheck: $tab y" > y-6
+echo ":recheck: yeah!$tab$tab " > y-7
+cat > y-10 <<END
+:foo:
+:recheck: ???
+END
+cat > y-11 <<END
+:recheck: YES
+:foo:
+END
+cat > y-12 <<END
+foo
+:recheck:yes
+
+bar
+zardoz
+END
+
+echo " $tab $tab$tab :recheck: yes" > y-8
+
+# The :test-result: fields and the fist line of the log should be
+# irrelevant for the decision of whether "make recheck" should or
+# should not re-run a test.
+
+echo ":test-result: PASS" > y-100
+
+echo "PASS: y-101"
+
+cat > y-102 <<END
+PASS: y-102
+===========
+
+:test-result: PASS
+END
+
+#
+# Tests *not* to be re-run by "make recheck"
+#
+
+echo ":recheck:no" > n-1
+echo ":recheck: no " > n-2
+echo ":recheck: $tab no" > n-3
+echo ":recheck: no $tab$tab " > n-4
+cat > n-5 <<END
+:foo:
+:recheck:no
+END
+cat > n-6 <<END
+:recheck: no
+:foo:
+END
+cat > n-7 <<END
+foo
+:recheck: no$tab$tab
+
+bar
+zardoz
+END
+
+echo " $tab $tab$tab :recheck: no" > n-8
+
+# The :test-result: fields should be irrelevant for the decision of
+# whether "make recheck" should or should not re-run a test.
+cat > n-100 <<END
+:test-result: FAIL
+:test-result: XPASS
+:test-result: ERROR
+:test-result: UNKNOWN
+:recheck: no
+END
+
+rechecked=`echo y-[0-9]*`
+
+for t in [yn]-[0-9]*; do echo $t; done \
+ | sed 's/.*/TESTS += &/' >> Makefile.am
+
+cat Makefile.am # For debugging.
+
+cat > dummy-driver <<'END'
+#!/bin/sh
+set -e; set -u
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+: > $test_name.run
+: > $log_file
+cp $1 $trs_file
+END
+chmod a+x dummy-driver
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# The ':test-result:' fields should be ignored by "make recheck",
+# but should cause the testsuite report to detect errors.
+$MAKE check && Exit 1
+ls -l
+for t in $tests; do test -f $t.run; done
+rm -f *.run
+
+# But now the tests that actually get re-run have only ':test-result:'
+# fields indicating success, so "make recheck" must pass. Still, the
+# next "make recheck" call should still re-run the same set of tests.
+for iteration in 1 2; do
+ $MAKE recheck
+ ls -l
+ for t in $rechecked; do test -f $t.run; done
+ find . -name 'n-*.run' | grep . && Exit 1
+done
+
+:
diff --git a/tests/test-metadata-results.test b/tests/test-metadata-results.test
new file mode 100755
index 0000000..923f88f
--- /dev/null
+++ b/tests/test-metadata-results.test
@@ -0,0 +1,176 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Parallel testsuite harness: check APIs for the registering of test
+# results in `*.trs' files, as documented in the automake manual.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = ./dummy-driver
+TESTS = foo.test bar.test
+END
+
+cat > dummy-driver <<'END'
+#! /bin/sh
+set -e; set -u
+while test $# -gt 0; do
+ case $1 in
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ --test-name) test_name=$2; shift;;
+ --expect-failure|--color-tests|--enable-hard-errors) shift;;
+ --) shift; break;;
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+: > $log_file
+cp $1 $trs_file
+END
+chmod a+x dummy-driver
+
+mk_check ()
+{
+ st=0
+ $MAKE check >stdout || st=$?
+ cat stdout
+ # Our dummy driver make no testsuite progress report.
+ grep ': .*\.test' stdout && Exit 1
+ # Nor it writes to the log files.
+ test -s foo.log && Exit 1
+ test -s bar.log && Exit 1
+ return $st
+}
+
+# This must be different from the one defined in `test/defs', as that
+# assumes that the driver does proper testsuite progress reporting.
+count_test_results ()
+{
+ total=ERR pass=ERR fail=ERR xpass=ERR xfail=ERR skip=ERR error=ERR
+ eval "$@"
+ st=0
+ grep "^# TOTAL: *$total$" stdout || rc=1
+ grep "^# PASS: *$pass$" stdout || rc=1
+ grep "^# XFAIL: *$xfail$" stdout || rc=1
+ grep "^# SKIP: *$skip$" stdout || rc=1
+ grep "^# FAIL: *$fail$" stdout || rc=1
+ grep "^# XPASS: *$xpass$" stdout || rc=1
+ grep "^# ERROR: *$error$" stdout || rc=1
+ test $st -eq 0 || Exit 1
+}
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE
+
+./configure
+
+# Basic checks. Also that that `:global-test-result:' fields and
+# "old-style" directives with format "RESULT: test-name" are now ignored.
+
+: > foo.test
+echo blah blah blah > bar.test
+mk_check
+count_test_results total=0 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+cat > foo.test <<END
+:test-global-result: PASS
+:test-result: FAIL
+END
+cat > bar.test <<END
+:test-result: SKIP
+:test-global-result: ERROR
+END
+mk_check && Exit 1
+count_test_results total=2 pass=0 fail=1 xpass=0 xfail=0 skip=1 error=0
+
+cat > foo.test <<END
+FAIL: foo.test
+:test-result: PASS
+:test-global-result: XPASS
+END
+echo ERROR: bar.test > bar.test
+mk_check
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+cat > foo.test <<END
+:test-global-result: SKIP
+:test-result: FAIL
+END
+cat > bar.test <<END
+:test-global-result: PASS
+END
+mk_check && Exit 1
+count_test_results total=1 pass=0 fail=1 xpass=0 xfail=0 skip=0 error=0
+
+cat > foo.test <<END
+:test-result: XFAIL
+:test-result: PASS
+:test-result: SKIP
+END
+cat > bar.test <<END
+:test-result: SKIP
+:test-result: PASS
+:test-result: SKIP
+:test-result: PASS
+:test-result: PASS
+END
+mk_check
+count_test_results total=8 pass=4 fail=0 xpass=0 xfail=1 skip=3 error=0
+
+# Check that all results expected to be supported are *really* supported.
+
+cat > foo.test <<END
+:test-result: PASS
+:test-result: SKIP
+:test-result: XFAIL
+:test-result: FAIL
+:test-result: XPASS
+:test-result: ERROR
+END
+: > bar.test
+mk_check && Exit 1
+count_test_results total=6 pass=1 fail=1 xpass=1 xfail=1 skip=1 error=1
+
+cp foo.test bar.test
+mk_check && Exit 1
+count_test_results total=12 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=2
+
+# Check that we are liberal w.r.t. whitespace use.
+
+: > foo.test
+: > bar.test
+for RESULT in PASS FAIL XPASS XFAIL SKIP ERROR; do
+ sed -e 's/^ *//' -e 's/|//g' >> foo.test <<END
+ |:test-result:$RESULT|
+ |:test-result: $tab $RESULT|
+ |:test-result:$RESULT $tab|
+ |:test-result:$tab$tab $RESULT$tab $tab |
+END
+ echo " $tab $tab$tab :test-result: $RESULT" >> bar.test
+done
+cat foo.test # For debugging.
+cat bar.test # Likewise.
+mk_check && Exit 1
+count_test_results total=30 pass=5 fail=5 xpass=5 xfail=5 skip=5 error=5
+
+:
diff --git a/tests/test-missing.test b/tests/test-missing.test
new file mode 100755
index 0000000..a4ac179
--- /dev/null
+++ b/tests/test-missing.test
@@ -0,0 +1,72 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests:
+# - non-existent scripts listed in TESTS get diagnosed
+# See also related test 'test-missing2.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = ok.test zardoz.test
+TEST_LOG_COMPILER = true
+END
+
+: > ok.test
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE check >output 2>&1 && { cat output; Exit 1; }
+cat output
+test -f ok.log
+grep '^PASS: ok\.test' output
+$FGREP 'zardoz.log' output
+test ! -f test-suite.log
+
+TESTS='zardoz2.test' $MAKE -e check >output 2>&1 \
+ && { cat output; Exit 1; }
+cat output
+$FGREP 'zardoz2.log' output
+test ! -f test-suite.log
+
+TEST_LOGS='zardoz3.log' $MAKE -e check >output 2>&1 \
+ && { cat output; Exit 1; }
+cat output
+$FGREP 'zardoz3.log' output
+test ! -f test-suite.log
+
+# The errors should persist even after `test-suite.log'
+# has been created.
+
+: > zardoz.test
+$MAKE check
+rm -f zardoz.test
+
+$MAKE check >output 2>&1 && { cat output; Exit 1; }
+cat output
+$FGREP 'zardoz.log' output
+test ! -f test-suite.log
+
+:
diff --git a/tests/test-missing2.test b/tests/test-missing2.test
new file mode 100755
index 0000000..bc2979a
--- /dev/null
+++ b/tests/test-missing2.test
@@ -0,0 +1,54 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# parallel-tests:
+# - non-existent scripts listed in TESTS get diagnosed, even when
+# all the $(TEST_LOGS) have a dummy dependency.
+# See also related test 'test-missing.test'.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in << 'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foobar1.test foobar2.test
+$(TEST_LOGS):
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+$MAKE foobar1.log foobar2.log || Exit 99
+test ! -f foobar1.log || Exit 99
+test ! -f foobar1.trs || Exit 99
+test ! -f foobar2.log || Exit 99
+test ! -f foobar2.trs || Exit 99
+
+$MAKE check >output 2>&1 && { cat output; Exit 1; }
+cat output
+grep 'test-suite\.log.*foobar1\.log' output
+grep 'test-suite\.log.*foobar1\.trs' output
+grep 'test-suite\.log.*foobar2\.log' output
+grep 'test-suite\.log.*foobar2\.trs' output
+test ! -f test-suite.log
+
+:
diff --git a/tests/test-trs-basic.test b/tests/test-trs-basic.test
new file mode 100755
index 0000000..8a8c3c8
--- /dev/null
+++ b/tests/test-trs-basic.test
@@ -0,0 +1,158 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel harness features:
+# - creation and removal of `.trs' auxiliary files
+# - check some internals regarding the use of `.trs' files.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<END
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_EXTENSIONS = .sh .test
+TESTS = foo.test bar.sh sub/zardoz.test
+TEST_LOG_COMPILER = $(SHELL)
+SH_LOG_COMPILER = $(SHELL)
+
+## Used to check some internal details. And yes, the quotes around
+## `$bases' are deliberate: they check for whitespace normalization.
+tb:
+ $(am__set_TESTS_bases); echo "$$bases" > $@
+END
+
+cat > foo.test << 'END'
+#! /bin/sh
+exit $FOO_STATUS
+END
+: > bar.sh
+mkdir sub
+: > sub/zardoz.test
+
+FOO_STATUS=0; export FOO_STATUS
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+
+#
+# Check some internal details first.
+#
+
+for vpath in : false; do
+
+ if $vpath; then
+ srcdir=..
+ mkdir build
+ cd build
+ else
+ srcdir=.
+ fi
+
+ $srcdir/configure
+
+ $MAKE tb
+ test x"`cat tb`" = x"foo bar sub/zardoz"
+ rm -f tb
+ # Please don't change the order of the stuff in TESTS, below.
+ TESTS='foo.test foo2.sh foo-log foolog.test a.log.b.sh 0.exe' $MAKE -e tb
+ test x"`cat tb`" = x"foo foo2 foo-log foolog a.log.b 0.exe"
+ rm -f tb
+
+ cd $srcdir
+
+done
+
+#
+# The `test-suite.stamp' file and the `.trs' files get created by
+# "make check" and removed by "make clean" and "make mostlyclean".
+#
+
+: > unrelated.trs
+: > sub/foo.trs
+
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+test -f sub/zardoz.trs
+$MAKE clean
+test ! -f foo.trs
+test ! -f bar.trs
+test ! -f sub/zardoz.trs
+# Unrelated `.trs' files shouldn't be removed.
+test -f unrelated.trs
+test -f sub/foo.trs
+
+# The files should be properly created in case of testsuite failure too.
+FOO_STATUS=1 $MAKE check && Exit 1
+test -f foo.trs
+test -f bar.trs
+test -f sub/zardoz.trs
+$MAKE mostlyclean
+test ! -f foo.trs
+test ! -f bar.trs
+test ! -f sub/zardoz.trs
+# Unrelated `.trs' files shouldn't be removed.
+test -f unrelated.trs
+test -f sub/foo.trs
+
+#
+# Try with a subset of TESTS.
+#
+
+TESTS=foo.test $MAKE -e check
+test -f foo.trs
+test ! -f bar.trs
+test ! -f sub/zardoz.trs
+$MAKE clean
+test ! -f foo.trs
+TESTS='foo.test bar.sh' $MAKE -e check
+test -f foo.trs
+test -f bar.trs
+test ! -f sub/zardoz.trs
+# "make clean" shouldn't remove `.trs' files for tests not in $(TESTS).
+TESTS=bar.sh $MAKE -e clean
+test -f foo.trs
+test ! -f bar.trs
+
+$MAKE clean
+
+#
+# Try with a subset of TEST_LOGS.
+#
+
+TEST_LOGS=sub/zardoz.log $MAKE -e check
+test ! -f foo.trs
+test ! -f bar.trs
+test -f sub/zardoz.trs
+$MAKE clean
+test ! -f sub/zardoz.trs
+TEST_LOGS='foo.log bar.log' $MAKE -e check
+test -f foo.trs
+test -f bar.trs
+test ! -f sub/zardoz.trs
+# "make clean" shouldn't remove `.trs' files for tests whose log
+# is not in $(TEST_LOGS).
+TEST_LOGS=foo.log $MAKE -e clean
+test ! -f foo.trs
+test -f bar.trs
+test ! -f sub/zardoz.trs
+
+:
diff --git a/tests/test-trs-recover.test b/tests/test-trs-recover.test
new file mode 100755
index 0000000..9893424
--- /dev/null
+++ b/tests/test-trs-recover.test
@@ -0,0 +1,167 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel harness features:
+# - recovery from deleted `.trs' files, in various scenarios
+# This test is complex and tricky, but that's acceptable since we are
+# testing semantics that are potentially complex and tricky.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<END
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test baz.test
+TEST_LOG_COMPILER = $(SHELL)
+END
+
+# FIXME: creative quoting below to please maintainer-check.
+echo exit '$TEST_STATUS' > foo.test
+echo exit '$TEST_STATUS' > bar.test
+: > baz.test
+
+TEST_STATUS=0; export TEST_STATUS
+
+# Slower and possible overkill in some situations, but also clearer
+# and safer.
+update_stamp () { $sleep && touch stamp && $sleep; }
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+: Create the required log files.
+$MAKE check
+
+: Recreate by hand.
+rm -f foo.trs bar.trs baz.trs
+$MAKE foo.trs
+test -f foo.trs
+test ! -f bar.trs
+test ! -f baz.trs
+
+: Recreate by hand, several at the same time.
+rm -f foo.trs bar.trs baz.trs
+$MAKE foo.trs bar.trs
+test -f foo.trs
+test -f bar.trs
+test ! -f baz.trs
+
+: Recreate by hand, with a failing test.
+rm -f foo.trs bar.trs
+TEST_STATUS=1 $MAKE bar.trs baz.trs >stdout || { cat stdout; Exit 1; }
+cat stdout
+test ! -f foo.trs
+test -f bar.trs
+test -f baz.trs
+grep '^FAIL: bar\.test' stdout
+$EGREP '^(baz|foo)\.test' stdout && Exit 1
+
+: Recreate with a sweeping "make check", and ensure that also up-to-date
+: '.trs' files are remade.
+update_stamp
+rm -f foo.trs bar.trs
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+is_newest baz.trs stamp
+
+: Recreate with a sweeping "make check" with failing tests. Again,
+: ensure that also up-to-date '.trs' files are remade -- this time we
+: grep the "make check" output verify that.
+rm -f foo.trs bar.trs
+TEST_STATUS=1 $MAKE check >stdout && { cat stdout; Exit 1; }
+test -f foo.trs
+test -f bar.trs
+grep '^FAIL: foo\.test' stdout
+grep '^FAIL: bar\.test' stdout
+grep '^PASS: baz\.test' stdout
+
+: Recreate with a "make check" with redefined TESTS.
+rm -f foo.trs bar.trs baz.trs
+TESTS=foo.test $MAKE -e check
+test -f foo.trs
+test ! -f bar.trs
+test ! -f baz.trs
+
+: Recreate with a "make check" with redefined TEST_LOGS.
+rm -f foo.trs bar.trs baz.trs
+TEST_LOGS=bar.log $MAKE -e check
+test ! -f foo.trs
+test -f bar.trs
+test ! -f baz.trs
+
+: Interactions with "make recheck" are OK.
+rm -f foo.trs bar.trs baz.log baz.trs
+$MAKE recheck >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -f foo.trs
+test -f bar.trs
+test ! -f baz.trs
+test ! -f baz.log
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+grep 'baz\.test' stdout && Exit 1
+count_test_results total=2 pass=2 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+: Setup for the next check.
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+test -f baz.trs
+
+: Recreate by remaking the global test log, and ensure that up-to-date
+: '.trs' files are *not* remade.
+update_stamp
+rm -f foo.trs bar.trs test-suite.log
+$MAKE test-suite.log >stdout || { cat stdout; Exit 1; }
+cat stdout
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+grep 'baz\.test' stdout && Exit 1
+stat *.trs *.log stamp || : # For debugging.
+# Check that make has updated what it needed to, but no more.
+test -f foo.trs
+test -f bar.trs
+is_newest stamp baz.trs
+is_newest test-suite.log foo.trs bar.trs
+
+: Setup for the next check.
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+test -f baz.trs
+
+: Interactions with lazy test reruns are OK.
+rm -f foo.trs
+update_stamp
+touch bar.test
+RECHECK_LOGS= $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+# Check that make has updated what it needed to, but no more.
+test -f foo.trs
+is_newest bar.trs bar.test
+is_newest stamp baz.trs
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+grep 'baz\.test' stdout && Exit 1
+
+:
diff --git a/tests/test-trs-recover2.test b/tests/test-trs-recover2.test
new file mode 100755
index 0000000..bc94b3b
--- /dev/null
+++ b/tests/test-trs-recover2.test
@@ -0,0 +1,133 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check parallel harness features:
+# - recovery from unreadable `.trs' files, in various scenarios
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+cat >> configure.in <<END
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TESTS = foo.test bar.test
+TEST_LOG_COMPILER = true
+END
+
+: > foo.test
+: > bar.test
+
+TEST_STATUS=0; export TEST_STATUS
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+./configure
+
+: > t
+chmod a-r t
+test ! -r t || Exit 77
+rm -f t
+
+: Create the required log files.
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+
+: Recreate with a sweeping "make check".
+chmod a-r bar.trs
+$MAKE check
+test -f foo.trs
+test -r foo.trs
+test -f bar.trs
+test -r bar.trs
+
+: Again, this time with one .trs file missing and the other
+: one unreadable.
+rm -f foo.trs
+chmod a-r bar.trs
+$MAKE check
+test -f foo.trs
+test -r foo.trs
+test -f bar.trs
+test -r bar.trs
+
+: Again, but using "make recheck" this time.
+rm -f foo.trs
+chmod a-r bar.trs
+$MAKE recheck >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -f foo.trs
+test -r foo.trs
+test -f bar.trs
+test -r bar.trs
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+
+: More complex interactions with "make recheck" are OK.
+chmod a-r bar.log bar.trs
+$MAKE recheck >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -f bar.trs
+test -r bar.trs
+grep '^PASS: bar\.test' stdout
+grep 'foo\.test' stdout && Exit 1
+count_test_results total=1 pass=1 fail=0 xpass=0 xfail=0 skip=0 error=0
+
+: Recreate by remaking the global test log.
+chmod a-r foo.trs
+rm -f test-suite.log
+$MAKE test-suite.log >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -f foo.trs
+test -r foo.trs
+grep '^PASS: foo\.test' stdout
+grep 'bar\.test' stdout && Exit 1
+# Also test that have only run before should be counted in the final
+# testsuite summary.
+grep '^# TOTAL: *2$' stdout
+
+: Setup for the next check.
+: > baz.test
+sed 's/^TESTS =.*/& baz.test/' Makefile > t
+diff t Makefile && Exit 99
+mv -f t Makefile
+$MAKE check
+test -f foo.trs
+test -f bar.trs
+test -f baz.trs
+
+: Interactions with lazy test reruns are OK.
+chmod a-r foo.trs
+$sleep
+touch stamp
+$sleep
+touch bar.test
+RECHECK_LOGS= $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+test -r foo.trs
+is_newest bar.trs bar.test
+grep '^PASS: foo\.test' stdout
+grep '^PASS: bar\.test' stdout
+grep 'baz\.test' stdout && Exit 1
+# Also test that have only run before should be counted in the final
+# testsuite summary.
+grep '^# TOTAL: *3$' stdout
+
+:
diff --git a/tests/tests-environment-and-log-compiler.test
b/tests/tests-environment-and-log-compiler.test
index 3753bb9..eafc3bf 100755
--- a/tests/tests-environment-and-log-compiler.test
+++ b/tests/tests-environment-and-log-compiler.test
@@ -65,7 +65,7 @@ $AUTOMAKE -a
./configure
-$MAKE -n check # For debugging.
+$MAKE -n check || : # For debugging.
VERBOSE=yes $MAKE check
cat a.log
cat b.log
diff --git a/tests/check-tests_environment.test b/tests/tests-environment.test
similarity index 100%
rename from tests/check-tests_environment.test
rename to tests/tests-environment.test
diff --git a/tests/testsuite-summary-checks.sh
b/tests/testsuite-summary-checks.sh
new file mode 100755
index 0000000..a4de864
--- /dev/null
+++ b/tests/testsuite-summary-checks.sh
@@ -0,0 +1,106 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check the testsuite summary with the parallel-tests harness. This
+# script is meant to be sourced by other test script, so that it can
+# be used to check different scenarios (colorized and non-colorized
+# testsuite output, packages with and without bug-report addresses,
+# testsuites in subdirectories, ...)
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+case $use_colors in
+ yes)
+ AM_COLOR_TESTS=always; export AM_COLOR_TESTS
+ TERM=ansi; export TERM
+ am_opts='parallel-tests color-tests'
+ ;;
+ no)
+ am_opts='parallel-tests'
+ ;;
+ *)
+ fatal_ "invalid use_colors='$use_colors'";;
+esac
+
+cat > configure.in <<END
+AC_INIT([GNU AutoFoo], [7.1], address@hidden)
+AM_INIT_AUTOMAKE([$am_opts])
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+TEST_EXTENSIONS = .t
+T_LOG_COMPILER = $(SHELL) $(srcdir)/log-compiler
+## Will be defined at runtime.
+TESTS =
+$(TESTS):
+END
+
+cat > log-compiler <<'END'
+#!/bin/sh
+case "$1" in
+ pass*|*/pass*|xpass*|*/xpass*) exit 0;;
+ fail*|*/fail*|xfail*|*/xfail*) exit 1;;
+ skip*|*/skip*) exit 77;;
+ error*|/error*) exit 99;;
+ copy*|*/copy*) cat "$1";;
+ *) exit 99;;
+esac
+END
+
+# Quite complexish, but allow the tests in client scripts to be written
+# in a "data-driven fashion".
+do_check ()
+{
+ cat > summary.exp
+ expect_failure=false
+ xfail_tests=''
+ tests="TESTS='$*'"
+ for t in $*; do
+ case $t in fail*|xpass*|error*) expect_failure=:;; esac
+ case $t in xfail*|xpass*) xfail_tests="$xfail_tests $t";; esac
+ done
+ test -z "$xfail_tests" || xfail_tests="XFAIL_TESTS='$xfail_tests'"
+ st=0
+ eval "env $tests $xfail_tests \$MAKE -e check > stdout || st=\$?"
+ cat stdout
+ if $expect_failure; then
+ test $st -gt 0 || Exit 1
+ else
+ test $st -eq 0 || Exit 1
+ fi
+ $PERL -w "$testsrcdir"/extract-testsuite-summary stdout > summary.got \
+ || fatal_ "cannot extract testsuite summary"
+ cat summary.exp
+ cat summary.got
+ if test $use_colors = yes; then
+ # Use cmp, not diff, because the files might contain binary data.
+ compare=cmp
+ else
+ compare=diff
+ fi
+ $compare summary.exp summary.got || Exit 1
+}
+
+br='============================================================================'
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE --add-missing
+
+:
diff --git a/tests/testsuite-summary-color.test
b/tests/testsuite-summary-color.test
new file mode 100755
index 0000000..9493744
--- /dev/null
+++ b/tests/testsuite-summary-color.test
@@ -0,0 +1,199 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check coloring of the testsuite summary.
+
+. ./defs-static || Exit 1
+
+use_colors=yes
+use_vpath=no
+
+. "$testsrcdir"/testsuite-summary-checks.sh || Exit 99
+
+./configure
+
+# ANSI colors.
+red='[0;31m'
+grn='[0;32m'
+lgn='[1;32m'
+blu='[1;34m'
+mgn='[0;35m'
+brg='[1m';
+std='[m';
+
+success_header="\
+${grn}${br}${std}
+${grn}Testsuite summary for GNU AutoFoo 7.1${std}
+${grn}${br}${std}"
+
+success_footer=${grn}${br}${std}
+
+failure_header="\
+${red}${br}${std}
+${red}Testsuite summary for GNU AutoFoo 7.1${std}
+${red}${br}${std}"
+
+failure_footer="\
+${red}${br}${std}
+${red}See ./test-suite.log${std}
+${red}Please report to address@hidden
+${red}${br}${std}"
+
+do_check '' <<END
+$success_header
+# TOTAL: 0
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check skip.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check xfail.t <<END
+$success_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check fail.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+do_check xpass.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+do_check error.t <<END
+$failure_header
+${brg}# TOTAL: 1${std}
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+do_check pass.t xfail.t skip.t <<END
+$success_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t fail.t skip.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+# XFAIL: 0
+${red}# FAIL: 1${std}
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+do_check pass.t xfail.t xpass.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+${grn}# PASS: 1${std}
+# SKIP: 0
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+${red}# XPASS: 1${std}
+# ERROR: 0
+$failure_footer
+END
+
+do_check skip.t xfail.t error.t <<END
+$failure_header
+${brg}# TOTAL: 3${std}
+# PASS: 0
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+# FAIL: 0
+# XPASS: 0
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+do_check pass.t skip.t xfail.t fail.t xpass.t error.t <<END
+$failure_header
+${brg}# TOTAL: 6${std}
+${grn}# PASS: 1${std}
+${blu}# SKIP: 1${std}
+${lgn}# XFAIL: 1${std}
+${red}# FAIL: 1${std}
+${red}# XPASS: 1${std}
+${mgn}# ERROR: 1${std}
+$failure_footer
+END
+
+:
diff --git a/tests/testsuite-summary-count-many.test
b/tests/testsuite-summary-count-many.test
new file mode 100755
index 0000000..b0232c7
--- /dev/null
+++ b/tests/testsuite-summary-count-many.test
@@ -0,0 +1,105 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check test counts in the testsuite summary, with test drivers allowing
+# multiple test results per test script, and for a huge number of tests.
+# Incidentally, this test also checks that the testsuite summary doesn't
+# give any bug-report address if it's not defined.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+for s in trivial-test-driver extract-testsuite-summary; do
+ cp "$testsrcdir/$s" . || fatal_ "failed to fetch auxiliary script $s"
+done
+
+br='============================================================================'
+
+header="\
+${br}
+Testsuite summary for $me 1.0
+${br}"
+
+footer="\
+${br}
+See ./test-suite.log
+${br}"
+
+echo AC_OUTPUT >> configure.in
+
+cat > Makefile.am << 'END'
+TEST_LOG_DRIVER = $(SHELL) $(srcdir)/trivial-test-driver
+TESTS = all.test
+# Without this, the test driver will be horrendously slow.
+END
+
+cat > all.test <<'END'
+#!/bin/sh
+cat results.txt || { echo ERROR: weird; exit 99; }
+END
+chmod a+x all.test
+
+$PERL -w -e '
+ use warnings FATAL => "all";
+ use strict;
+
+ # FIXME: we would like this to be 1000 or even 10000, but the current
+ # implementation is too slow to handle that :-(
+ my $base = 5;
+ my %count = (
+ TOTAL => $base * 1000,
+ PASS => $base * 700,
+ SKIP => $base * 200,
+ XFAIL => $base * 80,
+ FAIL => $base * 10,
+ XPASS => $base * 7,
+ ERROR => $base * 3,
+ );
+ my @results = qw/PASS SKIP XFAIL FAIL XPASS ERROR/;
+
+ open (RES, ">results.txt") or die "opening results.txt: $!\n";
+ open (CNT, ">count.txt") or die "opening count.txt: $!\n";
+
+ printf CNT "# %-6s %d\n", "TOTAL:", $count{TOTAL};
+ for my $res (@results)
+ {
+ my $uc_res = uc $res;
+ print STDERR "Generating list of $res ...\n";
+ for (1..$count{$res})
+ {
+ print RES "$uc_res: $_\n";
+ }
+ printf CNT "# %-6s %d\n", $res . ":", $count{$res};
+ }
+'
+
+(echo "$header" && cat count.txt && echo "$footer") > summary.exp
+
+$ACLOCAL
+$AUTOMAKE -a
+$AUTOCONF
+
+./configure
+
+($MAKE check || : > make.fail) | tee stdout
+test -f make.fail
+
+$PERL -w extract-testsuite-summary stdout > summary.got
+cat summary.exp
+cat summary.got
+diff summary.exp summary.got || Exit 1
+
+:
diff --git a/tests/testsuite-summary-count.test
b/tests/testsuite-summary-count.test
new file mode 100755
index 0000000..090fabb
--- /dev/null
+++ b/tests/testsuite-summary-count.test
@@ -0,0 +1,161 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check test counts in the testsuite summary.
+
+. ./defs-static || Exit 1
+
+use_colors=no
+use_vpath=no
+
+. "$testsrcdir"/testsuite-summary-checks.sh || Exit 99
+
+./configure
+
+header="\
+${br}
+Testsuite summary for GNU AutoFoo 7.1
+${br}"
+
+success_footer=${br}
+
+failure_footer="\
+${br}
+See ./test-suite.log
+Please report to address@hidden
+${br}"
+
+# Corner cases.
+
+do_check '' <<END
+$header
+# TOTAL: 0
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t <<END
+$header
+# TOTAL: 1
+# PASS: 1
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check fail.t <<END
+$header
+# TOTAL: 1
+# PASS: 0
+# SKIP: 0
+# XFAIL: 0
+# FAIL: 1
+# XPASS: 0
+# ERROR: 0
+$failure_footer
+END
+
+# Some simpler checks, with low or moderate number of tests.
+
+do_check skip.t skip2.t skip3.t xfail.t xfail2.t <<END
+$header
+# TOTAL: 5
+# PASS: 0
+# SKIP: 3
+# XFAIL: 2
+# FAIL: 0
+# XPASS: 0
+# ERROR: 0
+$success_footer
+END
+
+do_check pass.t pass2.t xfail.t xpass.t error.t error2.t <<END
+$header
+# TOTAL: 6
+# PASS: 2
+# SKIP: 0
+# XFAIL: 1
+# FAIL: 0
+# XPASS: 1
+# ERROR: 2
+$failure_footer
+END
+
+pass_count=22
+skip_count=19
+xfail_count=21
+fail_count=18
+xpass_count=23
+error_count=17
+tests_count=120
+
+pass=` seq_ 1 $pass_count | sed 's/.*/pass-&.t/'`
+skip=` seq_ 1 $skip_count | sed 's/.*/skip-&.t/'`
+xfail=`seq_ 1 $xfail_count | sed 's/.*/xfail-&.t/'`
+fail=` seq_ 1 $fail_count | sed 's/.*/fail-&.t/'`
+xpass=`seq_ 1 $xpass_count | sed 's/.*/xpass-&.t/'`
+error=`seq_ 1 $error_count | sed 's/.*/error-&.t/'`
+
+do_check $pass $skip $xfail $fail $xpass $error <<END
+$header
+# TOTAL: $tests_count
+# PASS: $pass_count
+# SKIP: $skip_count
+# XFAIL: $xfail_count
+# FAIL: $fail_count
+# XPASS: $xpass_count
+# ERROR: $error_count
+$failure_footer
+END
+
+# Mild stress test with a lot of test scripts.
+
+tests_count=1888
+pass_count=1403
+skip_count=292
+xfail_count=41
+fail_count=126
+xpass_count=17
+error_count=9
+
+pass=` seq_ 1 $pass_count | sed 's/.*/pass-&.t/'`
+skip=` seq_ 1 $skip_count | sed 's/.*/skip-&.t/'`
+xfail=`seq_ 1 $xfail_count | sed 's/.*/xfail-&.t/'`
+fail=` seq_ 1 $fail_count | sed 's/.*/fail-&.t/'`
+xpass=`seq_ 1 $xpass_count | sed 's/.*/xpass-&.t/'`
+error=`seq_ 1 $error_count | sed 's/.*/error-&.t/'`
+
+do_check $pass $skip $xfail $fail $xpass $error <<END
+$header
+# TOTAL: $tests_count
+# PASS: $pass_count
+# SKIP: $skip_count
+# XFAIL: $xfail_count
+# FAIL: $fail_count
+# XPASS: $xpass_count
+# ERROR: $error_count
+$failure_footer
+END
+
+:
diff --git a/tests/testsuite-summary-reference-log.test
b/tests/testsuite-summary-reference-log.test
new file mode 100755
index 0000000..7558b5f
--- /dev/null
+++ b/tests/testsuite-summary-reference-log.test
@@ -0,0 +1,88 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# Check that the global testsuite log file referenced in the testsuite
+# summary and in the global testsuite log itself is correct.
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+mv configure.in configure.stub
+
+cat > fail << 'END'
+#!/bin/sh
+exit 1
+END
+chmod a+x fail
+
+cat configure.stub - > configure.in <<'END'
+AC_OUTPUT
+END
+
+cat > Makefile.am << 'END'
+TEST_SUITE_LOG = my_test_suite.log
+TESTS = fail
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+mkdir build
+cd build
+
+../configure
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See \./my_test_suite\.log$' stdout
+
+mkdir bar
+TEST_SUITE_LOG=bar/bar.log $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See \./bar/bar\.log$' stdout
+
+cd ..
+
+echo SUBDIRS = sub > Makefile.am
+mkdir sub
+echo TESTS = fail > sub/Makefile.am
+mv fail sub
+
+cat configure.stub - > configure.in <<'END'
+AC_CONFIG_FILES([sub/Makefile])
+AC_OUTPUT
+END
+
+$ACLOCAL --force
+$AUTOCONF --force
+$AUTOMAKE
+
+./configure
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/test-suite\.log$' stdout
+cd sub
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/test-suite\.log$' stdout
+cd ..
+
+TEST_SUITE_LOG=foo.log $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+grep '^See sub/foo\.log$' stdout
+
+:
diff --git a/tests/trivial-test-driver b/tests/trivial-test-driver
new file mode 100644
index 0000000..7b3ef10
--- /dev/null
+++ b/tests/trivial-test-driver
@@ -0,0 +1,102 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#
+# Test driver for a very simple test protocol used by the Automake
+# testsuite to check support for custom test drivers allowing for more
+# test results per test script.
+#
+# The exit status of the wrapped script is ignored. Lines in its stdout
+# and stderr beginning with `PASS', `FAIL', `XFAIL', `XPASS', `SKIP' or
+# `ERROR' count as a test case result with the obviously-corresponding
+# outcome. Every other line is ignored for what concerns the testsuite
+# outcome.
+#
+# This script is used at least by the `driver-custom-multitest*.test'
+# tests.
+#
+
+# Help to avoid typo-related bugs.
+set -u
+
+## Option parsing.
+
+test_name=INVALID.NAME
+log_file=BAD.LOG
+trs_file=BAD.TRS
+while test $# -gt 0; do
+ case $1 in
+ --test-name) test_name=$2; shift;;
+ --log-file) log_file=$2; shift;;
+ --trs-file) trs_file=$2; shift;;
+ # Ignored.
+ --expect-failure) shift;;
+ --color-tests) shift;;
+ --enable-hard-errors) shift;;
+ # Explicitly terminate option list.
+ --) shift; break;;
+ # Shouldn't happen
+ *) echo "$0: invalid option/argument: '$1'" >&2; exit 2;;
+ esac
+ shift
+done
+
+## Log file header.
+{
+ echo "RUN: $test_name"
+ echo "RUN: $test_name" | sed 's/./=/g'
+ echo
+} > $log_file
+
+## Run the test script, get test cases results, display them on console.
+
+"$@" 2>&1 | tee -a $log_file | (
+ i=0 st=0
+ exec 5> $trs_file
+ while read line; do
+ result=
+ case $line in
+ PASS:*) result=PASS ;;
+ FAIL:*) result=FAIL ;;
+ XPASS:*) result=XPASS ;;
+ XFAIL:*) result=XFAIL ;;
+ SKIP:*) result=SKIP ;;
+ ERROR:*) result=ERROR ;;
+ esac
+ if test -n "$result"; then
+ case $result in FAIL|XPASS|ERROR) st=1;; esac
+ # Output testcase result to console.
+ echo "$result: $test_name"
+ # Register testcase outcome for the log file.
+ echo ":test-result: $line" >&5
+ echo >&5
+ fi
+ done
+ if test $st -eq 0; then
+ recheck=no
+ copy_in_global_log=no
+ else
+ recheck=yes
+ copy_in_global_log=yes
+ fi
+ echo ":recheck: $recheck" >&5
+ echo ":copy-in-global-log: $copy_in_global_log" >&5
+ exec 5>&-
+) | awk '{ print $0 ", testcase " NR }'
+
+## And we're done.
+
+exit 0
diff --git a/tests/warnings-win-over-strictness.test
b/tests/warnings-win-over-strictness.test
index 0db3176..83fced7 100755
--- a/tests/warnings-win-over-strictness.test
+++ b/tests/warnings-win-over-strictness.test
@@ -24,13 +24,13 @@ AUTOMAKE=$original_AUTOMAKE
ok ()
{
- AUTOMAKE_run 0 $*
+ AUTOMAKE_run $*
test ! -s stderr
}
ko ()
{
- AUTOMAKE_run 0 $*
+ AUTOMAKE_run $*
grep '^Makefile\.am:.*:=.*not portable' stderr
test `wc -l <stderr` -eq 1
}
diff --git a/tests/werror3.test b/tests/werror3.test
index 98cc8f0..33ad6f7 100755
--- a/tests/werror3.test
+++ b/tests/werror3.test
@@ -37,11 +37,11 @@ $ACLOCAL
# The issue would not manifest with threaded execution.
unset AUTOMAKE_JOBS || :
-AUTOMAKE_run 0 -Wno-error
+AUTOMAKE_run -Wno-error
grep 'VAR multiply defined' stderr
AUTOMAKE_JOBS=2
export AUTOMAKE_JOBS
-AUTOMAKE_run 0 -Wno-error
+AUTOMAKE_run -Wno-error
grep 'VAR multiply defined' stderr
:
diff --git a/tests/werror4.test b/tests/werror4.test
index 2ffaa4e..df41bdc 100755
--- a/tests/werror4.test
+++ b/tests/werror4.test
@@ -58,7 +58,7 @@ grep 'VAR multiply defined' stderr
grep 'SUB multiply defined' stderr
test `grep -c 'warnings are treated as errors' stderr` -eq 1
-AUTOMAKE_run 0 -Wno-error
+AUTOMAKE_run -Wno-error
grep 'VAR multiply defined' stderr
grep 'SUB multiply defined' stderr
grep 'warnings are treated as errors' stderr && Exit 1
hooks/post-receive
--
GNU Automake
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Automake-commit] [SCM] GNU Automake branch, testsuite-work, updated. v1.11-1174-ga956d98,
Stefano Lattarini <=