[FYI 0/2] {test-protocols} tap: improve granularity for tests on problematic TAP messages

[Stefano Lattarini]

* tests/tap-message-0.test: Break up into ...
* tests/tap-msg0-result.test, tests/tap-msg0-directive.test,
tests/tap-msg0-planskip.test, tests/tap-msg0-misc.test,
tests/tap-msg0-bailout.test: ... these new tests, and extend
a little.
* tests/Makefile.am (tap_with_common_setup_tests): Update.
---
 ChangeLog                                          |   10 +++
 tests/Makefile.am                                  |    6 +-
 tests/Makefile.in                                  |    6 +-
 tests/tap-msg0-bailout.test                        |   38 +++++++++
 tests/tap-msg0-directive.test                      |   41 ++++++++++
 tests/{tap-message-0.test => tap-msg0-misc.test}   |   83 ++++++++++----------
 tests/tap-msg0-planskip.test                       |   37 +++++++++
 tests/{tap-message-0.test => tap-msg0-result.test} |   52 ++++--------
 8 files changed, 195 insertions(+), 78 deletions(-)
 create mode 100755 tests/tap-msg0-bailout.test
 create mode 100755 tests/tap-msg0-directive.test
 copy tests/{tap-message-0.test => tap-msg0-misc.test} (50%)
 create mode 100755 tests/tap-msg0-planskip.test
 rename tests/{tap-message-0.test => tap-msg0-result.test} (54%)

diff --git a/ChangeLog b/ChangeLog
index 985cc86..4d83efb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,15 @@
 2011-08-17  Stefano Lattarini  <address@hidden>
 
+       tap: improve granularity for tests on problematic TAP messages
+       * tests/tap-message-0.test: Break up into ...
+       * tests/tap-msg0-result.test, tests/tap-msg0-directive.test,
+       tests/tap-msg0-planskip.test, tests/tap-msg0-misc.test,
+       tests/tap-msg0-bailout.test: ... these new tests, and extend
+       a little.
+       * tests/Makefile.am (tap_with_common_setup_tests): Update.
+
+2011-08-17  Stefano Lattarini  <address@hidden>
+
        tap: correctly handle string "0" in TAP messages
        * lib/tap-driver.pl (is_null_string): New function, can be used
        to determine whether a given string variable is empty or undefined.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 174f07b..853831d 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1161,9 +1161,13 @@ tap-global-log.test \
 tap-global-result.test \
 tap-html.test \
 tap-log.test \
+tap-msg0-result.test \
+tap-msg0-directive.test \
+tap-msg0-planskip.test \
+tap-msg0-bailout.test \
+tap-msg0-misc.test \
 tap-merge-stdout-stderr.test \
 tap-no-merge-stdout-stderr.test \
-tap-message-0.test \
 tap-no-disable-hard-error.test \
 tap-no-spurious-summary.test \
 tap-no-spurious.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index b014a8d..a1c152f 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -1402,9 +1402,13 @@ tap-global-log.test \
 tap-global-result.test \
 tap-html.test \
 tap-log.test \
+tap-msg0-result.test \
+tap-msg0-directive.test \
+tap-msg0-planskip.test \
+tap-msg0-bailout.test \
+tap-msg0-misc.test \
 tap-merge-stdout-stderr.test \
 tap-no-merge-stdout-stderr.test \
-tap-message-0.test \
 tap-no-disable-hard-error.test \
 tap-no-spurious-summary.test \
 tap-no-spurious.test \
diff --git a/tests/tap-msg0-bailout.test b/tests/tap-msg0-bailout.test
new file mode 100755
index 0000000..6aa6d60
--- /dev/null
+++ b/tests/tap-msg0-bailout.test
@@ -0,0 +1,38 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+#  - literal strings "0" and "0.0" as a test description in the message
+#    of a "plan with skip" TAP plan
+# generally true!
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo 'Bail out! 0' > a.test
+echo 'Bail out! 0.0' > b.test
+
+TESTS='a.test b.test' $MAKE -e check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=2
+
+grep '^ERROR: a.test - Bail out! 0$' stdout
+grep '^ERROR: b.test - Bail out! 0\.0$' stdout
+
+:
diff --git a/tests/tap-msg0-directive.test b/tests/tap-msg0-directive.test
new file mode 100755
index 0000000..6ec63f7
--- /dev/null
+++ b/tests/tap-msg0-directive.test
@@ -0,0 +1,41 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+#  - literal string "0" as a TODO or SKIP message
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+cat > all.test << 'END'
+1..3
+ok 1 # TODO 0
+not ok 2 # TODO 0
+ok 3 # SKIP 0
+END
+
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=3 pass=0 fail=0 xpass=1 xfail=1 skip=1 error=0
+
+grep '^XPASS: all\.test 1 # TODO 0$' stdout
+grep '^XFAIL: all\.test 2 # TODO 0$' stdout
+grep '^SKIP: all\.test 3 # SKIP 0$' stdout
+
+:
diff --git a/tests/tap-message-0.test b/tests/tap-msg0-misc.test
similarity index 50%
copy from tests/tap-message-0.test
copy to tests/tap-msg0-misc.test
index 87b2492..c9399bf 100755
--- a/tests/tap-message-0.test
+++ b/tests/tap-msg0-misc.test
@@ -15,8 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # TAP support:
-#  - having "0" as a test description (or TODO or SKIP or "Bail out!"
-#    message) should be supported
+#  - literal "0" and "0.0" in a test description and a TODO/SKIP message
+#    at the same time
 
 parallel_tests=yes
 . ./defs || Exit 1
@@ -24,58 +24,59 @@ parallel_tests=yes
 . "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
 
 cat > all.test << 'END'
-1..10
+1..14
+
 ok 1 0
-ok - 0
-not ok 3 0 # TODO
-not ok - 0 # TODO
-ok 5 0 # SKIP
-ok - 0 # SKIP
-not ok 7 # TODO 0
-ok 8 # SKIP 0
-not ok 9 0 # TODO 0
-ok 10 0 # SKIP 0
-END
+ok 2 0.0
 
-$MAKE check >stdout || { cat stdout; Exit 1; }
-cat stdout
+ok 3 0   # TODO 0
+ok 4 0.0 # TODO 0
+ok 5 0   # TODO 0.0
+ok 6 0.0 # TODO 0.0
 
-count_test_results total=10 pass=2 fail=0 xpass=0 xfail=4 skip=4 error=0
+not ok 7 0   # TODO 0
+not ok 8 0.0 # TODO 0
+not ok 9 0   # TODO 0.0
+not ok 10 0.0 # TODO 0.0
+
+ok 11 0   # SKIP 0
+ok 12 0.0 # SKIP 0
+ok 13 0   # SKIP 0.0
+ok 14 0.0 # SKIP 0.0
 
-cat > exp << 'END'
-PASS: all.test 1 0
-PASS: all.test 2 - 0
-XFAIL: all.test 3 0 # TODO
-XFAIL: all.test 4 - 0 # TODO
-SKIP: all.test 5 0 # SKIP
-SKIP: all.test 6 - 0 # SKIP
-XFAIL: all.test 7 # TODO 0
-SKIP: all.test 8 # SKIP 0
-XFAIL: all.test 9 0 # TODO 0
-SKIP: all.test 10 0 # SKIP 0
 END
 
-$FGREP ': all.test' stdout > got
+$MAKE check >stdout && { cat stdout; Exit 1; }
+cat stdout
 
-cat exp
-cat got
-diff exp got
+count_test_results total=14 pass=2 fail=0 xpass=4 xfail=4 skip=4 error=0
 
-echo '1..0 # SKIP 0' > all.test
-$MAKE check >stdout || { cat stdout; Exit 1; }
-cat stdout
+sed '/^ *$/d' > exp << 'END'
 
-count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
+PASS: all.test 1 0
+PASS: all.test 2 0.0
 
-grep '^SKIP: all.test - 0' stdout
+XPASS: all.test 3 0 # TODO 0
+XPASS: all.test 4 0.0 # TODO 0
+XPASS: all.test 5 0 # TODO 0.0
+XPASS: all.test 6 0.0 # TODO 0.0
 
-echo 'Bail out! 0' > all.test
-$MAKE check >stdout && { cat stdout; Exit 1; }
-cat stdout
+XFAIL: all.test 7 0 # TODO 0
+XFAIL: all.test 8 0.0 # TODO 0
+XFAIL: all.test 9 0 # TODO 0.0
+XFAIL: all.test 10 0.0 # TODO 0.0
+
+SKIP: all.test 11 0 # SKIP 0
+SKIP: all.test 12 0.0 # SKIP 0
+SKIP: all.test 13 0 # SKIP 0.0
+SKIP: all.test 14 0.0 # SKIP 0.0
 
-count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
+END
 
-grep '^ERROR: all.test - Bail out! 0' stdout
+$FGREP ': all.test' stdout > got
 
+cat exp
+cat got
+diff exp got
 
 :
diff --git a/tests/tap-msg0-planskip.test b/tests/tap-msg0-planskip.test
new file mode 100755
index 0000000..b8c3d6c
--- /dev/null
+++ b/tests/tap-msg0-planskip.test
@@ -0,0 +1,37 @@
+#! /bin/sh
+# Copyright (C) 2011 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# TAP support:
+#  - literal strings "0" and "0.0" as the reason of the skip in a "TAP
+#    plan with skip" (i.e., "1..0 # SKIP ...").
+
+parallel_tests=yes
+. ./defs || Exit 1
+
+. "$testsrcdir"/tap-setup.sh || fatal_ "sourcing tap-setup.sh"
+
+echo '1..0 # SKIP 0' > a.test
+echo '1..0 # SKIP 0.0' > b.test
+
+TESTS='a.test b.test' $MAKE -e check >stdout || { cat stdout; Exit 1; }
+cat stdout
+
+count_test_results total=2 pass=0 fail=0 xpass=0 xfail=0 skip=2 error=0
+
+grep '^SKIP: a.test - 0$' stdout
+grep '^SKIP: b.test - 0\.0$' stdout
+
+:
diff --git a/tests/tap-message-0.test b/tests/tap-msg0-result.test
similarity index 54%
rename from tests/tap-message-0.test
rename to tests/tap-msg0-result.test
index 87b2492..86259e5 100755
--- a/tests/tap-message-0.test
+++ b/tests/tap-msg0-result.test
@@ -15,8 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # TAP support:
-#  - having "0" as a test description (or TODO or SKIP or "Bail out!"
-#    message) should be supported
+#  - the string "0" as a test description
 
 parallel_tests=yes
 . ./defs || Exit 1
@@ -27,32 +26,32 @@ cat > all.test << 'END'
 1..10
 ok 1 0
 ok - 0
-not ok 3 0 # TODO
+not ok 3 0
+not ok - 0
+ok 5 0 # TODO
+ok - 0 # TODO
+not ok 7 0 # TODO
 not ok - 0 # TODO
-ok 5 0 # SKIP
+ok 9 0 # SKIP
 ok - 0 # SKIP
-not ok 7 # TODO 0
-ok 8 # SKIP 0
-not ok 9 0 # TODO 0
-ok 10 0 # SKIP 0
 END
 
-$MAKE check >stdout || { cat stdout; Exit 1; }
+$MAKE check >stdout && { cat stdout; Exit 1; }
 cat stdout
 
-count_test_results total=10 pass=2 fail=0 xpass=0 xfail=4 skip=4 error=0
+count_test_results total=10 pass=2 fail=2 xpass=2 xfail=2 skip=2 error=0
 
 cat > exp << 'END'
 PASS: all.test 1 0
 PASS: all.test 2 - 0
-XFAIL: all.test 3 0 # TODO
-XFAIL: all.test 4 - 0 # TODO
-SKIP: all.test 5 0 # SKIP
-SKIP: all.test 6 - 0 # SKIP
-XFAIL: all.test 7 # TODO 0
-SKIP: all.test 8 # SKIP 0
-XFAIL: all.test 9 0 # TODO 0
-SKIP: all.test 10 0 # SKIP 0
+FAIL: all.test 3 0
+FAIL: all.test 4 - 0
+XPASS: all.test 5 0 # TODO
+XPASS: all.test 6 - 0 # TODO
+XFAIL: all.test 7 0 # TODO
+XFAIL: all.test 8 - 0 # TODO
+SKIP: all.test 9 0 # SKIP
+SKIP: all.test 10 - 0 # SKIP
 END
 
 $FGREP ': all.test' stdout > got
@@ -61,21 +60,4 @@ cat exp
 cat got
 diff exp got
 
-echo '1..0 # SKIP 0' > all.test
-$MAKE check >stdout || { cat stdout; Exit 1; }
-cat stdout
-
-count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=1 error=0
-
-grep '^SKIP: all.test - 0' stdout
-
-echo 'Bail out! 0' > all.test
-$MAKE check >stdout && { cat stdout; Exit 1; }
-cat stdout
-
-count_test_results total=1 pass=0 fail=0 xpass=0 xfail=0 skip=0 error=1
-
-grep '^ERROR: all.test - Bail out! 0' stdout
-
-
 :
-- 
1.7.2.3