[PATCH] Refactor tests on Automake TESTS color output.

[Stefano Lattarini]

Hello automakers.

The current version of the testcase `color.test' (which checks 
Automake TESTS color output) runs first some tests using only
make and grep, and then runs some more tests using the expect(1)
program; but, in doing so, it makes the *testcase resulting SKIP'd if 
a call to expect(1) fails*.

The fact that a test can be considered SKIP'd after having run some
real checks (which could have potentially failed, making the test
itself FAIL'd) "smells bad" to me.  IMHO, the best option would be to
divide the current test script into two tests, one which does not use
expect(1), the other one which does and which is SKIP'd upfront if no 
working expect(1) program is found.  This is exactly what's done in
the attached patch.  I hope you'll agree with me, and find the patch
useful.

Regards,
    Stefano

From 5258dd5abcc2f9d33ba2a407cde71f1660b8b464 Mon Sep 17 00:00:00 2001
From: Stefano Lattarini <address@hidden>
Date: Wed, 14 Apr 2010 13:52:51 +0200
Subject: [PATCH] Refactor tests on Automake TESTS color output.

* tests/color.test: Tests using the expect program moved out to...
* tests/color2.test: ... this new file.
* tests/Makefile.am (TESTS): Extended accordingly.
---
 ChangeLog         |    7 +++
 tests/Makefile.am |    1 +
 tests/Makefile.in |    2 +
 tests/color.test  |   22 ++--------
 tests/color2.test |  124 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 138 insertions(+), 18 deletions(-)
 create mode 100755 tests/color2.test

diff --git a/ChangeLog b/ChangeLog
index d7f3c23..21a51cb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2010-04-14  Stefano Lattarini  <address@hidden>
+
+       Refactor tests on Automake TESTS color output.
+       * tests/color.test: Tests using the expect program moved out to...
+       * tests/color2.test: ... this new file.
+       * tests/Makefile.am (TESTS): Extended accordingly.
+
 2010-04-11  Ralf Wildenhues  <address@hidden>
 
        Print 'warnings are treated as errors' note if needed.
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 2255af6..feb5374 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -149,6 +149,7 @@ colon5.test \
 colon6.test \
 colon7.test \
 color.test \
+color2.test \
 comment.test \
 comment2.test \
 comment3.test \
diff --git a/tests/Makefile.in b/tests/Makefile.in
index beb1ac9..6cca2bf 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -270,6 +270,7 @@ check7-p.test \
 check8-p.test \
 check9-p.test \
 color-p.test \
+color2-p.test \
 comment9-p.test \
 dejagnu-p.test \
 exeext4-p.test \
@@ -389,6 +390,7 @@ colon5.test \
 colon6.test \
 colon7.test \
 color.test \
+color2.test \
 comment.test \
 comment2.test \
 comment3.test \
diff --git a/tests/color.test b/tests/color.test
index 6fa686d..9d86785 100755
--- a/tests/color.test
+++ b/tests/color.test
@@ -1,5 +1,5 @@
 #! /bin/sh
-# Copyright (C) 2007, 2008, 2010  Free Software Foundation, Inc.
+# Copyright (C) 2007, 2008, 2010 Free Software Foundation, Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,7 +14,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-# Test Automake TESTS color output.
+# Test Automake TESTS color output, by forcing it.
+# Keep this in sync with the sister test `color2.test'.
 
 . ./defs || Exit 1
 
@@ -34,7 +35,7 @@ std=''
 # GNU or BSD 'grep -a' works on files, but is not portable.
 case `echo "$std" | grep .` in
   $std) ;;
-  *) Exit 77 ;;
+  *) echo "$me: grep can't parse nonprinting characters" >&2; Exit 77;;
 esac
 
 cat >>configure.in <<END
@@ -93,21 +94,6 @@ test_no_color ()
   :
 }
 
-cat >expect-make <<'END'
-#! /usr/bin/expect -f
-spawn $env(MAKE) -e check
-expect eof
-END
-
 AM_COLOR_TESTS=always $MAKE -e check >stdout && { cat stdout; Exit 1; }
 cat stdout
 test_color
-
-MAKE=$MAKE expect -f expect-make >stdout || { cat stdout; Exit 77; }
-cat stdout
-test_color
-
-AM_COLOR_TESTS=no MAKE=$MAKE expect -f expect-make >stdout || { cat stdout; 
Exit 77; }
-cat stdout
-test_no_color
-:
diff --git a/tests/color2.test b/tests/color2.test
new file mode 100755
index 0000000..eedd37d
--- /dev/null
+++ b/tests/color2.test
@@ -0,0 +1,124 @@
+#! /bin/sh
+# Copyright (C) 2007, 2008, 2010 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Test Automake TESTS color output, using the expect(1) program.
+# Keep this in sync with the sister test `color.test'.
+
+. ./defs || Exit 1
+
+set -e
+
+TERM=ansi
+export TERM
+
+red=''
+grn=''
+lgn=''
+blu=''
+std=''
+
+# Check that grep can parse nonprinting characters.
+# BSD 'grep' works from a pipe, but not a seekable file.
+# GNU or BSD 'grep -a' works on files, but is not portable.
+case `echo "$std" | grep .` in
+  $std) ;;
+  *) echo "$me: grep can't parse nonprinting characters" >&2; Exit 77;;
+esac
+
+# Check that we have a working expect program.
+cat >expect-check <<'END'
+#! /usr/bin/expect -f
+spawn $env(THE_SYSTEM_SHELL) -c :
+expect eof
+END
+THE_SYSTEM_SHELL=/bin/sh expect -f expect-check || {
+    echo "$me: failed to find a working expect program" >&2
+    Exit 77
+}
+
+# Do the tests.
+
+cat >>configure.in <<END
+AC_OUTPUT
+END
+
+cat >Makefile.am <<'END'
+AUTOMAKE_OPTIONS = color-tests
+TESTS = $(check_SCRIPTS)
+check_SCRIPTS = pass fail skip xpass xfail
+XFAIL_TESTS = xpass xfail
+END
+
+cat >pass <<END
+#! /bin/sh
+exit 0
+END
+
+cat >fail <<END
+#! /bin/sh
+exit 1
+END
+
+cat >skip <<END
+#! /bin/sh
+exit 77
+END
+
+cp fail xfail
+cp pass xpass
+chmod +x pass fail skip xpass xfail
+
+$ACLOCAL
+$AUTOMAKE
+$AUTOCONF
+./configure
+
+test_color ()
+{
+  # Not a useless use of cat; see above comments about grep.
+  cat stdout | grep ": pass" | $FGREP "$grn"
+  cat stdout | grep ": fail" | $FGREP "$red"
+  cat stdout | grep ": xfail" | $FGREP "$lgn"
+  cat stdout | grep ": xpass" | $FGREP "$red"
+  cat stdout | grep ": skip" | $FGREP "$blu"
+}
+
+test_no_color ()
+{
+  # Not a useless use of cat; see above comments about grep.
+  cat stdout | grep ": pass" | $FGREP "$grn" && Exit 1
+  cat stdout | grep ": fail" | $FGREP "$red" && Exit 1
+  cat stdout | grep ": xfail" | $FGREP "$lgn" && Exit 1
+  cat stdout | grep ": xpass" | $FGREP "$red" && Exit 1
+  cat stdout | grep ": skip" | $FGREP "$blu" && Exit 1
+  :
+}
+
+cat >expect-make <<'END'
+#! /usr/bin/expect -f
+spawn $env(MAKE) -e check
+expect eof
+END
+
+MAKE=$MAKE expect -f expect-make >stdout \
+  || { cat stdout; Exit 1; }
+cat stdout
+test_color
+
+AM_COLOR_TESTS=no MAKE=$MAKE expect -f expect-make >stdout \
+  || { cat stdout; Exit 1; }
+cat stdout
+test_no_color
-- 
1.6.5