FYI: Breaking the test suite

[Akim Demaille]

Well, more accurately, fine tuning the test suite and uncovering
things to fix identically once for all in all the parsers.  This was
while preparing the test suite to exercise more yydestruct cases.

Issue number one is that yacc.c (Paul Eggert IIRC) and lalr1.c
(Alexandre Duret-Lutz IIRC) address the "error" location differently,
and IMHO, on the examples I have, lalr1.cc's answer is better (and so
does believe valgrind by the way).  For instance:

bison/tests/_build % _srcdir/testsuite -d 41 43                 nostromo 16:27
## ---------------------------- ##
## GNU Bison 1.875e test suite. ##
## ---------------------------- ##
 41: Printers and Destructors :                   FAILED (actions.at:470)
 43: Printers and Destructors : %defines %skeleton lalr1.ccok

## ------------- ##
## Test results. ##
## ------------- ##

ERROR: All 2 tests were run,
1 failed unexpectedly.
## -------------------------- ##
## testsuite.log was created. ##
## -------------------------- ##

Please send `testsuite.log' and all information you think might help:

   To: <address@hidden>
   Subject: [GNU Bison 1.875e] testsuite: 41 failed

bison/tests/_build % testsuite.dir/041/input '(y)' | grep error nostromo 16:28
10-19: syntax error, unexpected 'y', expecting 'x'
line (address@hidden): '(' (address@hidden) error (@0-19) ')' (address@hidden)
bison/tests/_build % testsuite.dir/043/input '(y)' | grep error nostromo 16:28
10-19: syntax error, unexpected 'y', expecting 'x'
line (address@hidden): '(' (address@hidden) error (@10-19) ')' (address@hidden)

Yes, the location of the error is more 10-19 (i.e., 2nd character)
than 0-19 (i.e. spanning from the first to the second).

Anyway, I don't have time right now to address this.  I'll have a
closer look later.


Index: ChangeLog
from  Akim Demaille  <address@hidden>

        * tests/actions.at (_AT_CHECK_PRINTER_AND_DESTRUCTOR): Move the
        Bison directive from the Bison file to the invocation of this
        macro, so that these directives are passed to
        AT_BISON_OPTION_PUSHDEFS to get correct help macros.
        Use these helping macros (e.g., AT_LOC, AT_VAL and so forth).
        Move the AT_SETUP/AT_CLEANUP outside, to report as test title
        the extra Bison directives instead of the whole series.
        Change the grammar so that there are recoverable errors, and
        unrecoverable errors.  Now we can have the parser give up before
        consuming the whole input.  As a result we now can observe that
        the lookahead is freed when needed.
        Change the parser source to parse argv[1] instead of a hard coded
        string.
        Simplify yylex, and give a value and location to EOF.
        Simplify some invocations of AT_CHECK_PRINTER_AND_DESTRUCTOR that
        passed directives already coded in the file.
        Add some tests to check the location of "error".
        For some tests, the C++ parser is correct, and not yacc.c.
        For other tests, they provide different, but unsatisfying, values,
        so keep the C++ value so that at least one parser is "correct"
        according to the test suite.
        (Actions after errors): Remove, this is subsumed by the
        AT_CHECK_PRINTER_AND_DESTRUCTOR series.

Index: tests/actions.at
===================================================================
RCS file: /cvsroot/bison/bison/tests/actions.at,v
retrieving revision 1.34
diff -u -u -r1.34 actions.at
--- tests/actions.at 2 Sep 2004 13:04:11 -0000 1.34
+++ tests/actions.at 8 Sep 2004 14:24:14 -0000
@@ -82,151 +82,6 @@
 
 
 
-## ---------------------- ##
-## Actions after errors.  ##
-## ---------------------- ##
-
-AT_SETUP([Actions after errors])
-
-
-
-AT_DATA_GRAMMAR([[input.y]],
-[[%{
-#include <stdio.h>
-#include <stdlib.h>
-
-static int yylex (void);
-static void yyerror (char const *);
-
-#define YYDEBUG 1
-%}
-%union { int ival; }
-%type <ival> 'x' ';' thing line input
-
-%%
-input:
-  /* Nothing. */
-    {
-      $$ = 0;
-      printf ("input (%d): /* Nothing */\n", $$);
-    }
-| line input /* Right recursive to load the stack so that popping at
-               EOF can be exercised.  */
-    {
-      $$ = 2;
-      printf ("input (%d): line (%d) input (%d)\n", $$, $1, $2);
-    }
-;
-
-line:
-  thing thing thing ';'
-    {
-      $$ = $1;
-      printf ("line (%d): thing (%d) thing (%d) thing (%d) ';' (%d)\n",
-             $$, $1, $2, $3, $4);
-    }
-| thing thing ';'
-    {
-      $$ = $1;
-      printf ("line (%d): thing (%d) thing (%d) ';' (%d)\n", $$, $1, $2, $3);
-    }
-| thing ';'
-    {
-      $$ = $1;
-      printf ("line (%d): thing (%d) ';' (%d)\n", $$, $1, $2);
-    }
-| error ';'
-    {
-      $$ = -1;
-      printf ("line (%d): error ';' (%d)\n", $$, $2);
-    }
-;
-
-thing:
-  'x'
-    {
-      $$ = $1;
-      printf ("thing (%d): 'x' (%d)\n", $$, $1);
-    }
-;
-%%
-static size_t counter;
-
-static int
-yylex (void)
-{
-  static char const input[] =
-    {
-      /* Exercise the discarding of stack top and input until `error'
-         can be reduced.  */
-      'x', 'x', 'x', 'x', 'x', 'x', ';',
-
-      /* Load the stack and provoke an error that cannot be caught by
-         the grammar, to check that the stack is cleared. */
-      'x', 'x', ';',
-      'x', ';',
-      'y'
-    };
-
-  if (counter < sizeof input)
-    {
-       yylval.ival = counter;
-       printf ("sending: '%c' (%d)\n", input[counter], yylval.ival);
-       return input[counter++];
-    }
-  else
-    {
-      printf ("sending: EOF\n");
-      return EOF;
-    }
-}
-
-static void
-yyerror (char const *msg)
-{
-  printf ("%lu: %s\n", (unsigned long int) counter, msg);
-}
-
-int
-main (void)
-{
-  yydebug = !!getenv ("YYDEBUG");
-  return yyparse ();
-}
-]])
-
-AT_CHECK([bison -o input.c input.y])
-AT_COMPILE([input])
-AT_PARSER_CHECK([./input], 1,
-[[sending: 'x' (0)
-thing (0): 'x' (0)
-sending: 'x' (1)
-thing (1): 'x' (1)
-sending: 'x' (2)
-thing (2): 'x' (2)
-sending: 'x' (3)
-4: syntax error
-sending: 'x' (4)
-sending: 'x' (5)
-sending: ';' (6)
-line (-1): error ';' (6)
-sending: 'x' (7)
-thing (7): 'x' (7)
-sending: 'x' (8)
-thing (8): 'x' (8)
-sending: ';' (9)
-line (7): thing (7) thing (8) ';' (9)
-sending: 'x' (10)
-thing (10): 'x' (10)
-sending: ';' (11)
-line (10): thing (10) ';' (11)
-sending: 'y' (12)
-13: syntax error
-sending: EOF
-]])
-
-AT_CLEANUP
-
 
 
 ## ---------------- ##
@@ -307,47 +162,44 @@
 # _AT_CHECK_PRINTER_AND_DESTRUCTOR($1, $2, $3, $4, BISON-DIRECTIVE, UNION-FLAG)
 # -----------------------------------------------------------------------------
 m4_define([_AT_CHECK_PRINTER_AND_DESTRUCTOR],
-[m4_if([$1$2$3], $[1]$[2]$[3], [],
+[# Make sure complex $n work.
+m4_if([$1$2$3], $[1]$[2]$[3], [],
        [m4_fatal([$0: Invalid arguments: address@hidden)])dnl
 
-AT_SETUP([Printers and Destructors $6: $5])
-
-# Make sure complex $n work.
-
+# Be sure to pass all the %directives to this macro to have correct
+# helping macros.  So don't put any directly in the Bison file.
 AT_BISON_OPTION_PUSHDEFS([$5])
 AT_DATA_GRAMMAR([[input.y]],
-[[$5
-%{
+[[%{
 #include <stdio.h>
 #include <stdlib.h>
+#include <assert.h>
 ]AT_LALR1_CC_IF(
   [#define RANGE(Location) (Location).begin.line, (Location).end.line],
   [#define RANGE(Location) (Location).first_line, (Location).last_line])
 [%}
-%error-verbose
-%debug
-%verbose
-%locations
-]m4_ifval([$6], [%union
+
+$5]
+m4_ifval([$6], [%union
 {
   int ival;
 }])
 [
 %{
 ]AT_LALR1_CC_IF([typedef yy::Location YYLTYPE;
-m4_ifval([$6], , [#define YYSTYPE int])])
+                m4_ifval([$6], , [#define YYSTYPE int])])
 [static int yylex (]AT_LEX_FORMALS[);
 ]AT_LALR1_CC_IF([], [static void yyerror (const char *msg);])
 [%}
 
-]m4_ifval([$6], [%type <ival> 'x' ';' thing line input])[
+]m4_ifval([$6], [%type <ival> '(' 'x' 'y' ')' ';' thing line input])[
 
 %printer
   {
     ]AT_LALR1_CC_IF([cdebug_ << @$ << ": " << $$;],
                     [fprintf (yyoutput, "address@hidden", $$, RANGE (@$))])[;
   }
-  input line thing 'x'
+  input line thing 'x' 'y'
 
 %destructor
   { printf ("Freeing nterm input (address@hidden)\n", $$, RANGE (@$)); }
@@ -365,7 +217,18 @@
   { printf ("Freeing token 'x' (address@hidden)\n", $$, RANGE (@$)); }
   'x'
 
+%destructor
+  { printf ("Freeing token 'y' (address@hidden)\n", $$, RANGE (@$)); }
+  'y'
+
 %%
+/*
+   This grammar is made to exercise error recovery.
+   "Lines" starting with `(' support error recovery, with
+   ')' as synchronizing token.  Lines starting with 'x' can never
+   be recovered from if in error.
+*/
+
 input:
   /* Nothing. */
     {
@@ -389,23 +252,24 @@
               $$, RANGE (@$), $1, RANGE (@1), $2, RANGE (@2),
               $3, RANGE (@3), $4, RANGE (@4));
     }
-| thing thing ';'
+| '(' thing thing ')'
     {
       $$ = $1;
-      printf ("line (address@hidden): thing (address@hidden) thing 
(address@hidden) ';' (address@hidden)\n",
-              $$, RANGE (@$), $1, RANGE (@1), $2, RANGE (@2), $3, RANGE (@3));
+      printf ("line (address@hidden): '(' (address@hidden) thing 
(address@hidden) thing (address@hidden) ')' (address@hidden)\n",
+              $$, RANGE (@$), $1, RANGE (@1), $2, RANGE (@2),
+              $3, RANGE (@3), $4, RANGE (@4));
     }
-| thing ';'
+| '(' thing ')'
     {
       $$ = $1;
-      printf ("line (address@hidden): thing (address@hidden) ';' 
(address@hidden)\n",
-              $$, RANGE (@$), $1, RANGE (@1), $2, RANGE (@2));
+      printf ("line (address@hidden): '(' (address@hidden) thing 
(address@hidden) ')' (address@hidden)\n",
+              $$, RANGE (@$), $1, RANGE (@1), $2, RANGE (@2), $3, RANGE (@3));
     }
-| error ';'
+| '(' error ')'
     {
       $$ = -1;
-      printf ("line (address@hidden): error (@%d-%d) ';' (address@hidden)\n",
-              $$, RANGE (@$), RANGE (@1), $2, RANGE (@2));
+      printf ("line (address@hidden): '(' (address@hidden) error (@%d-%d) ')' 
(address@hidden)\n",
+              $$, RANGE (@$), $1, RANGE (@1), RANGE (@2), $3, RANGE (@3));
     }
 ;
 
@@ -418,48 +282,30 @@
     }
 ;
 %%
+/* Alias to ARGV[1]. */
+const char *yysource = 0;
+
 static int
 yylex (]AT_LEX_FORMALS[)
 {
-  static const char input[] =
-    {
-      /* Exercise the discarding of stack top and input until `error'
-         can be reduced.  */
-      'x', 'x', 'x', 'x', 'x', 'x', ';',
-
-      /* Load the stack and provoke an error that cannot be caught by
-         the grammar, to check that the stack is cleared. */
-      'x', 'x', ';',
-      'x', ';',
-      'y'
-    };
   static unsigned int counter = 0;
 
-  if (counter < (sizeof(input) / sizeof (input[0])))
-    {
+  int c = ]AT_VAL[]m4_ifval([$6], [.ival])[ = counter++;
+  /* As in BASIC, line numbers go from 10 to 10.  */
 ]AT_LALR1_CC_IF(
-[       int c = m4_ifval([$6], [yylval->ival], [*yylval]) = counter++;
-       /* As in BASIC, line numbers go from 10 to 10.  */
-       yylloc->begin.line = yylloc->begin.column = 10 * c;
-       yylloc->end.line = yylloc->end.column = yylloc->begin.line + 9;
-       printf ("sending: '%c' (address@hidden)\n",
-               input[[c]], c, RANGE (*yylloc));
-       return input[[c]];
+[ AT_LOC.begin.line = AT_LOC.begin.column = 10 * c;
+  AT_LOC.end.line = AT_LOC.end.column = AT_LOC.begin.line + 9;
 ],
-[       int c = m4_ifval([$6], [yylval.ival], [yylval]) = counter++;
-       /* As in BASIC, line numbers go from 10 to 10.  */
-       yylloc.first_line = yylloc.first_column = 10 * c;
-       yylloc.last_line = yylloc.last_column = yylloc.first_line + 9;
-       printf ("sending: '%c' (address@hidden)\n",
-               input[[c]], c, RANGE (yylloc));
-       return input[[c]];
+[ AT_LOC.first_line = AT_LOC.first_column = 10 * c;
+  AT_LOC.last_line = AT_LOC.last_column = AT_LOC.first_line + 9;
 ])[
-    }
+
+  if (yysource[c])
+    printf ("sending: '%c'", yysource[c]);
   else
-    {
-      printf ("sending: EOF\n");
-      return EOF;
-    }
+    printf ("sending: EOF");
+  printf (" (address@hidden)\n", c, RANGE (]AT_LOC[));
+  return yysource[c];
 }
 
 ]AT_LALR1_CC_IF(
@@ -492,9 +338,11 @@
 }])[
 
 int
-main (void)
+main (int argc, const char *argv[])
 {
   yydebug = !!getenv ("YYDEBUG");
+  assert (argc == 2);
+  yysource = argv[1];
   if (yyparse ())
     {
       printf ("Parsing FAILED.\n");
@@ -510,61 +358,119 @@
    AT_COMPILE_CXX([input])],
   [AT_CHECK([bison -o input.c input.y])
    AT_COMPILE([input])])
-AT_PARSER_CHECK([./input], 1,
-[[sending: 'x' (address@hidden)
-thing (address@hidden): 'x' (address@hidden)
+
+
+# Check the location of "empty"
+# -----------------------------
+# I.e., epsilon-reductions, as in "(x)" which ends by reducing
+# an empty "line" nterm.
+# FIXME: This location is not satisfying.  Depend on the lookahead?
+AT_PARSER_CHECK([./input '(x)'], 0,
+[[sending: '(' (address@hidden)
+sending: 'x' (address@hidden)
+thing (address@hidden): 'x' (address@hidden)
+sending: ')' (address@hidden)
+line (address@hidden): '(' (address@hidden) thing (address@hidden) ')' 
(address@hidden)
+sending: EOF (address@hidden)
+input (address@hidden): /* Nothing */
+input (address@hidden): line (address@hidden) input (address@hidden)
+Successful parse.
+]])
+
+
+# Check locations in error recovery
+# ---------------------------------
+# '(y)' is an error, but can be recovered from.  But what's the location
+# of the error itself ('y'), and of the resulting reduction ('(error)').
+AT_PARSER_CHECK([./input '(y)'], 0,
+[[sending: '(' (address@hidden)
+sending: 'y' (address@hidden)
+10-19: syntax error, unexpected 'y', expecting 'x'
+Freeing token 'y' (address@hidden)
+sending: ')' (address@hidden)
+line (address@hidden): '(' (address@hidden) error (@10-19) ')' (address@hidden)
+sending: EOF (address@hidden)
+input (address@hidden): /* Nothing */
+input (address@hidden): line (address@hidden) input (address@hidden)
+Successful parse.
+]])
+
+
+# Syntax errors caught by the parser
+# ----------------------------------
+# Exercise the discarding of stack top and input until `error'
+# can be reduced.
+#
+#     '(', 'x', 'x', 'x', 'x', 'x', ')',
+#
+# Load the stack and provoke an error that cannot be caught by the
+# grammar, to check that the stack is cleared.  And make sure the
+# lookahead is freed.
+#
+#     '(', 'x', ')',
+#     '(', 'x', ')',
+#     'y'
+AT_PARSER_CHECK([./input '(xxxxx)(x)(x)y'], 1,
+[[sending: '(' (address@hidden)
 sending: 'x' (address@hidden)
 thing (address@hidden): 'x' (address@hidden)
 sending: 'x' (address@hidden)
 thing (address@hidden): 'x' (address@hidden)
 sending: 'x' (address@hidden)
-30-39: syntax error, unexpected 'x', expecting ';'
+30-39: syntax error, unexpected 'x', expecting ')'
 Freeing nterm thing (address@hidden)
 Freeing nterm thing (address@hidden)
-Freeing nterm thing (address@hidden)
 Freeing token 'x' (address@hidden)
 sending: 'x' (address@hidden)
 Freeing token 'x' (address@hidden)
 sending: 'x' (address@hidden)
 Freeing token 'x' (address@hidden)
-sending: ';' (address@hidden)
-line (address@hidden): error (@0-59) ';' (address@hidden)
-sending: 'x' (address@hidden)
-thing (address@hidden): 'x' (address@hidden)
+sending: ')' (address@hidden)
+line (address@hidden): '(' (address@hidden) error (@10-59) ')' (address@hidden)
+sending: '(' (address@hidden)
 sending: 'x' (address@hidden)
 thing (address@hidden): 'x' (address@hidden)
-sending: ';' (address@hidden)
-line (address@hidden): thing (address@hidden) thing (address@hidden) ';' 
(address@hidden)
-sending: 'x' (address@hidden)
-thing (address@hidden): 'x' (address@hidden)
-sending: ';' (address@hidden)
-line (address@hidden): thing (address@hidden) ';' (address@hidden)
-sending: 'y' (address@hidden)
-120-129: syntax error, unexpected $undefined, expecting $end or 'x'
-sending: EOF
-Freeing nterm line (address@hidden)
-Freeing nterm line (address@hidden)
-Freeing nterm line (address@hidden)
+sending: ')' (address@hidden)
+line (address@hidden): '(' (address@hidden) thing (address@hidden) ')' 
(address@hidden)
+sending: '(' (address@hidden)
+sending: 'x' (address@hidden)
+thing (address@hidden): 'x' (address@hidden)
+sending: ')' (address@hidden)
+line (address@hidden): '(' (address@hidden) thing (address@hidden) ')' 
(address@hidden)
+sending: 'y' (address@hidden)
+input (address@hidden): /* Nothing */
+input (address@hidden): line (address@hidden) input (address@hidden)
+input (address@hidden): line (address@hidden) input (address@hidden)
+input (address@hidden): line (address@hidden) input (address@hidden)
+130-139: syntax error, unexpected 'y', expecting $end
+Freeing nterm input (address@hidden)
+Freeing token 'y' (address@hidden)
 Parsing FAILED.
 ]])
 
-AT_CLEANUP
 ])
 
 
 # AT_CHECK_PRINTER_AND_DESTRUCTOR([BISON-OPTIONS], [UNION-FLAG])
 # --------------------------------------------------------------
-# Produce `calc.y'.
 m4_define([AT_CHECK_PRINTER_AND_DESTRUCTOR],
-[_AT_CHECK_PRINTER_AND_DESTRUCTOR($[1], $[2], $[3], $[4], [$1], [$2])
+[AT_SETUP([Printers and Destructors $2: $1])
+
+_AT_CHECK_PRINTER_AND_DESTRUCTOR($[1], $[2], $[3], $[4],
+[%error-verbose
+%debug
+%verbose
+%locations
+$1], [$2])
+
+AT_CLEANUP
 ])
 
 
 AT_CHECK_PRINTER_AND_DESTRUCTOR([])
 AT_CHECK_PRINTER_AND_DESTRUCTOR([], [with union])
-AT_CHECK_PRINTER_AND_DESTRUCTOR([%locations %defines %skeleton "lalr1.cc"])
-AT_CHECK_PRINTER_AND_DESTRUCTOR([%locations %defines %skeleton "lalr1.cc"],
-                               [with union])
+AT_CHECK_PRINTER_AND_DESTRUCTOR([%defines %skeleton "lalr1.cc"])
+AT_CHECK_PRINTER_AND_DESTRUCTOR([%defines %skeleton "lalr1.cc"], [with union])
 
 # FIXME.  These test cases fail.
 #AT_CHECK_PRINTER_AND_DESTRUCTOR([%glr-parser])