19-fyi-tok-error.patch

[Akim Demaille]

Index: ChangeLog
from  Akim Demaille  <address@hidden>
        * src/main.c (main): If there are complains after grammar
        reductions, then output the report anyway if requested, then die.
        * src/symtab.c (bucket_new): Initialize `value' to -1, not 0.
        * src/reader.c (eoftoken): New.
        (parse_token_decl): If the token being defined has value `0', it
        is the eoftoken.
        (packsymbols): No longer hack `tags' to insert `$' by hand.
        Be sure to preserve the value of the eoftoken.
        (reader): Make sure eoftoken is defined.
        Initialize nsyms to 0: now eoftoken is created just like the others.
        * src/print.c (print_grammar): Don't special case the eof token.
        * src/regression.at: Adjust: `$' has value 0, not -1, which was a
        lie anyway, albeit pleasant.
        * tests/calc.at: Exercise error messages with eoftoken.
        Change the grammar so that empty input is invalid.
        Adjust expectations.
        When yyungeting, be sure to use a valid yylloc: use last_yylloc.
        
        
Index: NEWS
--- NEWS Wed, 26 Dec 2001 20:07:27 +0100 akim
+++ NEWS Thu, 27 Dec 2001 13:54:46 +0100 akim
@@ -3,10 +3,18 @@
 
 Changes in version 1.49a:
 
-* items overflow
+* Items overflow
   Bison no longer dumps core when there are too many items, it just
   dies.
 
+* Token end-of-file
+  The token end of file may be specified by the user, in which case,
+  the user symbol is used in the reports, the graphs, and the verbose
+  error messages instead of `$', which remains being the defaults.
+  For instance
+     %token YYEOF 0
+  or
+     %token YYEOF 0 "end of file"
 
 Changes in version 1.30:
 
Index: src/main.c
--- src/main.c Wed, 26 Dec 2001 19:05:39 +0100 akim
+++ src/main.c Thu, 27 Dec 2001 12:39:29 +0100 akim
@@ -86,14 +86,14 @@
   /* Output file names. */
   compute_output_file_names ();
 
+  /* Output the detailed report on the grammar.  */
+  if (verbose_flag)
+    print_results ();
+
   /* Stop if there were errors, to avoid trashing previous output
      files.  */
   if (complain_message_count)
     exit (1);
-
-  /* Output the detailed report on the grammar.  */
-  if (verbose_flag)
-    print_results ();
 
   /* Output the VCG graph.  */
   if (graph_flag)
Index: src/print.c
--- src/print.c Thu, 27 Dec 2001 11:44:04 +0100 akim
+++ src/print.c Thu, 27 Dec 2001 12:52:15 +0100 akim
@@ -390,8 +390,6 @@
 
   /* TERMINAL (type #) : rule #s terminal is on RHS */
   fprintf (out, "%s\n\n", _("Terminals, with rules where they appear"));
-  fprintf (out, "%s (-1)\n", escape (tags[0]));
-
   for (i = 0; i <= max_user_token_number; i++)
     if (token_translations[i] != 2)
       {
Index: src/reader.c
--- src/reader.c Thu, 27 Dec 2001 12:01:10 +0100 akim
+++ src/reader.c Thu, 27 Dec 2001 14:02:22 +0100 akim
@@ -68,9 +68,9 @@
 /* Incremented for each %left, %right or %nonassoc seen */
 static int lastprec;
 
-static bucket *errtoken;
-static bucket *undeftoken;
-
+static bucket *errtoken = NULL;
+static bucket *undeftoken = NULL;
+static bucket *eoftoken = NULL;
 
 static symbol_list *
 symbol_list_new (bucket *sym)
@@ -441,7 +441,7 @@
     {
       obstack_fgrow2 (&attrs_obstack, muscle_find ("linef"),
                      lineno, quotearg_style (c_quoting_style,
-                                             muscle_find("filename")));
+                                             muscle_find ("filename")));
     }
 
   after_percent = 0;
@@ -573,6 +573,9 @@
       else if (symbol && token == tok_number)
        {
          symbol->user_token_number = numval;
+         /* User defined EOF token? */
+         if (numval == 0)
+           eoftoken = symbol;
        }
       else
        {
@@ -902,8 +905,8 @@
 parse_muscle_decl (void)
 {
   int ch = ungetc (skip_white_space (), finput);
-  char* muscle_key;
-  char* muscle_value;
+  char *muscle_key;
+  char *muscle_value;
 
   /* Read key. */
   if (!isalpha (ch) && ch != '_')
@@ -1486,7 +1489,7 @@
     {
       obstack_fgrow2 (&el_obstack, muscle_find ("linef"),
                      lineno, quotearg_style (c_quoting_style,
-                                             muscle_find("filename")));
+                                             muscle_find ("filename")));
     }
 
   while ((c = getc (finput)) != EOF)
@@ -1549,7 +1552,6 @@
   bucket *bp = NULL;
   int tokno = 1;
   int last_user_token_number;
-  static char DOLLAR[] = "$";
 
   tags = XCALLOC (char *, nsyms + 1);
   user_toknums = XCALLOC (short, nsyms + 1);
@@ -1557,10 +1559,6 @@
   sprec = XCALLOC (short, nsyms);
   sassoc = XCALLOC (short, nsyms);
 
-  /* The EOF token. */
-  tags[0] = DOLLAR;
-  user_toknums[0] = 0;
-
   max_user_token_number = 256;
   last_user_token_number = 256;
 
@@ -1572,11 +1570,18 @@
        }
       else if (bp->alias)
        {
-         /* this symbol and its alias are a single token defn.
-            allocate a tokno, and assign to both check agreement of
-            ->prec and ->assoc fields and make both the same */
-         if (bp->value == 0)
-           bp->value = bp->alias->value = tokno++;
+         /* This symbol and its alias are a single token defn.
+            Allocate a tokno, and assign to both check agreement of
+            prec and assoc fields and make both the same */
+         if (bp->value == -1)
+           {
+             if (bp == eoftoken || bp->alias == eoftoken)
+               bp->value = bp->alias->value = 0;
+             else
+               {
+                 bp->value = bp->alias->value = tokno++;
+               }
+           }
 
          if (bp->prec != bp->alias->prec)
            {
@@ -1602,13 +1607,17 @@
                bp->assoc = bp->alias->assoc;
            }
 
+         /* Do not do processing below for SALIASs.  */
          if (bp->user_token_number == SALIAS)
-           continue;           /* do not do processing below for SALIASs */
+           continue;
 
        }
-      else                     /* bp->class == token_sym */
+      else /* bp->class == token_sym */
        {
-         bp->value = tokno++;
+         if (bp == eoftoken)
+           bp->value = 0;
+         else
+           bp->value = tokno++;
        }
 
       if (bp->class == token_sym)
@@ -1774,7 +1783,7 @@
   start_flag = 0;
   startval = NULL;             /* start symbol not specified yet. */
 
-  nsyms = 1;
+  nsyms = 0;
   nvars = 0;
   nrules = 0;
   nitems = 0;
@@ -1818,6 +1827,16 @@
      TABLE_OBSTACK and FDEFINES file.  Also notice any %token, %left,
      etc. found there.  */
   read_declarations ();
+
+  /* If the user did not define her EOFTOKEN, do it now. */
+  if (!eoftoken)
+    {
+      eoftoken = getsym ("$");
+      eoftoken->class = token_sym;
+      /* Value specified by POSIX.  */
+      eoftoken->user_token_number = 0;
+    }
+
   /* Read in the grammar, build grammar in list form.  Write out
      guards and actions.  */
   readgram ();
Index: src/symtab.c
--- src/symtab.c Thu, 22 Nov 2001 21:32:24 +0100 akim
+++ src/symtab.c Thu, 27 Dec 2001 12:06:27 +0100 akim
@@ -55,7 +55,7 @@
   res->next = NULL;
   res->tag = xstrdup (tag);
   res->type_name = NULL;
-  res->value = 0;
+  res->value = -1;
   res->prec = 0;
   res->assoc = right_assoc;
   res->user_token_number = SUNDEF;
Index: tests/calc.at
--- tests/calc.at Sat, 08 Dec 2001 13:24:47 +0100 akim
+++ tests/calc.at Thu, 27 Dec 2001 14:55:16 +0100 akim
@@ -64,8 +64,8 @@ m4_define([_AT_DATA_CALC_Y],
 %}
 
 /* Bison Declarations */
-%token CALC_EOF 0
-%token NUM
+%token CALC_EOF 0 "end of file"
+%token NUM "number"
 
 %nonassoc '=' /* comparison           */
 %left '-' '+'
@@ -78,7 +78,7 @@ m4_define([_AT_DATA_CALC_Y],
 /* Grammar follows */
 %%
 input:
-  /* empty string */
+  line
 | input line
 ;
 
@@ -118,11 +118,16 @@ exp:
   fprintf (stderr, "%s\n", s);
 }
 
+
+#if YYLSP_NEEDED
+static YYLTYPE last_yylloc;
+#endif
 static int
 yygetc (void)
 {
   int res = getc (yyin);
 #if YYLSP_NEEDED
+  last_yylloc = yylloc;
   if (res == '\n')
     {
       yylloc.last_line++;
@@ -140,7 +145,7 @@ exp:
 {
 #if YYLSP_NEEDED
   /* Wrong when C == `\n'. */
-  yylloc.last_column--;
+  yylloc = last_yylloc;
 #endif
   ungetc (c, yyin);
 }
@@ -286,6 +291,9 @@ m4_define([_AT_CHECK_CALC],
 # ------------------------------------------------------------
 # Run `calc' on INPUT, and expect a `parse error' message.
 #
+# If INPUT starts with a slash, it is used as absolute input file name,
+# otherwise as contents.
+#
 # If BISON-OPTIONS contains `--location', then make sure the ERROR-LOCATION
 # is correctly output on stderr.
 #
@@ -296,11 +304,12 @@ m4_define([_AT_CHECK_CALC],
 # If BISON-OPTIONS contains `--debug', then NUM-STDERR-LINES is the number
 # of expected lines on stderr.
 m4_define([_AT_CHECK_CALC_ERROR],
-[AT_DATA([[input]],
+[m4_bmatch([$2], [^/],
+           [AT_CHECK([calc $2], 0, [], [stderr])],
+           [AT_DATA([[input]],
 [[$2
 ]])
-
-AT_CHECK([calc input], 0, [], [stderr])
+AT_CHECK([calc input], 0, [], [stderr])])
 
 
 AT_CHECK([wc -l <stderr | sed 's/[[^0-9]]//g'], 0,
@@ -362,27 +371,33 @@ m4_define([AT_CHECK_CALC],
 1 - (2 - 3) = 2
 
 2^2^3 = 256
-(2^2)^3 = 64], [491])
+(2^2)^3 = 64], [488])
 
 # Some parse errors.
-_AT_CHECK_CALC_ERROR([$1], [+1], [8],
-                     [1.0:1.1],
-                     [unexpected '+'])
-_AT_CHECK_CALC_ERROR([$1], [1//2], [17],
+_AT_CHECK_CALC_ERROR([$1], [0 0], [10],
+                     [1.2:1.3],
+                     [unexpected "number"])
+_AT_CHECK_CALC_ERROR([$1], [1//2], [13],
                      [1.2:1.3],
-                     [unexpected '/', expecting NUM or '-' or '('])
-_AT_CHECK_CALC_ERROR([$1], [error], [8],
+                     [unexpected '/', expecting "number" or '-' or '('])
+_AT_CHECK_CALC_ERROR([$1], [error], [4],
                      [1.0:1.1],
-                     [unexpected $undefined.])
-_AT_CHECK_CALC_ERROR([$1], [1 = 2 = 3], [23],
+                     [unexpected $undefined., expecting "number" or '-' or 
'\n' or '('])
+_AT_CHECK_CALC_ERROR([$1], [1 = 2 = 3], [19],
                      [1.6:1.7],
                      [unexpected '='])
 _AT_CHECK_CALC_ERROR([$1],
                      [
 +1],
-                     [16],
+                     [13],
                      [2.0:2.1],
                      [unexpected '+'])
+# Exercise error messages with EOF: work on an empty file.
+_AT_CHECK_CALC_ERROR([$1],
+                     [/dev/null],
+                     [4],
+                     [1.0:1.1],
+                     [unexpected "end of file", expecting "number" or '-' or 
'\n' or '('])
 
 AT_CLEANUP
 ])# AT_CHECK_CALC
Index: tests/regression.at
--- tests/regression.at Wed, 26 Dec 2001 19:05:39 +0100 akim
+++ tests/regression.at Thu, 27 Dec 2001 12:53:20 +0100 akim
@@ -73,7 +73,7 @@ exp: exp OP exp | NUM;
 
 Terminals, with rules where they appear
 
-$ (-1)
+$ (0)
 error (256)
 NUM (257) 2
 OP (258) 1
@@ -177,7 +177,7 @@ exp: exp OP exp | NUM;
 
 Terminals, with rules where they appear
 
-$ (-1)
+$ (0)
 error (256)
 NUM (257) 2
 OP (258) 1
@@ -304,7 +304,7 @@ expr:
 
 Terminals, with rules where they appear
 
-$ (-1)
+$ (0)
 'a' (97) 2
 'b' (98) 2
 'c' (99) 4
@@ -652,7 +652,7 @@ dummy:
     4  15 @1 -> /* empty */
     5  15 CONST_DEC -> @1 undef_id_tok '=' const_id_tok ';'
 Terminals, with rules where they appear
-$ (-1)
+$ (0)
 ';' (59) 5
 '=' (61) 5
 error (256)