miscellaneous cleanups for Bison scanner and parser

[Paul Eggert]

I noticed that the Bison parser defines a token CHARACTER that is
never used, and started cleaning things up.  One thing led to another,
and I ended up simplifying the scanner a bit.  It's now simpler and
easier to read (at least for me), and it generates an object module
that is 10% smaller (on Solaris sparc with GCC 3.2).  I installed the
following patch.

2002-11-07  Paul Eggert  <address@hidden>

        * src/parse-gram.y (CHARACTER): Remove unused token.
        All uses removed.

        * src/scan-gram.l: Remove stack option.  We no longer use the
        stack, since the stack was never deeper than 1; instead, use the
        new auto var c_context to record the stacked value.

        Remove nounput option.  At an unexpected end of file, we now unput
        the minimal input necessary to end cleanly; this simplifies the
        code.

        Avoid unbounded token sizes where this is easy.

        (unexpected_end_of_file): Reword so that beginning of token isn't
        needed.
        (last-string): Now auto, not static.
        (YY_OBS_FREE): Remove unnecessary do while (0) wrapper.
        (scanner_last_string_free): Remove; not used.
        (percent_percent_count): Move decl to just before use.
        (SC_ESCAPED_CHARACTER): Return ID at unexpected end of file,
        not the (never otherwised-used) CHARACTER.

Index: src/parse-gram.y
===================================================================
RCS file: /cvsroot/bison/bison/src/parse-gram.y,v
retrieving revision 1.24
diff -p -u -r1.24 parse-gram.y
--- src/parse-gram.y    19 Oct 2002 14:38:06 -0000      1.24
+++ src/parse-gram.y    8 Nov 2002 05:11:58 -0000
@@ -97,7 +97,6 @@ braced_code_t current_braced_code = acti
 /* Define the tokens together with their human representation.  */
 %token GRAM_EOF 0 "end of file"
 %token STRING     "string"
-%token CHARACTER  "character"
 %token INT        "integer"
 
 %token PERCENT_TOKEN       "%token"
@@ -157,7 +156,7 @@ braced_code_t current_braced_code = acti
 %token BRACED_CODE     "{...}"
 
 
-%type <string> CHARACTER TYPE STRING string_content
+%type <string> TYPE STRING string_content
                BRACED_CODE PROLOGUE EPILOGUE epilogue.opt action
 %type <integer> INT
 %type <symbol> ID symbol string_as_id
@@ -285,7 +284,7 @@ precedence_declarator:
 ;
 
 type.opt:
-  /* Nothing. */ { current_type = NULL;}
+  /* Nothing. */ { current_type = NULL; }
 | TYPE           { current_type = $1; }
 ;
 
@@ -390,7 +389,6 @@ rhs:
 symbol:
   ID              { $$ = $1; }
 | string_as_id    { $$ = $1; }
-| CHARACTER       { $$ = symbol_get ($1, @1); }
 ;
 
 action:
@@ -443,10 +441,6 @@ yyprint (FILE *file,
   fputc (' ', file);
   switch (type)
     {
-    case CHARACTER:
-      fprintf (file, " = '%s'", value->string);
-      break;
-
     case ID:
       fprintf (file, " = %s", value->symbol->tag);
       break;
Index: src/scan-gram.l
===================================================================
RCS file: /cvsroot/bison/bison/src/scan-gram.l,v
retrieving revision 1.35
diff -p -u -r1.35 scan-gram.l
--- src/scan-gram.l     6 Nov 2002 08:08:46 -0000       1.35
+++ src/scan-gram.l     8 Nov 2002 05:11:59 -0000
@@ -19,7 +19,7 @@
    02111-1307  USA
 */
 
-%option debug nodefault noyywrap nounput never-interactive stack
+%option debug nodefault noyywrap never-interactive
 %option prefix="gram_" outfile="lex.yy.c"
 
 %{
@@ -140,7 +140,6 @@ extend_location (location_t *loc, char c
    used, and which is used by YY_OBS_FREE to free the last string.  */
 
 static struct obstack string_obstack;
-char *last_string;
 
 #define YY_OBS_GROW   \
   obstack_grow (&string_obstack, yytext, yyleng)
@@ -151,19 +150,9 @@ char *last_string;
     last_string = obstack_finish (&string_obstack);    \
   } while (0)
 
-#define YY_OBS_FREE                                            \
-  do {                                                         \
-    obstack_free (&string_obstack, last_string);               \
-  } while (0)
-
-void
-scanner_last_string_free (void)
-{
-  YY_OBS_FREE;
-}
-
+#define YY_OBS_FREE \
+  obstack_free (&string_obstack, last_string)
 
-static int percent_percent_count = 0;
 
 /* Within well-formed rules, RULE_LENGTH is the number of values in
    the current rule so far, which says where to find `$0' with respect
@@ -179,6 +168,7 @@ static void handle_at (braced_code_t cod
                       char *cp, location_t location);
 static void handle_syncline (char *args, location_t *location);
 static int convert_ucn_to_byte (char const *hex_text);
+static void unexpected_end_of_file (location_t, char const *);
 
 %}
 %x SC_COMMENT SC_LINE_COMMENT SC_YACC_COMMENT
@@ -202,8 +192,15 @@ splice      (\\[ \f\t\v]*\n)*
 
 %%
 %{
+  /* Nesting level of the current code in braces.  */
   int braces_level IF_LINT (= 0);
 
+  /* Scanner context when scanning C code.  */
+  int c_context IF_LINT (= 0);
+
+  /* A string representing the most recently saved token.  */
+  char *last_string;
+
   /* At each yylex invocation, mark the current position as the
      start of the next token.  */
   YY_STEP;
@@ -263,7 +260,7 @@ splice       (\\[ \f\t\v]*\n)*
   ","                     return COMMA;
   ";"                     return SEMICOLON;
 
-  [ \f\n\t\v]+  YY_STEP;
+  [ \f\n\t\v]  YY_STEP;
 
   {id}        {
     yylval->symbol = symbol_get (yytext, *yylloc);
@@ -285,20 +282,20 @@ splice     (\\[ \f\t\v]*\n)*
   }
 
   /* Characters.  We don't check there is only one.  */
-  "'"         YY_OBS_GROW; yy_push_state (SC_ESCAPED_CHARACTER);
+  "'"         YY_OBS_GROW; BEGIN SC_ESCAPED_CHARACTER;
 
   /* Strings. */
-  "\""        YY_OBS_GROW; yy_push_state (SC_ESCAPED_STRING);
+  "\""        YY_OBS_GROW; BEGIN SC_ESCAPED_STRING;
 
   /* Comments. */
   "/*"        BEGIN SC_YACC_COMMENT;
   "//".*      YY_STEP;
 
   /* Prologue. */
-  "%{"        yy_push_state (SC_PROLOGUE);
+  "%{"        BEGIN SC_PROLOGUE;
 
   /* Code in between braces.  */
-  "{"         YY_OBS_GROW; braces_level = 0; yy_push_state (SC_BRACED_CODE);
+  "{"         YY_OBS_GROW; braces_level = 0; BEGIN SC_BRACED_CODE;
 
   /* A type. */
   "<"{tag}">" {
@@ -308,33 +305,20 @@ splice     (\\[ \f\t\v]*\n)*
     return TYPE;
   }
 
-
-  "%%"   {
+  "%%" {
+    static int percent_percent_count;
     if (++percent_percent_count == 2)
-      yy_push_state (SC_EPILOGUE);
+      BEGIN SC_EPILOGUE;
     return PERCENT_PERCENT;
   }
 
-  .           {
+  . {
     complain_at (*yylloc, _("invalid character: %s"), quote (yytext));
     YY_STEP;
   }
 }
 
 
-  /*-------------------------------------------------------------------.
-  | Whatever the start condition (but those which correspond to        |
-  | entities `swallowed' by Bison: SC_YACC_COMMENT, SC_ESCAPED_STRING, |
-  | and SC_ESCAPED_CHARACTER), no M4 character must escape as is.      |
-  `-------------------------------------------------------------------*/
-
-<SC_COMMENT,SC_LINE_COMMENT,SC_STRING,SC_CHARACTER,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
-{
-  \[   obstack_sgrow (&string_obstack, "@<:@");
-  \]   obstack_sgrow (&string_obstack, "@:>@");
-}
-
-
   /*---------------------------------------------------------------.
   | Scanning a Yacc comment.  The initial `/ *' is already eaten.  |
   `---------------------------------------------------------------*/
@@ -346,12 +330,8 @@ splice      (\\[ \f\t\v]*\n)*
     BEGIN INITIAL;
   }
 
-  [^*]+|"*"  ;
-
-  <<EOF>> {
-    complain_at (*yylloc, _("unexpected end of file in a comment"));
-    BEGIN INITIAL;
-  }
+  .|\n    ;
+  <<EOF>>  unexpected_end_of_file (*yylloc, "*/");
 }
 
 
@@ -361,13 +341,8 @@ splice      (\\[ \f\t\v]*\n)*
 
 <SC_COMMENT>
 {
-  "*"{splice}"/"  YY_OBS_GROW; yy_pop_state ();
-  [^*\[\]]+|"*"   YY_OBS_GROW;
-
-  <<EOF>> {
-    complain_at (*yylloc, _("unexpected end of file in a comment"));
-    yy_pop_state ();
-  }
+  "*"{splice}"/"  YY_OBS_GROW; BEGIN c_context;
+  <<EOF>>        unexpected_end_of_file (*yylloc, "*/");
 }
 
 
@@ -377,9 +352,9 @@ splice       (\\[ \f\t\v]*\n)*
 
 <SC_LINE_COMMENT>
 {
-  "\n"                  YY_OBS_GROW; yy_pop_state ();
-  ([^\n\[\]]|{splice})+  YY_OBS_GROW;
-  <<EOF>>               yy_pop_state ();
+  "\n"          YY_OBS_GROW; BEGIN c_context;
+  {splice}      YY_OBS_GROW;
+  <<EOF>>       BEGIN c_context;
 }
 
 
@@ -391,25 +366,16 @@ splice     (\\[ \f\t\v]*\n)*
 <SC_ESCAPED_STRING>
 {
   "\"" {
-    assert (yy_top_state () == INITIAL);
     YY_OBS_GROW;
     YY_OBS_FINISH;
     yylval->string = last_string;
-    yy_pop_state ();
     rule_length++;
+    BEGIN INITIAL;
     return STRING;
   }
 
-  [^\"\\]+  YY_OBS_GROW;
-
-  <<EOF>> {
-    complain_at (*yylloc, _("unexpected end of file in a string"));
-    assert (yy_top_state () == INITIAL);
-    YY_OBS_FINISH;
-    yylval->string = last_string;
-    yy_pop_state ();
-    return STRING;
-  }
+  .|\n     YY_OBS_GROW;
+  <<EOF>>   unexpected_end_of_file (*yylloc, "\"");
 }
 
   /*---------------------------------------------------------------.
@@ -421,30 +387,19 @@ splice     (\\[ \f\t\v]*\n)*
 {
   "'" {
     YY_OBS_GROW;
-    assert (yy_top_state () == INITIAL);
-    {
-      YY_OBS_FINISH;
-      yylval->symbol = symbol_get (last_string, *yylloc);
-      symbol_class_set (yylval->symbol, token_sym, *yylloc);
-      symbol_user_token_number_set (yylval->symbol,
-                                   (unsigned char) last_string[1], *yylloc);
-      YY_OBS_FREE;
-      yy_pop_state ();
-      rule_length++;
-      return ID;
-    }
-  }
-
-  [^\'\\]+  YY_OBS_GROW;
-
-  <<EOF>> {
-    complain_at (*yylloc, _("unexpected end of file in a character literal"));
-    assert (yy_top_state () == INITIAL);
     YY_OBS_FINISH;
-    yylval->string = last_string;
-    yy_pop_state ();
-    return CHARACTER;
+    yylval->symbol = symbol_get (last_string, *yylloc);
+    symbol_class_set (yylval->symbol, token_sym, *yylloc);
+    symbol_user_token_number_set (yylval->symbol,
+                                 (unsigned char) last_string[1], *yylloc);
+    YY_OBS_FREE;
+    rule_length++;
+    BEGIN INITIAL;
+    return ID;
   }
+
+  .|\n     YY_OBS_GROW;
+  <<EOF>>   unexpected_end_of_file (*yylloc, "'");
 }
 
 
@@ -504,8 +459,6 @@ splice       (\\[ \f\t\v]*\n)*
                 quote (yytext));
     YY_OBS_GROW;
   }
-  /* FLex wants this rule, in case of a `\<<EOF>>'. */
-  \\                   YY_OBS_GROW;
 }
 
 
@@ -516,23 +469,10 @@ splice     (\\[ \f\t\v]*\n)*
 
 <SC_CHARACTER>
 {
-  "'" {
-    YY_OBS_GROW;
-    assert (yy_top_state () != INITIAL);
-    yy_pop_state ();
-  }
-
-  [^'\[\]\\]+         YY_OBS_GROW;
+  "'"                 YY_OBS_GROW; BEGIN c_context;
   \\{splice}[^\[\]]    YY_OBS_GROW;
   {splice}            YY_OBS_GROW;
-  /* Needed for `\<<EOF>>', `\\<<newline>>[', and `\\<<newline>>]'.  */
-  \\                   YY_OBS_GROW;
-
-  <<EOF>> {
-    complain_at (*yylloc, _("unexpected end of file in a character literal"));
-    assert (yy_top_state () != INITIAL);
-    yy_pop_state ();
-  }
+  <<EOF>>             unexpected_end_of_file (*yylloc, "'");
 }
 
 
@@ -543,23 +483,10 @@ splice     (\\[ \f\t\v]*\n)*
 
 <SC_STRING>
 {
-  "\"" {
-    assert (yy_top_state () != INITIAL);
-    YY_OBS_GROW;
-    yy_pop_state ();
-  }
-
-  [^\"\[\]\\]+        YY_OBS_GROW;
+  "\""                YY_OBS_GROW; BEGIN c_context;
   \\{splice}[^\[\]]    YY_OBS_GROW;
   {splice}            YY_OBS_GROW;
-  /* Needed for `\<<EOF>>', `\\<<newline>>[', and `\\<<newline>>]'.  */
-  \\                   YY_OBS_GROW;
-
-  <<EOF>> {
-    complain_at (*yylloc, _("unexpected end of file in a string"));
-    assert (yy_top_state () != INITIAL);
-    yy_pop_state ();
-  }
+  <<EOF>>             unexpected_end_of_file (*yylloc, "\"");
 }
 
 
@@ -569,18 +496,10 @@ splice     (\\[ \f\t\v]*\n)*
 
 <SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
 {
-  /* Characters.  We don't check there is only one.  */
-  "'"         YY_OBS_GROW; yy_push_state (SC_CHARACTER);
-
-  /* Strings. */
-  "\""        YY_OBS_GROW; yy_push_state (SC_STRING);
-
-  /* Comments. */
-  "/"{splice}"*"  YY_OBS_GROW; yy_push_state (SC_COMMENT);
-  "/"{splice}"/"  YY_OBS_GROW; yy_push_state (SC_LINE_COMMENT);
-
-  /* Not comments. */
-  "/"         YY_OBS_GROW;
+  "'"            YY_OBS_GROW; c_context = YY_START; BEGIN SC_CHARACTER;
+  "\""           YY_OBS_GROW; c_context = YY_START; BEGIN SC_STRING;
+  "/"{splice}"*"  YY_OBS_GROW; c_context = YY_START; BEGIN SC_COMMENT;
+  "/"{splice}"/"  YY_OBS_GROW; c_context = YY_START; BEGIN SC_LINE_COMMENT;
 }
 
 
@@ -598,31 +517,24 @@ splice     (\\[ \f\t\v]*\n)*
     braces_level--;
     if (braces_level < 0)
       {
-       yy_pop_state ();
        YY_OBS_FINISH;
        yylval->string = last_string;
        rule_length++;
+       BEGIN INITIAL;
        return BRACED_CODE;
       }
   }
 
+  /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly
+     (as `<' `<%').  */
+  "<"{splice}"<"  YY_OBS_GROW;
+
   "$"("<"{tag}">")?(-?[0-9]+|"$") { handle_dollar (current_braced_code,
                                                   yytext, *yylloc); }
   "@"(-?[0-9]+|"$")               { handle_at (current_braced_code,
                                               yytext, *yylloc); }
 
-  /* `"<"{splice}"<"' tokenizes `<<%' correctly (as `<<' `%') rather
-     than incorrrectly (as `<' `<%').  */
-  [^\"$%\'/<@\[\]\{\}]+|[$%/<@]|"<"{splice}"<"  YY_OBS_GROW;
-
-  <<EOF>> {
-    complain_at (*yylloc, _("unexpected end of file in `{ ... }'"));
-    yy_pop_state ();
-    YY_OBS_FINISH;
-    yylval->string = last_string;
-    return BRACED_CODE;
-  }
-
+  <<EOF>>  unexpected_end_of_file (*yylloc, "}");
 }
 
 
@@ -633,22 +545,13 @@ splice     (\\[ \f\t\v]*\n)*
 <SC_PROLOGUE>
 {
   "%}" {
-    yy_pop_state ();
     YY_OBS_FINISH;
     yylval->string = last_string;
+    BEGIN INITIAL;
     return PROLOGUE;
   }
 
-  [^%\[\]/\'\"]+     YY_OBS_GROW;
-  "%"                YY_OBS_GROW;
-
-  <<EOF>> {
-    complain_at (*yylloc, _("unexpected end of file in `%%{ ... %%}'"));
-    yy_pop_state ();
-    YY_OBS_FINISH;
-    yylval->string = last_string;
-    return PROLOGUE;
-  }
+  <<EOF>>  unexpected_end_of_file (*yylloc, "%}");
 }
 
 
@@ -659,17 +562,28 @@ splice     (\\[ \f\t\v]*\n)*
 
 <SC_EPILOGUE>
 {
-  [^\[\]]+  YY_OBS_GROW;
-
   <<EOF>> {
-    yy_pop_state ();
     YY_OBS_FINISH;
     yylval->string = last_string;
+    BEGIN INITIAL;
     return EPILOGUE;
   }
 }
 
 
+  /*----------------------------------------------------------------.
+  | By default, grow the string obstack with the input, escaping M4 |
+  | quoting characters.                                                    |
+  `----------------------------------------------------------------*/
+
+<SC_COMMENT,SC_LINE_COMMENT,SC_STRING,SC_CHARACTER,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
+{
+  \[   obstack_sgrow (&string_obstack, "@<:@");
+  \]   obstack_sgrow (&string_obstack, "@:>@");
+  .|\n  YY_OBS_GROW;
+}
+
+
 %%
 
 /*------------------------------------------------------------------.
@@ -871,7 +785,7 @@ convert_ucn_to_byte (char const *ucn)
     /* A non-ASCII host.  Use CODE to index into a table of the C
        basic execution character set, which is guaranteed to exist on
        all Standard C platforms.  This table also includes '$', '@',
-       and '`', which not in the basic execution character set but
+       and '`', which are not in the basic execution character set but
        which are unibyte characters on all the platforms that we know
        about.  */
     static signed char const table[] =
@@ -919,6 +833,24 @@ handle_syncline (char *args, location_t 
   location->file = infile;
   location->last_line = lineno;
 }
+
+
+/*-------------------------------------------------------------.
+| Report an unexpected end of file at LOC.  An end of file was |
+| encountered and the expected TOKEN_END was missing.  After   |
+| reporting the problem, pretend that TOKEN_END was found.     |
+`-------------------------------------------------------------*/
+
+static void
+unexpected_end_of_file (location_t loc, char const *token_end)
+{
+  size_t i;
+
+  complain_at (loc, _("missing `%s' at end of file"), token_end);
+  for (i = strlen (token_end); i != 0; i--)
+    unput (token_end[i - 1]);
+}
+
 
 /*-------------------------.
 | Initialize the scanner.  |