[PATCH 1/6] api.value.type: implement proper support, check, and document

[Akim Demaille]

* data/c.m4 (b4_symbol_type_register, b4_type_define_tag)
(b4_symbol_value_union, b4_value_type_setup_union)
(b4_value_type_setup_variant, b4_value_type_setup):
New.
(b4_value_type_define): Use it to set up properly the type.
Handle the various possible values of api.value.type.
* data/c++.m4 (b4_value_type_declare): Likewise.
* data/lalr1.cc (b4_value_type_setup_variant): Redefine.

* tests/types.at: New.
Exercise all the C/C++ skeletons with different types of
api.value.type values.
* tests/local.mk, tests/testsuite.at: Use it.

* doc/bison.texi (%define Summary): Document api.value.type.
* NEWS: Advertise it, together with api.token.constructor.
---
 NEWS               |  68 ++++++++++++++++++++++++++
 data/bison.m4      |   5 ++
 data/c++.m4        |  15 +++---
 data/c.m4          | 121 +++++++++++++++++++++++++++++++++++++++++----
 data/lalr1.cc      |   2 +
 doc/bison.texi     |  82 +++++++++++++++++++++++++++++--
 tests/local.mk     |   3 +-
 tests/testsuite.at |   7 ++-
 tests/types.at     | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 9 files changed, 421 insertions(+), 22 deletions(-)
 create mode 100644 tests/types.at

diff --git a/NEWS b/NEWS
index a48ef71..0aabc13 100644
--- a/NEWS
+++ b/NEWS
@@ -252,6 +252,74 @@ GNU Bison NEWS
   use these prefixed token names, although the grammar itself still
   uses the short names (as in the sample rule given above).
 
+** Variable api.value.type
+
+  This new %define variable supersedes the #define macro YYSTYPE.  The use
+  of YYSTYPE is discouraged.  In particular, #defining YYTSYPE *and* using
+  %union or %defining api.value.type results in undefined behavior.
+
+  The %define variable api.value.type supports several special values,
+  examplified below:
+
+  The value "%union" denotes that fact that %union is used.
+
+    %define api.value.type "%union" // implied by the next line
+    %union
+    {
+      int ival;
+      char *sval;
+    }
+    %token <ival> INT "integer"
+    %token <sval> STR "string"
+
+    /* In yylex().  */
+    yylval.ival = 42; return INT;
+    yylval.sval = "42"; return STR;
+
+  The value "union" means that the user provides genuine types, not
+  union members names such as "ival" and "sval" above.
+
+    %define api.value.type "union"
+    %token <int> INT "integer"
+    %token <char *> STR "string"
+
+    /* In yylex().  */
+    yylval.yytype_INT = 42; return INT;
+    yylval.yytype_STR = "42"; return STR;
+
+  The value "variant" is somewhat equivalent, but for C++ special provision
+  is made to allow classes to be used.
+
+    %define api.value.type "variant"
+    %token <int> INT "integer"
+    %token <std::string> STR "string"
+
+  Any other name is a user type to use.  This is where YYSTYPE used to be
+  used.
+
+    %code requires
+    {
+      struct my_value
+      {
+        enum
+        {
+          is_int, is_str
+        } kind;
+        union
+        {
+          int ival;
+          char *sval;
+        } u;
+      };
+    }
+    %define api.value.type "struct my_value"
+    %token <u.ival> INT "integer"
+    %token <u.sval> STR "string"
+
+    /* In yylex().  */
+    yylval.u.ival = 42; return INT;
+    yylval.u.sval = "42"; return STR;
+
 ** Variable parse.error
 
   This variable controls the verbosity of error messages.  The use of the
diff --git a/data/bison.m4 b/data/bison.m4
index 9c5d48f..4aa6448 100644
--- a/data/bison.m4
+++ b/data/bison.m4
@@ -357,9 +357,14 @@ b4_define_flag_if([yacc])               # Whether POSIX 
Yacc is emulated.
 #   The internalized number (used after yytranslate).
 # - has_type: 0, 1
 #   Whether has a semantic value.
+# - type_tag: string
+#   When api.value.type=union, the generated name for the union member.
+#   yytype_INT etc. for %tokens, otherwise yytype_1 etc.
 # - type
 #   If it has a semantic value, its type tag, or, if variant are used,
 #   its type.
+#   In the case of api.value.type=union, type is first defined
+#   as the real type (e.g. int), and then changed to be equal to type_tag.
 # - has_printer: 0, 1
 # - printer: string
 # - printer_file: string
diff --git a/data/c++.m4 b/data/c++.m4
index e0fc535..9c12e5c 100644
--- a/data/c++.m4
+++ b/data/c++.m4
@@ -118,15 +118,16 @@ m4_define([b4_token_enums],
 # ---------------------
 # Declare semantic_type.
 m4_define([b4_value_type_declare],
+[b4_value_type_setup[]dnl
 [    /// Symbol semantic values.
-m4_ifdef([b4_union_members],
-[    union semantic_type
+]m4_bmatch(b4_percent_define_get([api.value.type]),
+[^%union\|union$],
+[[    union semantic_type
     {
-b4_user_union_members
-    };],
-[m4_if(b4_tag_seen_flag, 0,
-[[    typedef int semantic_type;]],
-[[    typedef ]b4_api_PREFIX[STYPE semantic_type;]])])])
+]b4_user_union_members[
+    };]],
+[^$], [],
+[[    typedef ]b4_percent_define_get([api.value.type])[ semantic_type;]])])
 
 
 # b4_public_types_declare
diff --git a/data/c.m4 b/data/c.m4
index 9405ae3..35741b3 100644
--- a/data/c.m4
+++ b/data/c.m4
@@ -492,28 +492,131 @@ b4_locations_if([, yylocationp])[]b4_user_args[);
 }]dnl
 ])
 
+
+## ---------------- ##
+## api.value.type.  ##
+## ---------------- ##
+
+
+# ---------------------- #
+# api.value.type=union.  #
+# ---------------------- #
+
+# b4_symbol_type_register(SYMBOL-NUM)
+# -----------------------------------
+# Symbol SYMBOL-NUM has a type (for variant) instead of a type-tag.
+# Extend the definition of %union's body with a field of that type,
+# and extend the symbol's "type" field to point to the field name,
+# instead of the type name.
+m4_define([b4_symbol_type_register],
+[m4_define([b4_symbol($1, type_tag)],
+           [yytype_[]b4_symbol_if([$1], [is_token],
+                                  [b4_symbol_if([$1], [has_id],
+                                                [b4_symbol([$1], [id])],
+                                                [b4_symbol([$1], [number])])],
+                                  [b4_symbol([$1], [number])])])dnl
+m4_append([b4_user_union_members],
+m4_expand([
+  b4_symbol_tag_comment([$1])dnl
+  b4_symbol([$1], [type]) b4_symbol([$1], [type_tag]);]))
+m4_define([b4_symbol($1, type)],
+          [b4_symbol([$1], [type_tag])])dnl
+])
+
+
+# b4_type_define_tag(SYMBOL1-NUM, ...)
+# ------------------------------------
+# For the batch of symbols SYMBOL1-NUM... (which all have the same
+# type), enhance the %union definition for each of them, and set
+# there "type" field to the field tag name, instead of the type name.
+m4_define([b4_type_define_tag],
+[b4_symbol_if([$1], [has_type],
+              [m4_map([b4_symbol_type_register], address@hidden)])
+])
+
+
+# b4_symbol_value_union(VAL, [TYPE])
+# ----------------------------------
+# Same of b4_symbol_value, but when api.value.type=union.
+m4_define([b4_symbol_value_union],
+[m4_ifval([$2],
+          [(*($2*)(&$1))],
+          [$1])])
+])
+
+
+# b4_value_type_setup_union
+# -------------------------
+# Setup support for api.value.type=union.  Symbols are defined with a
+# type instead of a union member name: build the corresponding union,
+# and give the symbols their tag.
+m4_define([b4_value_type_setup_union],
+[m4_define([b4_union_members])
+b4_type_foreach([b4_type_define_tag])
+m4_copy_force([b4_symbol_value_union], [b4_symbol_value])
+])
+
+
+# ---------------- #
+# api.value.type.  #
+# ---------------- #
+
+
+# b4_value_type_setup_variant
+# ---------------------------
+# Setup support for api.value.type=variant.  By default, fail, specialized
+# by other skeletons.
+m4_define([b4_value_type_setup_variant],
+[b4_complain_at(b4_percent_define_get_loc([api.value.type]),
+                [['%s' does not support '%s']],
+                [b4_skeleton],
+                [%define api.value.type variant])])
+
+
+# b4_value_type_setup
+# -------------------
+# Check if api.value.type is properly defined, and possibly prepare
+# its use.
+m4_define([b4_value_type_setup],
+[b4_percent_define_default([[api.value.type]],
+[m4_ifdef([b4_union_members], [%union],
+          [m4_if(b4_tag_seen_flag, 0, [int],
+                 [])])])dnl
+m4_case(b4_percent_define_get([api.value.type]),
+   [union],   [b4_value_type_setup_union],
+   [variant], [b4_value_type_setup_variant])])
+
+
+
 ## -------------- ##
 ## Declarations.  ##
 ## -------------- ##
 
+
 # b4_value_type_define
 # --------------------
 m4_define([b4_value_type_define],
-[[/* Value type.  */
-#if ! defined ]b4_api_PREFIX[STYPE && ! defined 
]b4_api_PREFIX[STYPE_IS_DECLARED
-]m4_ifdef([b4_union_members],
-[[typedef union ]b4_union_name[ ]b4_api_PREFIX[STYPE;
+[b4_value_type_setup[]dnl
+/* Value type.  */
+m4_bmatch(b4_percent_define_get([api.value.type]),
+[^%?union$],
+[[#if ! defined ]b4_api_PREFIX[STYPE && ! defined 
]b4_api_PREFIX[STYPE_IS_DECLARED
+typedef union ]b4_union_name[ ]b4_api_PREFIX[STYPE;
 union ]b4_union_name[
 {
 ]b4_user_union_members[
 };
-# define ]b4_api_PREFIX[STYPE_IS_TRIVIAL 1]],
-[m4_if(b4_tag_seen_flag, 0,
-[[typedef int ]b4_api_PREFIX[STYPE;
-# define ]b4_api_PREFIX[STYPE_IS_TRIVIAL 1]])])[
+# define ]b4_api_PREFIX[STYPE_IS_TRIVIAL 1
 # define ]b4_api_PREFIX[STYPE_IS_DECLARED 1
 #endif
-]])
+]],
+[^$], [],
+[[#if ! defined ]b4_api_PREFIX[STYPE && ! defined 
]b4_api_PREFIX[STYPE_IS_DECLARED
+typedef ]b4_percent_define_get([api.value.type])[ ]b4_api_PREFIX[STYPE;
+# define ]b4_api_PREFIX[STYPE_IS_TRIVIAL 1
+# define ]b4_api_PREFIX[STYPE_IS_DECLARED 1
+#endif
+]])])
 
 
 # b4_location_type_define
diff --git a/data/lalr1.cc b/data/lalr1.cc
index 53e7455..1d864b5 100644
--- a/data/lalr1.cc
+++ b/data/lalr1.cc
@@ -17,6 +17,8 @@
 
 m4_include(b4_pkgdatadir/[c++.m4])
 
+# api.value.type=variant is valid.
+m4_define([b4_value_type_setup_variant])
 
 # b4_integral_parser_table_declare(TABLE-NAME, CONTENT, COMMENT)
 # --------------------------------------------------------------
diff --git a/doc/bison.texi b/doc/bison.texi
index 5b1b0de..873c782 100644
--- a/doc/bison.texi
+++ b/doc/bison.texi
@@ -5586,6 +5586,7 @@ Summary,,%skeleton}).
 Unaccepted @var{variable}s produce an error.
 Some of the accepted @var{variable}s are described below.
 
address@hidden ================================================== api.namespace
 @deffn Directive {%define api.namespace} "@var{namespace}"
 @itemize
 @item Languages(s): C++
@@ -5812,14 +5813,89 @@ introduced in Bison 2.8
 @deffn Directive {%define api.value.type} @var{type}
 @itemize @bullet
 @item Language(s):
-C++
+all
 
 @item Purpose:
-Request variant-based semantic values.
+The type for semantic values.
+
address@hidden Accepted Values:
address@hidden @asis
address@hidden @code{""}
+This grammar has no semantic value at all.  This is not properly supported
+yet.
address@hidden @code{%union} (C, C++)
+The type is defined thanks to the @code{%union} directive.  You don't have
+to define @code{api.value.type} in that case, using @code{%union} suffices.
address@hidden Decl, ,The Collection of Value Types}.
+For instance:
address@hidden
+%define api.value.type "%union"
+%union
address@hidden
+  int ival;
+  char *sval;
address@hidden
+%token <ival> INT "integer"
+%token <sval> STR "string"
address@hidden example
+
address@hidden @code{union} (C, C++)
+The symbols are defined with type names, from which Bison will generate a
address@hidden  For instance:
address@hidden
+%define api.value.type "union"
+%token <int> INT "integer"
+%token <char *> STR "string"
address@hidden example
+This feature needs user feedback to stabilize.  Note that most C++ objects
+cannot be stored in a @code{union}.
+
address@hidden @code{variant} (C++)
+This is similar to @code{union}, but special storage techniques are used to
+allow any kind of C++ object to be used. For instance:
address@hidden
+%define api.value.type "variant"
+%token <int> INT "integer"
+%token <std::string> STR "string"
address@hidden example
+This feature needs user feedback to stabilize.
 @xref{C++ Variants}.
 
address@hidden any other identifier
+Use this name as semantic value.
address@hidden
+%code requires
address@hidden
+  struct my_value
+  @{
+    enum
+    @{
+      is_int, is_str
+    @} kind;
+    union
+    @{
+      int ival;
+      char *sval;
+    @} u;
+  @};
address@hidden
+%define api.value.type "struct my_value"
+%token <u.ival> INT "integer"
+%token <u.sval> STR "string"
address@hidden example
address@hidden table
+
 @item Default Value:
-FIXME:
address@hidden @minus
address@hidden
address@hidden if @code{%union} is used, otherwise @dots{}
address@hidden
address@hidden if type tags are used (i.e., @samp{%token <@var{type}>@dots{}} or
address@hidden <@var{type}>@dots{}} is used), otherwise @dots{}
address@hidden
address@hidden""}
address@hidden itemize
+
 @item History:
 introduced in Bison 2.8.  Was introduced for Java only in 2.3b as
 @code{stype}.
diff --git a/tests/local.mk b/tests/local.mk
index 7bc8b78..5299c2e 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -63,7 +63,8 @@ TESTSUITE_AT =                                  \
   tests/sets.at                                 \
   tests/skeletons.at                            \
   tests/synclines.at                            \
-  tests/torture.at
+  tests/torture.at                              \
+  tests/types.at
 
 TESTSUITE = $(top_srcdir)/tests/testsuite
 
diff --git a/tests/testsuite.at b/tests/testsuite.at
index f11866b..4c99513 100644
--- a/tests/testsuite.at
+++ b/tests/testsuite.at
@@ -35,6 +35,9 @@ m4_include([sets.at])
 # Testing grammar reduction.
 m4_include([reduce.at])
 
+# Testing conflicts detection and resolution.
+m4_include([conflicts.at])
+
 # Testing that #lines are correct.
 m4_include([synclines.at])
 
@@ -44,8 +47,8 @@ m4_include([headers.at])
 # Testing that user actions are properly performed.
 m4_include([actions.at])
 
-# Testing conflicts detection and resolution.
-m4_include([conflicts.at])
+# Testing semantic types support.
+m4_include([types.at])
 
 # Fulling testing (compilation and execution of the parser) on calc.
 m4_include([calc.at])
diff --git a/tests/types.at b/tests/types.at
new file mode 100644
index 0000000..bf9e111
--- /dev/null
+++ b/tests/types.at
@@ -0,0 +1,140 @@
+# Value type.                                     -*- Autotest -*-
+
+# Copyright (C) 2013 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+AT_BANNER([[Value type tests.]])
+
+
+# AT_TEST($1: BISON-DIRECTIVES,
+#         $2: MORE-BISON-DIRECTIVES
+#         $3: PARSER-ACTION,
+#         $4: INPUT, $5: SCANNER-ACTION,
+#         $6: RESULT)
+# --------------------------------------
+# Compile the grammar and check the expected result.
+# BISON-DIRECTIVES are passed to AT_SETUP, contrary to MORE-BISON-DIRECTIVES.
+m4_pushdef([AT_TEST],
+[
+AT_SETUP([$1])
+AT_KEYWORDS([api.value.type])
+AT_BISON_OPTION_PUSHDEFS([$1 $2])
+AT_DATA_GRAMMAR([test.y],
+[[%debug
+
+%code
+{
+# include <stdio.h>
+# include <stdlib.h>
+]AT_YYERROR_DECLARE[
+]AT_YYLEX_DECLARE[
+}
+
+]$1[
+]$2[
+
+%%
+
+start: $3;
+
+%%
+]AT_YYERROR_DEFINE[
+]AT_YYLEX_DEFINE([$4], [$5])[
+]AT_MAIN_DEFINE[
+]])
+
+AT_FULL_COMPILE([[test]])
+AT_PARSER_CHECK([./test], 0, [$6
+], [stderr])
+AT_BISON_OPTION_POPDEFS
+AT_CLEANUP
+])
+
+m4_foreach([b4_skel], [[yacc.c], [glr.c], [lalr1.cc], [glr.cc]],
+ [# A built-in type.
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type double],
+          [],
+          ['1' '2' { printf ("%2.1f\n", $1 + $2); }],
+          ["12"],
+          [AT_VAL = (res - '0') / 10.0],
+          [0.3])
+
+  # A user defined struct.
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type "struct foo"],
+          [%code requires { struct foo { float fval; int ival; }; }],
+          ['1' '2'
+             { printf ("%d %2.1f\n", $1.ival + $2.ival, $1.fval + $2.fval); }],
+          ["12"],
+          [AT_VAL.ival = (res - '0') * 10;
+           AT_VAL.fval = (res - '0') / 10.f],
+          [30 0.3])
+
+  # A user defined union.
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type "union foo"],
+          [%code requires { union foo { float fval; int ival; }; }],
+          ['1' '2' { printf ("%d %2.1f\n", $1.ival, $2.fval); }],
+          ["12"],
+          [if (res == '1')
+             AT_VAL.ival = 10;
+           else
+             AT_VAL.fval = .2f],
+          [10 0.2])
+
+  # A %union.
+  AT_TEST([%skeleton "]b4_skel["
+           %union { float fval; int ival; };],
+          [%token <ival> '1';
+           %token <fval> '2';],
+          ['1' '2' { printf ("%d %2.1f\n", $1, $2); }],
+          ["12"],
+          [if (res == '1')
+             AT_VAL.ival = 10;
+           else
+             AT_VAL.fval = 0.2f],
+          [10 0.2])
+
+
+  # A Bison-defined union.
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type union],
+          [%token <int> ONE 101;
+           %token <float> TWO 102;],
+          [ONE TWO { printf ("%d %2.1f\n", $1, $2); }],
+          [{ 101, 102, EOF }],
+          [if (res == 101)
+             AT_VAL.yytype_ONE = 10;
+           else
+             AT_VAL.yytype_TWO = .2f],
+          [10 0.2])
+
+  # A Bison-defined variant, for lalr1.cc only.
+  m4_if(b4_skel, [lalr1.cc], [
+  AT_TEST([%skeleton "]b4_skel["
+           %define api.value.type variant],
+          [%token <int> '1';
+           %token <std::string> '2';],
+          ['1' '2' { std::cout << $1 << ", " << $2 << std::endl; }],
+          ["12"],
+          [if (res == '1')
+             AT_VAL.build(10);
+           else
+             AT_VAL.build<std::string>("two");],
+          [10, two])])
+])
+
+m4_popdef([AT_TEST])
-- 
1.8.1.3