[6217] parsetexi parse_texi_line

[Gavin D. Smith]

Revision: 6217
          http://svn.sv.gnu.org/viewvc/?view=rev&root=texinfo&revision=6217
Author:   gavin
Date:     2015-04-10 11:27:51 +0000 (Fri, 10 Apr 2015)
Log Message:
-----------
parsetexi parse_texi_line

Modified Paths:
--------------
    trunk/parsetexi/ChangeLog
    trunk/parsetexi/Parsetexi/Parsetexi.xs
    trunk/parsetexi/Parsetexi/lib/Parsetexi.pm
    trunk/parsetexi/TODO
    trunk/parsetexi/api.c
    trunk/parsetexi/commands.h
    trunk/parsetexi/macro.c
    trunk/parsetexi/parser.c
    trunk/parsetexi/parser.h

Modified: trunk/parsetexi/ChangeLog
===================================================================
--- trunk/parsetexi/ChangeLog   2015-04-09 19:40:48 UTC (rev 6216)
+++ trunk/parsetexi/ChangeLog   2015-04-10 11:27:51 UTC (rev 6217)
@@ -1,3 +1,13 @@
+2015-04-10  Gavin Smith  <address@hidden>
+
+       * Parsetexi/lib/Parsetexi.pm (parser): Add code to receive 
+       "values" configuration value.
+       (parse_texi_line): First implementation.
+       * Parsetexi/Parsetexi.xs (parse_string, store_value)
+       (wipe_values): Expose.
+       * api.c (parse_string): New function.
+       * macro.c (wipe_values): New function.
+
 2015-04-09  Gavin Smith  <address@hidden>
 
        * end_line.c (parse_line_command_args) <CM_synindex>: Record

Modified: trunk/parsetexi/Parsetexi/Parsetexi.xs
===================================================================
--- trunk/parsetexi/Parsetexi/Parsetexi.xs      2015-04-09 19:40:48 UTC (rev 
6216)
+++ trunk/parsetexi/Parsetexi/Parsetexi.xs      2015-04-10 11:27:51 UTC (rev 
6217)
@@ -1,3 +1,5 @@
+#define context perl_context
+
 #define PERL_NO_GET_CONTEXT
 #include "EXTERN.h"
 #include "perl.h"
@@ -3,11 +5,15 @@
 #include "XSUB.h"
 
+#undef context
+
 #include "ppport.h"
 
-#include "../tree_types.h"
+#include "../parser.h"
+
 #include "../tree.h"
 #include "../api.h"
 #include "../errors.h"
 
+
 HV *build_texinfo_tree (void);
 HV *build_label_list (void);
@@ -45,6 +51,18 @@
 parse_file(filename)
         char * filename
 
+void
+parse_string(string)
+        char * string
+
+void
+store_value (name, value)
+        char *name
+        char *value
+
+void
+wipe_values ()
+
 ELEMENT *
 get_root()
 

Modified: trunk/parsetexi/Parsetexi/lib/Parsetexi.pm
===================================================================
--- trunk/parsetexi/Parsetexi/lib/Parsetexi.pm  2015-04-09 19:40:48 UTC (rev 
6216)
+++ trunk/parsetexi/Parsetexi/lib/Parsetexi.pm  2015-04-10 11:27:51 UTC (rev 
6217)
@@ -117,6 +117,7 @@
   $parser->{'gettext'} = $parser_default_configuration{'gettext'};
   $parser->{'pgettext'} = $parser_default_configuration{'pgettext'};
 
+  wipe_values ();
   if (defined($conf)) {
     foreach my $key (keys (%$conf)) {
       if ($key eq 'include_directories') {
@@ -125,9 +126,25 @@
           warn "got dir $d\n";
           add_include_directory ($d);
         }
-
+      } elsif ($key eq 'values') {
+       # This is used by Texinfo::Structuring::gdt for substituted values
+       for my $v (keys %{$conf->{'values'}}) {
+         if (ref($conf->{'values'}->{$v}) eq 'HASH') {
+           if (defined ($conf->{'values'}->{$v}->{'text'})) {
+             store_value ($v, $conf->{'values'}->{$v}->{'text'});
+           } else {
+             store_value ($v, "<<HASH WITH NO TEXT>>");
+           }
+         } elsif (ref($conf->{'values'}->{$v}) eq 'SCALAR') {
+           store_value ($v, $conf->{'values'}->{$v});
+         } elsif (ref($conf->{'values'}->{$v}) eq 'ARRAY') {
+           store_value ($v, "<<ARRAY VALUE>>");
+         } else {
+           store_value ($v, "<<UNKNOWN VALUE>>");
+         }
+       }
       } else {
-        warn "ignoring parser configuration value \"$key\"\n";
+       #warn "ignoring parser configuration value \"$key\"\n";
       }
     }
   }
@@ -226,7 +243,7 @@
   }
 }
 
-# Stub for Texinfo::Parser::parse_texi_file (line 835)
+# Replacement for Texinfo::Parser::parse_texi_file (line 835)
 sub parse_texi_file ($$)
 {
   my $self = shift;
@@ -327,6 +344,25 @@
   return $TREE;
 }
 
+# Replacement for Texinfo::Parser::parse_texi_line (line 918)
+sub parse_texi_line($$;$$$$)
+{
+    my $self = shift;
+    my $text = shift;
+    my $lines_nr = shift;
+    my $file = shift;
+    my $macro = shift;
+    my $fixed_line_number = shift;
+
+    return undef if (!defined($text));
+
+    $self = parser() if (!defined($self));
+    parse_string($text);
+    my $tree = build_texinfo_tree ();
+    _add_parents ($tree);
+    return $tree;
+}
+
 # Public interfaces of Texinfo::Parser (starting line 942)
 sub indices_information($)
 {

Modified: trunk/parsetexi/TODO
===================================================================
--- trunk/parsetexi/TODO        2015-04-09 19:40:48 UTC (rev 6216)
+++ trunk/parsetexi/TODO        2015-04-10 11:27:51 UTC (rev 6217)
@@ -3,8 +3,7 @@
 * Check everywhere we store values, we check properly what to do if the
   values are already defined.
 * check valid nestings
-* @deffn, @deffnx, etc.
-* Implement parser options
+* Implement more parser options
 * Character encoding issues - read and pass to Perl correctly.
 * gnulib integration for strrchr, strchrnul, asprintf, etc.  Eliminate
   #define _GNU_SOURCE.
@@ -15,6 +14,92 @@
 efficiency.  Also newSVpv (..., 0);
 * Add "TODO" anywhere in the code with explanations of what is not done yet.
 
+Texinfo::Report::gdt and parse_texi_line -
+
+Multiple Texinfo::Parser objects are created in a run of texi2any.  The
+main one is to parse the input file, but it is also used by gdt - the 
+string translation function.  (I haven't ruled out that it is also used 
+somewhere else as well.)
+
+This makes replacing the Texinfo::Parser module more complicated.
+
+gdt parses pieces Texinfo code that are the translations of strings in 
+po_document/texinfo_document.pot.  It does this by adding "@value" 
+before strings surrounded by braces, and then calling "parse_texi_line" 
+with these @value's set to the appropriate value.  These values could be 
+a string, a hash reference (representing a tree element), or an array 
+reference (representing a "content array" of tree elements).
+
+parse_texi_line returns a Perl Texinfo tree which is at last converted 
+into the output Info/HTML/etc.
+
+These values that are not strings are a challenge for how they can be 
+passed back into C.  We have code (in api.c) to translate our C 
+representations of the Texinfo tree to Perl data structures, but not 
+vice versa.
+
+Suggestions:
+* Convert the values with $self->_convert to strings, then give the 
+values as strings, followed by parsing as Texinfo.  Care would have to 
+be taken to quote correctly special characters in the values, like "@", 
+"{" and "}".
+* Don't call parse_texi_line at all.  This would be a simplification of 
+the overall structure of the makeinfo program and might not lose much 
+flexibility in return.  Most of the strings in texinfo_document.pot 
+don't use any Texinfo commands, and the others could be expressed 
+directly in the output format, e.g.  instead of
+
+msgid "{category} on {class}: @strong{{name}}
+
+it could be
+
+msgid "{category} on {class}: <strong>{name}</strong>"
+
+Some of the translation strings use Texinfo commands for characters, 
+e.g. "@'e".  This would have to be replaced with whatever method other 
+programs using gettext use for special characters.
+* Write code translating Perl tree elements into C data structures and 
+allow them to be passed to the parser as @value's.
+
+To use the current implementation, do the following to 
+tp/Texinfo/Report.pm:
+
+Index: Report.pm
+===================================================================
+--- Report.pm   (revision 6198)
++++ Report.pm   (working copy)
+@@ -48,7 +48,8 @@ use File::Basename;
+ 
+ use Locale::Messages;
+ # to be able to load a parser if none was given to gdt.
+-use Texinfo::Parser;
++#use Texinfo::Parser;
++use Parsetexi;
+ 
+ # return the errors and warnings
+ sub errors($)
+@@ -399,7 +400,8 @@ sub gdt($$;$$)
+     }
+   }
+   #my $parser = Texinfo::Parser::parser($parser_conf);
+-  my $parser = Texinfo::Parser::simple_parser($parser_conf);
++  #my $parser = Texinfo::Parser::simple_parser($parser_conf);
++  my $parser = Parsetexi::parser($parser_conf);
+   if ($parser->{'DEBUG'}) {
+     print STDERR "GDT $translation_result\n";
+   }
+
+===================================================================
+
+As said, the implementation is not complete so it gives output like
+
+ -- <<HASH WITH NO TEXT>>: count-loop <<ARRAY VALUE>>
+      This imaginary special form implements a loop that executes the
+      BODY forms and then increments the variable VAR on each iteration.
+
+
+
+
 Integration with rest of Perl code:
 
 Passing data as Perl code to be evaluated in very slow, which limits the 

Modified: trunk/parsetexi/api.c
===================================================================
--- trunk/parsetexi/api.c       2015-04-09 19:40:48 UTC (rev 6216)
+++ trunk/parsetexi/api.c       2015-04-10 11:27:51 UTC (rev 6217)
@@ -49,6 +49,19 @@
   return Root;
 }
 
+/* Set ROOT to root of tree obtained by parsing the Texinfo code in STRING.
+   Used for parse_texi_line. */
+void
+parse_string (char *string)
+{
+  ELEMENT *root;
+  init_index_commands (); /* FIXME - probably not necessary */
+  root = new_element (ET_root_line);
+  input_push_text (strdup (string));
+  Root = parse_texi (root);
+}
+
+
 char *
 element_type_name (ELEMENT *e)
 {

Modified: trunk/parsetexi/commands.h
===================================================================
--- trunk/parsetexi/commands.h  2015-04-09 19:40:48 UTC (rev 6216)
+++ trunk/parsetexi/commands.h  2015-04-10 11:27:51 UTC (rev 6217)
@@ -33,7 +33,7 @@
    ? builtin_command_data[(id)] \
    : user_defined_command_data[(id) & ~USER_COMMAND_BIT])
 
-#define command_flags(e) (command_data((e)->cmd).flags)
+#define command_flags(e) (!(e) ? 0 : (command_data((e)->cmd).flags))
 #define command_name(cmd) (command_data(cmd).cmdname)
 
 enum command_id add_texinfo_command (char *name);

Modified: trunk/parsetexi/macro.c
===================================================================
--- trunk/parsetexi/macro.c     2015-04-09 19:40:48 UTC (rev 6216)
+++ trunk/parsetexi/macro.c     2015-04-10 11:27:51 UTC (rev 6217)
@@ -451,14 +451,26 @@
 static size_t value_space;
 
 void
+wipe_values (void)
+{
+  size_t i;
+  for (i = 0; i < value_number; i++)
+    {
+      free (value_list[i].name);
+      free (value_list[i].value);
+    }
+  value_number = 0;
+}
+
+void
 store_value (char *name, char *value)
 {
   if (value_number == value_space)
     {
       value_list = realloc (value_list, (value_space += 5) * sizeof (VALUE));
     }
-  value_list[value_number].name = name;
-  value_list[value_number++].value = value;
+  value_list[value_number].name = strdup (name);
+  value_list[value_number++].value = strdup (value);
 }
 /* TODO: What if it is already defined? */
 

Modified: trunk/parsetexi/parser.c
===================================================================
--- trunk/parsetexi/parser.c    2015-04-09 19:40:48 UTC (rev 6216)
+++ trunk/parsetexi/parser.c    2015-04-10 11:27:51 UTC (rev 6217)
@@ -30,9 +30,6 @@
 
 //macro_stack;
 
-/* Declarations of functions in this file */
-ELEMENT *parse_texi (ELEMENT *root_elt);
-
 
 /* Utility functions */
 
@@ -896,6 +893,8 @@
                 {
                   /* TODO: The Perl code has cases for the value being
                      an array or hash - check when this can happen. */
+                  /* This happens when the values are set by the "gdt" 
function 
+                     in Structuring.pm.  */
 
                   line++; /* past '}' */
                   input_push_text (strdup (line));

Modified: trunk/parsetexi/parser.h
===================================================================
--- trunk/parsetexi/parser.h    2015-04-09 19:40:48 UTC (rev 6216)
+++ trunk/parsetexi/parser.h    2015-04-10 11:27:51 UTC (rev 6217)
@@ -30,6 +30,7 @@
                            char **line_inout);
 
 /* In parser.c */
+ELEMENT *parse_texi (ELEMENT *root_elt);
 void push_conditional_stack (enum command_id cond);
 enum command_id pop_conditional_stack (void);
 extern size_t conditional_number;