regex alloca fixes

[Paul Eggert]

I noticed that regex invoked alloca with unbounded size.  Rather
than use the allocsa module I thought it better to stick to glibc
style, and define __libc_use_alloca when _LIBC isn't defined.
I installed this (and filed glibc bug 1245):

2005-08-25  Paul Eggert  <address@hidden>

        * config/srclist.txt: Add glibc bug 1245.

        * lib/regexec.c (set_regs): Don't alloca with an unbounded size.

        alloca modernization/simplification for regex.
        * lib/regex.c: Remove portability cruft for alloca.  This no longer
        needs to be at the start of the file, and can be moved into
        regex_internal.h and simplified.
        * lib/regex_internal.h: Include <alloca.h>.
        (__libc_use_alloca) [!defined _LIBC]: New macro.
        * lib/regexec.c (build_trtable): Remove "#ifdef _LIBC",
        since the code now works outside glibc.

--- config/srclist.txt.~1.86.~  2005-08-25 13:39:57.000000000 -0700
+++ config/srclist.txt  2005-08-25 22:55:59.000000000 -0700
@@ -105,6 +105,7 @@ $LIBCSRC/stdlib/getsubopt.c         lib gpl
 #$LIBCSRC/posix/regcomp.c              lib gpl
 #
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1238
+# http://sources.redhat.com/bugzilla/show_bug.cgi?id=1245
 #$LIBCSRC/posix/regex.c                        lib gpl
 #
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1201
@@ -128,6 +129,7 @@ $LIBCSRC/stdlib/getsubopt.c         lib gpl
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1221
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1237
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1241
+# http://sources.redhat.com/bugzilla/show_bug.cgi?id=1245
 #$LIBCSRC/posix/regex_internal.h               lib gpl
 #
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1216
@@ -137,6 +139,7 @@ $LIBCSRC/stdlib/getsubopt.c         lib gpl
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1231
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1237
 # http://sources.redhat.com/bugzilla/show_bug.cgi?id=1241
+# http://sources.redhat.com/bugzilla/show_bug.cgi?id=1245
 #$LIBCSRC/posix/regexec.c              lib gpl
 #
 # c89 changes $LIBCSRC/string/strdup.c         lib gpl
--- lib/regex.c 24 Aug 2005 23:43:01 -0000      1.91
+++ lib/regex.c 26 Aug 2005 05:41:47 -0000
@@ -21,30 +21,6 @@
 #include "config.h"
 #endif
 
-#ifdef _AIX
-#pragma alloca
-#else
-# ifndef allocax           /* predefined by HP cc +Olibcalls */
-#  ifdef __GNUC__
-#   define alloca(size) __builtin_alloca (size)
-#  else
-#   if HAVE_ALLOCA_H
-#    include <alloca.h>
-#   else
-#    ifdef __hpux
-        void *alloca ();
-#    else
-#     if !defined __OS2__ && !defined WIN32
-        char *alloca ();
-#     else
-#      include <malloc.h>       /* OS/2 defines alloca in here */
-#     endif
-#    endif
-#   endif
-#  endif
-# endif
-#endif
-
 #ifdef _LIBC
 /* We have to keep the namespace clean.  */
 # define regfree(preg) __regfree (preg)
--- lib/regex_internal.h        25 Aug 2005 20:39:57 -0000      1.4
+++ lib/regex_internal.h        26 Aug 2005 05:41:47 -0000
@@ -431,6 +431,21 @@ static unsigned char re_string_fetch_byt
 #define re_string_skip_bytes(pstr,idx) ((pstr)->cur_idx += (idx))
 #define re_string_set_index(pstr,idx) ((pstr)->cur_idx = (idx))
 
+#include <alloca.h>
+
+#ifndef __LIBC
+# if HAVE_ALLOCA
+/* The OS usually guarantees only one guard page at the bottom of the stack,
+   and a page size can be as small as 4096 bytes.  So we cannot safely
+   allocate anything larger than 4096 bytes.  Also care for the possibility
+   of a few compiler-allocated temporary stack slots.  */
+#  define __libc_use_alloca(n) ((n) < 4032)
+# else
+/* alloca is implemented with malloc, so just use malloc.  */
+#  define __libc_use_alloca(n) 0
+# endif
+#endif
+
 #define re_malloc(t,n) ((t *) malloc ((n) * sizeof (t)))
 #define re_calloc(t,n) ((t *) calloc (n, sizeof (t)))
 #define re_realloc(p,t,n) ((t *) realloc (p, (n) * sizeof (t)))
--- lib/regexec.c       25 Aug 2005 20:39:57 -0000      1.8
+++ lib/regexec.c       26 Aug 2005 05:41:47 -0000
@@ -1354,6 +1354,7 @@ set_regs (const regex_t *preg, const re_
   struct re_fail_stack_t *fs;
   struct re_fail_stack_t fs_body = { 0, 2, NULL };
   regmatch_t *prev_idx_match;
+  int prev_idx_match_malloced = 0;
 
 #ifdef DEBUG
   assert (nmatch > 1);
@@ -1372,7 +1373,18 @@ set_regs (const regex_t *preg, const re_
   cur_node = dfa->init_node;
   re_node_set_init_empty (&eps_via_nodes);
 
-  prev_idx_match = (regmatch_t *) alloca (sizeof (regmatch_t) * nmatch);
+  if (__libc_use_alloca (nmatch * sizeof (regmatch_t)))
+    prev_idx_match = (regmatch_t *) alloca (nmatch * sizeof (regmatch_t));
+  else
+    {
+      prev_idx_match = re_malloc (regmatch_t, nmatch);
+      if (prev_idx_match == NULL)
+       {
+         free_fail_stack_return (fs);
+         return REG_ESPACE;
+       }
+      prev_idx_match_malloced = 1;
+    }
   memcpy (prev_idx_match, pmatch, sizeof (regmatch_t) * nmatch);
 
   for (idx = pmatch[0].rm_so; idx <= pmatch[0].rm_eo ;)
@@ -1390,6 +1402,8 @@ set_regs (const regex_t *preg, const re_
              if (reg_idx == nmatch)
                {
                  re_node_set_free (&eps_via_nodes);
+                 if (prev_idx_match_malloced)
+                   re_free (prev_idx_match);
                  return free_fail_stack_return (fs);
                }
              cur_node = pop_fail_stack (fs, &idx, nmatch, pmatch,
@@ -1398,6 +1412,8 @@ set_regs (const regex_t *preg, const re_
          else
            {
              re_node_set_free (&eps_via_nodes);
+             if (prev_idx_match_malloced)
+               re_free (prev_idx_match);
              return REG_NOERROR;
            }
        }
@@ -1411,6 +1427,8 @@ set_regs (const regex_t *preg, const re_
          if (BE (cur_node == -2, 0))
            {
              re_node_set_free (&eps_via_nodes);
+             if (prev_idx_match_malloced)
+               re_free (prev_idx_match);
              free_fail_stack_return (fs);
              return REG_ESPACE;
            }
@@ -1420,11 +1438,15 @@ set_regs (const regex_t *preg, const re_
          else
            {
              re_node_set_free (&eps_via_nodes);
+             if (prev_idx_match_malloced)
+               re_free (prev_idx_match);
              return REG_NOMATCH;
            }
        }
     }
   re_node_set_free (&eps_via_nodes);
+  if (prev_idx_match_malloced)
+    re_free (prev_idx_match);
   return free_fail_stack_return (fs);
 }
 
@@ -3235,12 +3257,10 @@ build_trtable (re_dfa_t *dfa, re_dfastat
      from `state'.  `dests_node[i]' represents the nodes which i-th
      destination state contains, and `dests_ch[i]' represents the
      characters which i-th destination state accepts.  */
-#ifdef _LIBC
   if (__libc_use_alloca ((sizeof (re_node_set) + sizeof (bitset)) * SBC_MAX))
     dests_node = (re_node_set *)
       alloca ((sizeof (re_node_set) + sizeof (bitset)) * SBC_MAX);
   else
-#endif
     {
       dests_node = (re_node_set *)
        malloc ((sizeof (re_node_set) + sizeof (bitset)) * SBC_MAX);
@@ -3273,13 +3293,11 @@ build_trtable (re_dfa_t *dfa, re_dfastat
   if (BE (err != REG_NOERROR, 0))
     goto out_free;
 
-#ifdef _LIBC
   if (__libc_use_alloca ((sizeof (re_node_set) + sizeof (bitset)) * SBC_MAX
                         + ndests * 3 * sizeof (re_dfastate_t *)))
     dest_states = (re_dfastate_t **)
       alloca (ndests * 3 * sizeof (re_dfastate_t *));
   else
-#endif
     {
       dest_states = (re_dfastate_t **)
        malloc (ndests * 3 * sizeof (re_dfastate_t *));