[Bug-gnulib] linebreak.c proposed patches for size-calculation overflows

[Paul Eggert]

While auditing the gnulib code I found some potential size-calculation
overflow problems in linebreak.c.  Sorry, I don't use linebreak.c, so
I haven't compiled or tested the following fix.

2003-10-30  Paul Eggert  <address@hidden>

        * linebreak.c (iconv_string_length): Return (size_t)(-1) when
        there is an size-calculation overflow.
        (mbs_possible_linebreaks, mbs_width_linebreaks):
        Do not overrun output buffer if there is an overflow
        in size calculations.

--- linebreak.c.~1.4.~  Wed Jul 30 23:27:05 2003
+++ linebreak.c Thu Oct 30 23:49:24 2003
@@ -1379,9 +1379,10 @@ iconv_string_length (iconv_t cd, const c
       char *outptr = tmpbuf;
       size_t outsize = TMPBUFSIZE;
       size_t res = iconv (cd, (ICONV_CONST char **) &inptr, &insize, &outptr, 
&outsize);
-      if (res == (size_t)(-1) && errno != E2BIG)
+      size_t tmpcount = outptr - tmpbuf;
+      count += tmpcount;
+      if ((res == (size_t)(-1) && errno != E2BIG) || count < tmpcount)
         return (size_t)(-1);
-      count += outptr - tmpbuf;
     }
   /* Avoid glibc-2.1 bug and Solaris 7 through 9 bug.  */
 #if defined _LIBICONV_VERSION \
@@ -1390,9 +1391,10 @@ iconv_string_length (iconv_t cd, const c
     char *outptr = tmpbuf;
     size_t outsize = TMPBUFSIZE;
     size_t res = iconv (cd, NULL, NULL, &outptr, &outsize);
-    if (res == (size_t)(-1))
+    size_t tmpcount = outptr - tmpbuf;
+    count += tmpcount;
+    if (res == (size_t)(-1) || count < tmpcount)
       return (size_t)(-1);
-    count += outptr - tmpbuf;
   }
   /* Return to the initial state.  */
   iconv (cd, NULL, NULL, NULL, NULL);
@@ -1515,11 +1517,14 @@ mbs_possible_linebreaks (const char *s, 
         {
           /* Determine the length of the resulting UTF-8 string.  */
           size_t m = iconv_string_length (to_utf8, s, n);
-          if (m != (size_t)(-1))
+          size_t two_m = 2 * m;
+          size_t memory_size = n * sizeof (size_t) + two_m;
+          if (n <= (size_t)(-1) / sizeof (size_t)
+              && m <= two_m && two_m <= memory_size)
             {
               /* Convert the string to UTF-8 and build a translation table
                  from offsets into s to offsets into the translated string.  */
-              char *memory = malloc (n * sizeof (size_t) + m + m);
+              char *memory = malloc (memory_size);
               if (memory != NULL)
                 {
                   size_t *offtable = (size_t *) memory;
@@ -1608,11 +1613,16 @@ mbs_width_linebreaks (const char *s, siz
         {
           /* Determine the length of the resulting UTF-8 string.  */
           size_t m = iconv_string_length (to_utf8, s, n);
-          if (m != (size_t)(-1))
+          size_t two_m = 2 * m;
+          size_t two_or_three_m = two_m + (o != NULL ? m : 0);
+          size_t memory_size = n * sizeof (size_t) + two_or_three_m;
+          if (n <= (size_t)(-1) / sizeof (size_t)
+              && m <= two_m && two_m <= two_or_three_m
+              && two_or_three_m <= memory_size)
             {
               /* Convert the string to UTF-8 and build a translation table
                  from offsets into s to offsets into the translated string.  */
-              char *memory = malloc (n * sizeof (size_t) + m + m + (o != NULL 
? m : 0));
+              char *memory = malloc (memory_size);
               if (memory != NULL)
                 {
                   size_t *offtable = (size_t *) memory;