fix possible buffer overrun in vasnprintf

[Bruno Haible]

This fixes a wrong estimate of a buffer size, when a format directive
'a' or 'A' is used.

Fortunately this affects only platforms without snprintf or _snprintf
functions, i.e. systems such as OSF/1 4.0 or Solaris < 2.6.


2007-02-25  Bruno Haible  <address@hidden>

        * lib/vasnprintf.c (VASNPRINTF): Fix estimate of size needed for a
        'a' or 'A' conversion.

*** lib/vasnprintf.c    30 Jan 2007 01:07:22 -0000      1.22
--- lib/vasnprintf.c    25 Feb 2007 21:10:43 -0000
***************
*** 430,441 ****
                      break;
  
                    case 'e': case 'E': case 'g': case 'G':
-                   case 'a': case 'A':
                      tmp_length =
                        12; /* sign, decimal point, exponent etc. */
                      tmp_length = xsum (tmp_length, precision);
                      break;
  
                    case 'c':
  # if HAVE_WINT_T && !WIDE_CHAR_VERSION
                      if (type == TYPE_WIDE_CHAR)
--- 430,461 ----
                      break;
  
                    case 'e': case 'E': case 'g': case 'G':
                      tmp_length =
                        12; /* sign, decimal point, exponent etc. */
                      tmp_length = xsum (tmp_length, precision);
                      break;
  
+                   case 'a': case 'A':
+ # if HAVE_LONG_DOUBLE
+                     if (type == TYPE_LONGDOUBLE)
+                       tmp_length =
+                         (unsigned int) (LDBL_DIG
+                                         * 0.831 /* decimal -> hexadecimal */
+                                        )
+                         + 1; /* turn floor into ceil */
+                     else
+ # endif
+                       tmp_length =
+                         (unsigned int) (DBL_DIG
+                                         * 0.831 /* decimal -> hexadecimal */
+                                        )
+                         + 1; /* turn floor into ceil */
+                     if (tmp_length < precision)
+                       tmp_length = precision;
+                     /* Account for sign, decimal point etc. */
+                     tmp_length = xsum (tmp_length, 12);
+                     break;
+ 
                    case 'c':
  # if HAVE_WINT_T && !WIDE_CHAR_VERSION
                      if (type == TYPE_WIDE_CHAR)