[Bug-gnulib] portability fix for calloc size_t overflow bug

[Paul Eggert]

While we're on the subject of working around buggy allocators, I
installed this:

2004-11-17  Paul Eggert  <address@hidden>

        * lib/calloc.c (rpl_calloc): Defend against buggy calloc implementations
        that mishandle size_t overflow.
        * m4/calloc.m4 (_AC_FUNC_CALLOC_IF): Check for buggy calloc
        implementations that mishandle size_t overflow.

--- lib/calloc.c        10 Jun 2004 08:29:26 -0000      1.1
+++ lib/calloc.c        17 Nov 2004 22:47:27 -0000
@@ -1,4 +1,4 @@
-/* Work around the condition whereby calloc (n, s) fails when n*s is 0.
+/* calloc() function that is glibc compatible.
    This wrapper function is required at least on Tru64 UNIX 5.1.
    Copyright (C) 2004 Free Software Foundation, Inc.
 
@@ -31,9 +31,17 @@
 void *
 rpl_calloc (size_t n, size_t s)
 {
+  size_t bytes;
   if (n == 0)
     n = 1;
   if (s == 0)
     s = 1;
+
+  /* Defend against buggy calloc implementations that mishandle
+     size_t overflow.  */
+  bytes = n * s;
+  if (bytes / s != n)
+    return NULL;
+
   return calloc (n, s);
 }
--- m4/calloc.m4        7 Aug 2004 00:09:39 -0000       1.2
+++ m4/calloc.m4        17 Nov 2004 22:47:27 -0000
@@ -1,6 +1,25 @@
-#serial 2
+# calloc.m4 serial 3
 
-# Determine whether calloc (N, S) returns non-NULL when N*S is zero.
+# Copyright (C) 2004 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+# Written by Jim Meyering.
+
+# Determine whether calloc (N, S) returns non-NULL when N*S is zero,
+# and returns NULL when N*S overflows.
 # If so, define HAVE_CALLOC.  Otherwise, define calloc to rpl_calloc
 # and arrange to use a calloc wrapper function that does work in that case.
 
@@ -9,17 +28,11 @@
 # If `calloc (0, 0)' is properly handled, run IF-WORKS, otherwise, IF-NOT.
 AC_DEFUN([_AC_FUNC_CALLOC_IF],
 [AC_REQUIRE([AC_HEADER_STDC])dnl
+AC_REQUIRE([AC_TYPE_SIZE_T])dnl
 AC_CHECK_HEADERS(stdlib.h)
 AC_CACHE_CHECK([for GNU libc compatible calloc], ac_cv_func_calloc_0_nonnull,
-[AC_RUN_IFELSE(
-[AC_LANG_PROGRAM(
-[[#if STDC_HEADERS || HAVE_STDLIB_H
-# include <stdlib.h>
-#else
-char *calloc ();
-#endif
-]],
-                [exit (calloc (0, 0) ? 0 : 1);])],
+[AC_RUN_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT],
+                 [exit (!calloc (0, 0) || calloc ((size_t) -1 / 8 + 1, 8));])],
               [ac_cv_func_calloc_0_nonnull=yes],
               [ac_cv_func_calloc_0_nonnull=no],
               [ac_cv_func_calloc_0_nonnull=no])])