Re: getting EBADF on MSVC

[Bruno Haible]

> The newer runtime libraries of MSVC no longer return error codes from
> functions like printf(), close(), dup2(), _get_osfhandle(), when you pass
> an invalid format string or invalid file descriptor.[1][2]

This patch protects the dup2() calls in gnulib.


2011-09-18  Bruno Haible  <address@hidden>

        dup2: Support for MSVC.
        * lib/dup2.c: Include msvc-inval.h.
        (rpl_dup2): Handle invalid parameter notifications during dup2 and
        _get_osfhandle calls.
        * modules/dup2 (Depends-on): Add msvc-inval.
        * doc/posix-functions/dup2.texi: Mention problem on MSVC.

--- doc/posix-functions/dup2.texi.orig  Sun Sep 18 10:34:07 2011
+++ doc/posix-functions/dup2.texi       Sun Sep 18 03:33:16 2011
@@ -17,6 +17,10 @@
 mingw, MSVC 9.
 
 @item
+This function crashes when invoked with invalid arguments on some platforms:
+MSVC 9.
+
address@hidden
 This function resets the @code{FD_CLOEXEC} flag when duplicating an fd
 to itself on some platforms:
 Haiku.
--- lib/dup2.c.orig     Sun Sep 18 10:34:07 2011
+++ lib/dup2.c  Sun Sep 18 10:28:03 2011
@@ -31,6 +31,8 @@
 # include <windows.h>
 #endif
 
+#include "msvc-inval.h"
+
 #if HAVE_DUP2
 
 # undef dup2
@@ -45,7 +47,19 @@
      future dup2 calls will hang.  */
   if (fd == desired_fd)
     {
-      if ((HANDLE) _get_osfhandle (fd) == INVALID_HANDLE_VALUE)
+      HANDLE handle;
+
+      TRY_MSVC_INVAL
+        {
+          handle = (HANDLE) _get_osfhandle (fd);
+        }
+      CATCH_MSVC_INVAL
+        {
+          handle = INVALID_HANDLE_VALUE;
+        }
+      DONE_MSVC_INVAL
+
+      if (handle == INVALID_HANDLE_VALUE)
         {
           errno = EBADF;
           return -1;
@@ -64,7 +78,18 @@
   if (fd == desired_fd)
     return fcntl (fd, F_GETFL) == -1 ? -1 : fd;
 # endif
-  result = dup2 (fd, desired_fd);
+
+  TRY_MSVC_INVAL
+    {
+      result = dup2 (fd, desired_fd);
+    }
+  CATCH_MSVC_INVAL
+    {
+      result = -1;
+      errno = EBADF;
+    }
+  DONE_MSVC_INVAL
+
 # ifdef __linux__
   /* Correct a Linux return value.
      
<http://git.kernel.org/?p=linux/kernel/git/stable/linux-2.6.30.y.git;a=commitdiff;h=2b79bc4f7ebbd5af3c8b867968f9f15602d5f802>
--- modules/dup2.orig   Sun Sep 18 10:34:07 2011
+++ modules/dup2        Sun Sep 18 10:34:02 2011
@@ -8,6 +8,7 @@
 Depends-on:
 unistd
 dup2-obsolete
+msvc-inval      [test $HAVE_DUP2 = 0 || test $REPLACE_DUP2 = 1]
 
 configure.ac:
 gl_FUNC_DUP2
-- 
In memoriam Bernhard Bästlein <http://en.wikipedia.org/wiki/Bernhard_Bästlein>