ptsname_r test

[Bruno Haible]

On Solaris 10 and IRIX 6.5, I'm seeing this test failure:

test-ptsname_r.c:117: assertion failed
FAIL: test-ptsname_r

The reason is that isatty() has not set errno. This fixes it.


2012-06-24  Bruno Haible  <address@hidden>

        ptsname_r: Fix test failures on IRIX, Solaris.
        * m4/ptsname_r.m4 (gl_PREREQ_PTSNAME_R): Test whether isatty sets
        errno when it fails. Define ISATTY_FAILS_WITHOUT_SETTING_ERRNO
        accordingly.
        * lib/ptsname_r.c: Include <fcntl.h>.
        (__ptsname_r): When isatty returned false, then on IRIX, Solaris
        set errno if fd is invalid.
        * tests/test-isatty.c (main): Update comments.

--- lib/ptsname_r.c.orig        Sun Jun 24 16:59:15 2012
+++ lib/ptsname_r.c     Sun Jun 24 16:49:16 2012
@@ -19,6 +19,7 @@
 #include <stdlib.h>
 
 #include <errno.h>
+#include <fcntl.h>
 #include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -59,8 +60,16 @@
     }
 
   if (!__isatty (fd))
-    /* We rely on isatty to set errno properly (i.e. EBADF or ENOTTY).  */
-    return errno;
+    {
+#if ISATTY_FAILS_WITHOUT_SETTING_ERRNO && defined F_GETFL /* IRIX, Solaris */
+      /* Set errno.  */
+      if (fcntl (fd, F_GETFL) != -1)
+        errno = ENOTTY;
+#else
+      /* We rely on isatty to set errno properly (i.e. EBADF or ENOTTY).  */
+#endif
+      return errno;
+    }
 
   if (buflen < strlen (_PATH_TTY) + 3)
     {
--- m4/ptsname_r.m4.orig        Sun Jun 24 16:59:15 2012
+++ m4/ptsname_r.m4     Sun Jun 24 16:49:51 2012
@@ -1,4 +1,4 @@
-# ptsname_r.m4 serial 2
+# ptsname_r.m4 serial 3
 dnl Copyright (C) 2010-2012 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -46,5 +46,38 @@
 
 # Prerequisites of lib/ptsname.c.
 AC_DEFUN([gl_PREREQ_PTSNAME_R], [
-  :
+  AC_REQUIRE([AC_CANONICAL_HOST]) dnl for cross-compiles
+  AC_CACHE_CHECK([whether isatty sets errno when it fails],
+    [gl_cv_func_isatty_sets_errno],
+    [AC_RUN_IFELSE(
+       [AC_LANG_PROGRAM(
+          [[#include <errno.h>
+            #if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
+            # include <io.h>
+            #else
+            # include <unistd.h>
+            #endif
+          ]],
+          [[errno = 0;
+            isatty (-1);
+            return errno == 0;
+          ]])
+       ],
+       [gl_cv_func_isatty_sets_errno=yes],
+       [gl_cv_func_isatty_sets_errno=no],
+       [case "$host_os" in
+          irix* | solaris* | mingw*)
+            gl_cv_func_isatty_sets_errno="guessing no";;
+          *)
+            gl_cv_func_isatty_sets_errno="guessing yes";;
+        esac
+       ])
+    ])
+  case "$gl_cv_func_isatty_sets_errno" in
+    *yes) ;;
+    *)
+      AC_DEFINE([ISATTY_FAILS_WITHOUT_SETTING_ERRNO], [1],
+        [Define to 1 if isatty() may fail without setting errno.])
+      ;;
+  esac
 ])
--- tests/test-isatty.c.orig    Sun Jun 24 16:59:15 2012
+++ tests/test-isatty.c Sun Jun 24 14:54:46 2012
@@ -45,14 +45,14 @@
     errno = 0;
     ASSERT (isatty (-1) == 0);
     ASSERT (errno == EBADF
-            || errno == 0 /* seen on Solaris 10 */
+            || errno == 0 /* seen on IRIX 6.5, Solaris 10 */
            );
   }
   {
     errno = 0;
     ASSERT (isatty (99) == 0);
     ASSERT (errno == EBADF
-            || errno == 0 /* seen on Solaris 10 */
+            || errno == 0 /* seen on IRIX 6.5, Solaris 10 */
            );
   }