[RFC PATCH 2.5/4] ptsname_r: work with POSIX isatty, for Solaris

[Eric Blake]

POSIX does not require isatty to set a sane errno; and Solaris 10
leaves errno unchanged.  Fix this with a helper function, so that
the body of __ptsname_r from glibc still syncs up.

* lib/ptsname_r.c (isatty_errno): New helper function.
* modules/ptsname_r (Depends-on): Add fcntl-h.

Signed-off-by: Eric Blake <address@hidden>
---

Alas, I won't be applying this, at least not yet.  It turns out that
on Solaris, isatty(master) is false.  So we need to provide an
isatty-gnu module anyways (POSIX is silent on whether isatty() must
succeed or fail on the master side of a pty, but glibc assumes it
will succeed), or at least figure out how to make isatty_errno
return 1 on a master side instead of failure.

 ChangeLog         |    4 ++++
 lib/ptsname_r.c   |   18 +++++++++++++++++-
 modules/ptsname_r |    1 +
 3 files changed, 22 insertions(+), 1 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 404b95d..8e60b0c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,9 @@
 2011-11-09  Eric Blake  <address@hidden>

+       ptsname_r: work with POSIX isatty, for Solaris
+       * lib/ptsname_r.c (isatty_errno): New helper function.
+       * modules/ptsname_r (Depends-on): Add fcntl-h.
+
        ptsname_r-tests: new test module
        * modules/ptsname_r-tests: New module.
        * tests/test-ptsname_r.c: New file.
diff --git a/lib/ptsname_r.c b/lib/ptsname_r.c
index e7f2d5b..b85dfce 100644
--- a/lib/ptsname_r.c
+++ b/lib/ptsname_r.c
@@ -33,12 +33,28 @@
 #  define _PATH_DEV "/dev/"
 # endif

+# include <fcntl.h>
+
 # define __set_errno(e) errno = (e)
-# define __isatty isatty
+# define __isatty isatty_errno
 # define __stat stat
 # define __ttyname_r ttyname_r
 # define __ptsname_r ptsname_r

+/* POSIX does not require isatty to set errno, but glibc assumes it to
+   be EBADF or ENOTTY on failure.  */
+static int
+isatty_errno (int fd)
+{
+  int result = isatty (fd);
+  /* No need to drag in the fcntl module for mingw.  */
+# ifdef F_GETFL
+  if (!result && fcntl (fd, F_GETFL) >= 0)
+    errno = ENOTTY;
+# endif
+  return result;
+}
+
 #endif


diff --git a/modules/ptsname_r b/modules/ptsname_r
index daf1d98..accd8a4 100644
--- a/modules/ptsname_r
+++ b/modules/ptsname_r
@@ -8,6 +8,7 @@ m4/ptsname_r.m4
 Depends-on:
 stdlib
 extensions
+fcntl-h         [test $HAVE_PTSNAME_R = 0]
 ttyname_r       [test $HAVE_PTSNAME_R = 0]

 configure.ac:
-- 
1.7.4.4