posix_openpt on OpenBSD

[Bruno Haible]

On OpenBSD 4.9, I get these test results:

PASS: test-openpty

test-posix_openpt.c:52: assertion failed
FAIL: test-posix_openpt

On this platform, gnulib replaces openpty (thin wrapper) and
posix_openpt (new implementation). But it fails because there is no
/dev/ptmx.

Adding this workaround:

# if defined __OpenBSD__
  if (master < 0)
    {
      /* Try various master names of *BSD: /dev/pty[p-sP-S][0-9a-v]  */
      int upper;
      int char1;
      int char2;

      for (upper = 0; upper <= 1; upper++)
        for (char1 = (upper ? 'P' : 'p'); char1 <= (upper ? 'S' : 's'); char1++)
          for (char2 = '0'; char2 <= 'v'; (char2 == '9' ? char2 = 'a' : 
char2++))
            {
              char master_name[32];

              sprintf (master_name, "/dev/pty%c%c", char1, char2);
              master = open (master_name, flags);
              if (master >= 0)
                break;
            }
  }
# endif

does not fix it: The first 3 open() calls fail with EIO, the remaining ones
with ENOENT. But this fixes it:


2011-10-20  Bruno Haible  <address@hidden>

        posix_openpt: Support for FreeBSD.
        * lib/posix_openpt.c [OpenBSD]: Include <sys/ioctl.h>, <sys/tty.h>.
        (posix_openpt) [OpenBSD]: New code.
        * lib/grantpt.c: Include <fcntl.h>.
        (grantpt) [OpenBSD]: Only test whether fd is valid, nothing else.
        * modules/grantpt (Depends-on): Add fcntl-h.

--- lib/posix_openpt.c.orig     Thu Oct 20 17:32:54 2011
+++ lib/posix_openpt.c  Thu Oct 20 17:30:40 2011
@@ -19,8 +19,12 @@
 /* Specification.  */
 #include <stdlib.h>
 
-#include <fcntl.h>
 #include <errno.h>
+#include <fcntl.h>
+#if defined __OpenBSD__
+# include <sys/ioctl.h>
+# include <sys/tty.h>
+#endif
 
 int
 posix_openpt (int flags)
@@ -37,7 +41,34 @@
   master = -1;
   errno = ENOSYS;
 
-#else /* MacOS, OpenBSD, HP-UX, IRIX, Solaris 9, Cygwin 1.5 */
+#elif defined __OpenBSD__
+
+  /* On OpenBSD, master and slave of a pseudo-terminal are allocated together,
+     by opening /dev/ptm and applying the PTMGET ioctl to it.  */
+  int fd;
+  struct ptmget data;
+
+  fd = open (PATH_PTMDEV, O_RDWR);
+  if (fd >= 0)
+    {
+      if (ioctl (fd, PTMGET, &data) >= 0)
+        {
+          master = data.cfd;
+          close (data.sfd);
+          close (fd);
+        }
+      else
+        {
+          int saved_errno = errno;
+          close (fd);
+          errno = saved_errno;
+          master = -1;
+        }
+    }
+  else
+    master = -1;
+
+#else /* MacOS X, Minix, HP-UX, IRIX, OSF/1, Solaris 9, Cygwin 1.5 */
 
   /* Most systems that lack posix_openpt() have /dev/ptmx.  */
   master = open ("/dev/ptmx", flags);
--- lib/grantpt.c.orig  Thu Oct 20 17:32:54 2011
+++ lib/grantpt.c       Thu Oct 20 17:25:00 2011
@@ -21,6 +21,7 @@
 
 #include <assert.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <string.h>
 #include <sys/wait.h>
 #include <unistd.h>
@@ -46,6 +47,13 @@
 int
 grantpt (int fd)
 {
+#if defined __OpenBSD__
+  /* On OpenBSD, master and slave of a pseudo-terminal are allocated together,
+     through an ioctl on /dev/ptm.  There is no need for grantpt().  */
+  if (fcntl (fd, F_GETFD) < 0)
+    return -1;
+  return 0;
+#else
   /* This function is most often called from a process without 'root'
      credentials.  Use the helper program.  */
   int retval = -1;
@@ -56,20 +64,20 @@
     {
       /* This is executed in the child process.  */
 
-#if HAVE_SETRLIMIT && defined RLIMIT_CORE
+# if HAVE_SETRLIMIT && defined RLIMIT_CORE
       /* Disable core dumps.  */
       struct rlimit rl = { 0, 0 };
       __setrlimit (RLIMIT_CORE, &rl);
-#endif
+# endif
 
       /* We pass the master pseudo terminal as file descriptor PTY_FILENO.  */
       if (fd != PTY_FILENO)
         if (__dup2 (fd, PTY_FILENO) < 0)
           _exit (FAIL_EBADF);
 
-#ifdef CLOSE_ALL_FDS
+# ifdef CLOSE_ALL_FDS
       CLOSE_ALL_FDS ();
-#endif
+# endif
 
       execle (_PATH_PT_CHOWN, strrchr (_PATH_PT_CHOWN, '/') + 1, NULL, NULL);
       _exit (FAIL_EXEC);
@@ -111,4 +119,5 @@
 
  cleanup:
   return retval;
+#endif
 }
--- modules/grantpt.orig        Thu Oct 20 17:32:55 2011
+++ modules/grantpt     Thu Oct 20 17:25:32 2011
@@ -9,6 +9,7 @@
 Depends-on:
 stdlib
 extensions
+fcntl-h         [test $HAVE_GRANTPT = 0]
 pt_chown        [test $HAVE_GRANTPT = 0]
 waitpid         [test $HAVE_GRANTPT = 0]
 configmake      [test $HAVE_GRANTPT = 0]

-- 
In memoriam Eduard Brücklmeier <http://en.wikipedia.org/wiki/Eduard_Brücklmeier>