Re: fopen bug on Solaris, HP-UX

[Bruno Haible]

Eric Blake wrote:
> >     * lib/fopen.c (rpl_fopen): Return NULL if the mode specifies write
> >     access and the filename ends in a slash. Code copied from lib/open.c.
> 
> Should we also check for the bug, present in at least Irix 5.3, where
> fopen("/dev/null/","r") succeeds instead of failing with the required
> ENOTDIR?

The bug appears on the same platforms, namely HP-UX and Solaris 9. Therefore
for the moment an extra autoconf check is not needed. But I agree that it
should be worked around. I'm applying the patch below. It causes additional
system calls on these platforms, but only in the case when people use filenames
ending in a slash and really mean the directory (rare situation).

> What about working around the fact that mingw fails with EINVAL
> instead of ENOTDIR?

Although in general I don't bother much whether a function fails with one error
or another, here it's easy to get it right with mingw as well, using the trick
that we talked about yesterday.

Bruno


2008-09-24  Bruno Haible  <address@hidden>

        Ensure that a filename ending in a slash cannot be used to access a
        non-directory.
        * lib/open.c (rpl_open): When the filename ends in a slash, use fstat()
        to check whether it's really a directory.
        * lib/fopen.c: Include fcntl.h, unistd.h.
        (rpl_fopen): When the filename ends in a slash, use open(), fstat(),
        and fdopen().
        * modules/fopen (Depends-on): Add unistd.
        * tests/test-open.c (main): Try to open "/dev/null/" as a directory.
        * tests/test-fopen.c (main): Likewise.
        * doc/posix-functions/open.texi: Mention the HP-UX, Solaris bug.
        * doc/posix-functions/fopen.texi: Likewise.
        Reported by Eric Blake.

*** doc/posix-functions/fopen.texi.orig 2008-09-24 13:42:44.000000000 +0200
--- doc/posix-functions/fopen.texi      2008-09-24 13:01:12.000000000 +0200
***************
*** 10,16 ****
  @itemize
  @item
  This function does not fail when the file name argument ends in a slash
! and (without the slash) names a nonexistent file, on some platforms:
  HP-UX 11.00, Solaris 9.
  @item
  On Windows platforms (excluding Cygwin), this function does usually not
--- 10,17 ----
  @itemize
  @item
  This function does not fail when the file name argument ends in a slash
! and (without the slash) names a nonexistent file or a file that is not a
! directory, on some platforms:
  HP-UX 11.00, Solaris 9.
  @item
  On Windows platforms (excluding Cygwin), this function does usually not
*** doc/posix-functions/open.texi.orig  2008-09-24 13:42:44.000000000 +0200
--- doc/posix-functions/open.texi       2008-09-24 13:01:10.000000000 +0200
***************
*** 10,16 ****
  @itemize
  @item
  This function does not fail when the file name argument ends in a slash
! and (without the slash) names a nonexistent file, on some platforms:
  HP-UX 11.00, Solaris 9.
  @item
  On Windows platforms (excluding Cygwin), this function does usually not
--- 10,17 ----
  @itemize
  @item
  This function does not fail when the file name argument ends in a slash
! and (without the slash) names a nonexistent file or a file that is not a
! directory, on some platforms:
  HP-UX 11.00, Solaris 9.
  @item
  On Windows platforms (excluding Cygwin), this function does usually not
*** lib/fopen.c.orig    2008-09-24 13:42:44.000000000 +0200
--- lib/fopen.c 2008-09-24 13:26:42.000000000 +0200
***************
*** 22,33 ****
--- 22,40 ----
  #include <stdio.h>
  
  #include <errno.h>
+ #include <fcntl.h>
  #include <string.h>
+ #include <unistd.h>
  
  FILE *
  rpl_fopen (const char *filename, const char *mode)
  #undef fopen
  {
+ #if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
+   if (strcmp (filename, "/dev/null") == 0)
+     filename = "NUL";
+ #endif
+ 
  #if FOPEN_TRAILING_SLASH_BUG
    /* If the filename ends in a slash and a mode that requires write access is
       specified, then fail.
***************
*** 45,65 ****
       fails with errno = EISDIR in this case.
       If the named file does not exist or does not name a directory, then
       fopen() must fail since the file does not contain a '.' directory.  */
!   if (mode[0] == 'w' || mode[0] == 'a')
!     {
!       size_t len = strlen (filename);
!       if (len > 0 && filename[len - 1] == '/')
!       {
!         errno = EISDIR;
          return NULL;
-       }
-     }
- #endif
  
! #if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
!   if (strcmp (filename, "/dev/null") == 0)
!     filename = "NUL";
! #endif
  
    return fopen (filename, mode);
  }
--- 52,92 ----
       fails with errno = EISDIR in this case.
       If the named file does not exist or does not name a directory, then
       fopen() must fail since the file does not contain a '.' directory.  */
!   {
!     size_t len = strlen (filename);
!     if (len > 0 && filename[len - 1] == '/')
!       {
!       int fd;
!       struct stat statbuf;
!       FILE *fp;
! 
!       if (mode[0] == 'w' || mode[0] == 'a')
!         {
!           errno = EISDIR;
!           return NULL;
!         }
! 
!       fd = open (filename, O_RDONLY);
!       if (fd < 0)
          return NULL;
  
!       if (fstat (fd, &statbuf) >= 0 && !S_ISDIR (statbuf.st_mode))
!         {
!           errno = ENOTDIR;
!           return NULL;
!         }
! 
!       fp = fdopen (fd, mode);
!       if (fp == NULL)
!         {
!           int saved_errno = errno;
!           close (fd);
!           errno = saved_errno;
!         }
!       return fp;
!       }
!   }
! # endif
  
    return fopen (filename, mode);
  }
*** lib/open.c.orig     2008-09-24 13:42:44.000000000 +0200
--- lib/open.c  2008-09-24 13:21:57.000000000 +0200
***************
*** 35,40 ****
--- 35,41 ----
  # undef open
  {
    mode_t mode;
+   int fd;
  
    mode = 0;
    if (flags & O_CREAT)
***************
*** 52,57 ****
--- 53,63 ----
        va_end (arg);
      }
  
+ # if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
+   if (strcmp (filename, "/dev/null") == 0)
+     filename = "NUL";
+ # endif
+ 
  # if OPEN_TRAILING_SLASH_BUG
    /* If the filename ends in a slash and one of O_CREAT, O_WRONLY, O_RDWR
       is specified, then fail.
***************
*** 85,95 ****
      }
  # endif
  
! # if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
!   if (strcmp (filename, "/dev/null") == 0)
!     filename = "NUL";
  # endif
  
!   return open (filename, flags, mode);
  }
  #endif
--- 91,127 ----
      }
  # endif
  
!   fd = open (filename, flags, mode);
! 
! # if OPEN_TRAILING_SLASH_BUG
!   /* If the filename ends in a slash and fd does not refer to a directory,
!      then fail.
!      Rationale: POSIX 
<http://www.opengroup.org/susv3/basedefs/xbd_chap04.html>
!      says that
!        "A pathname that contains at least one non-slash character and that
!         ends with one or more trailing slashes shall be resolved as if a
!         single dot character ( '.' ) were appended to the pathname."
!      and
!        "The special filename dot shall refer to the directory specified by
!         its predecessor."
!      If the named file without the slash is not a directory, open() must fail
!      with ENOTDIR.  */
!   if (fd >= 0)
!     {
!       size_t len = strlen (filename);
!       if (len > 0 && filename[len - 1] == '/')
!       {
!         struct stat statbuf;
! 
!         if (fstat (fd, &statbuf) >= 0 && !S_ISDIR (statbuf.st_mode))
!           {
!             errno = ENOTDIR;
!             return -1;
!           }
!       }
!     }
  # endif
  
!   return fd;
  }
  #endif
*** modules/fopen.orig  2008-09-24 13:42:44.000000000 +0200
--- modules/fopen       2008-09-24 13:40:26.000000000 +0200
***************
*** 7,12 ****
--- 7,13 ----
  
  Depends-on:
  stdio
+ unistd
  
  configure.ac:
  gl_FUNC_FOPEN
*** tests/test-fopen.c.orig     2008-09-24 13:42:44.000000000 +0200
--- tests/test-fopen.c  2008-09-24 13:02:02.000000000 +0200
***************
*** 37,42 ****
--- 37,43 ----
  main ()
  {
    ASSERT (fopen ("nonexist.ent/", "w") == NULL);
+   ASSERT (fopen ("/dev/null/", "r") == NULL);
  
    ASSERT (fopen ("/dev/null", "r") != NULL);
  
*** tests/test-open.c.orig      2008-09-24 13:42:44.000000000 +0200
--- tests/test-open.c   2008-09-24 13:02:39.000000000 +0200
***************
*** 39,44 ****
--- 39,45 ----
  main ()
  {
    ASSERT (open ("nonexist.ent/", O_CREAT, 0600) < 0);
+   ASSERT (open ("/dev/null/", O_RDONLY) < 0);
  
    ASSERT (open ("/dev/null", O_RDONLY) >= 0);