acl: add Solaris and Cygwin support

[Bruno Haible]

So far, the test results on Solaris and Cygwin are:

  Solaris 10:
  PASS: test-file-has-acl.sh
  PASS: test-set-mode-acl.sh
  files tmpfile0 and tmpfile2 have different number of ACLs: 5 and 4
  FAIL: test-copy-acl.sh

  Cygwin:
  PASS: test-file-has-acl.sh
  PASS: test-set-mode-acl.sh
  files tmpfile0 and tmpfile2 have different number of ACLs: 5 and 4
  FAIL: test-copy-acl.sh

The reason is simply that, so far, the 'acl' module does not use the Solaris/
Cygwin ACL API.

Solaris 7..9 and Cygwin have an ACL API that allows to get/set an array
of ACL entries per file. Unlike in the POSIX-draft API, a single call
handles the "access" and the "default" ACL at once.

Solaris 10 adds to this API a variant (prefixed with "ACE_") where the
ACL entries are of a different type. The doc says that the older API applies
to UFS file systems, and the newer one to ZFS and NFSv4 file systems.
There is an API for determining which kind of ACL applies to a given file
([f]pathconf), but it may not work right in all situations, says the Sun doc.
So I think it's the safest to try both APIs one after the other.

It gets especially tricky when e.g. copying an ACL from an UFS to a ZFS
file system or vice versa. Fortunately we don't have to do the conversion
ourselves, I think: The docs indicate that the kernel does the conversion
automatically.

Then, a newer version of Solaris 10 added some sugar around these two variants:
an acl_t type, and acl_get, acl_get, acl_trivial, acl_fromtext etc. functions
in libsec.so.

I have only access to machines up to Solaris 10; Paul's original code is for
the augmented Solaris 10 systems. Since I cannot test in this situation,
I leave Paul's code intact.

Also, the machines I don't have access to don't have ZFS nor NFSv4 mounts;
so testing by people who have such machines is welcome!


2008-06-08  Bruno Haible  <address@hidden>

        Add support for Solaris 7..10 ACLs.
        * lib/acl-internal.h (acl_nontrivial, acl_ace_nontrivial): New
        declarations.
        * lib/file-has-acl.c (acl_nontrivial, acl_ace_nontrivial): New
        functions.
        (file_has_acl): Add implementation using Solaris 7..10 ACL API.
        * lib/set-mode-acl.c (qset_acl): Likewise.
        * lib/copy-acl.c (qcopy_acl): Likewise.

*** lib/acl-internal.h.orig     2008-06-08 15:58:20.000000000 +0200
--- lib/acl-internal.h  2008-06-08 15:57:16.000000000 +0200
***************
*** 158,163 ****
--- 158,176 ----
  extern int acl_access_nontrivial (acl_t);
  #  endif
  
+ # elif HAVE_ACL && defined GETACL /* Solaris, Cygwin, not HP-UX */
+ 
+ /* Return 1 if the given ACL is non-trivial.
+    Return 0 if it is trivial, i.e. equivalent to a simple stat() mode.  */
+ extern int acl_nontrivial (int count, aclent_t *entries);
+ 
+ #  ifdef ACE_GETACL
+ /* Test an ACL retrieved with ACE_GETACL.
+    Return 1 if the given ACL, consisting of COUNT entries, is non-trivial.
+    Return 0 if it is trivial, i.e. equivalent to a simple stat() mode.  */
+ extern int acl_ace_nontrivial (int count, ace_t *entries);
+ #  endif
+ 
  # endif
  
  #endif
*** lib/file-has-acl.c.orig     2008-06-08 15:58:20.000000000 +0200
--- lib/file-has-acl.c  2008-06-08 15:57:16.000000000 +0200
***************
*** 117,122 ****
--- 117,179 ----
  
  # endif
  
+ 
+ #elif USE_ACL && HAVE_ACL && defined GETACL /* Solaris, Cygwin, not HP-UX */
+ 
+ /* Test an ACL retrieved with GETACL.
+    Return 1 if the given ACL, consisting of COUNT entries, is non-trivial.
+    Return 0 if it is trivial, i.e. equivalent to a simple stat() mode.  */
+ int
+ acl_nontrivial (int count, aclent_t *entries)
+ {
+   int i;
+ 
+   for (i = 0; i < count; i++)
+     {
+       aclent_t *ace = &entries[i];
+ 
+       /* Note: If ace->a_type = USER_OBJ, ace->a_id is the st_uid from stat().
+        If ace->a_type = GROUP_OBJ, ace->a_id is the st_gid from stat().
+        We don't need to check ace->a_id in these cases.  */
+       if (!(ace->a_type == USER_OBJ
+           || ace->a_type == GROUP_OBJ
+           || ace->a_type == OTHER_OBJ
+           /* Note: Cygwin does not return a CLASS_OBJ ("mask:") entry
+              sometimes.  */
+           || ace->a_type == CLASS_OBJ))
+       return 1;
+     }
+   return 0;
+ }
+ 
+ # ifdef ACE_GETACL
+ 
+ /* Test an ACL retrieved with ACE_GETACL.
+    Return 1 if the given ACL, consisting of COUNT entries, is non-trivial.
+    Return 0 if it is trivial, i.e. equivalent to a simple stat() mode.  */
+ int
+ acl_ace_nontrivial (int count, ace_t *entries)
+ {
+   int i;
+ 
+   for (i = 0; i < count; i++)
+     {
+       ace_t *ace = &entries[i];
+ 
+       /* Note: If ace->a_flags = ACE_OWNER, ace->a_who is the st_uid from
+        stat().  If ace->a_flags = ACE_GROUP, ace->a_who is the st_gid from
+        stat().  We don't need to check ace->a_who in these cases.  */
+       if (!(ace->a_type == ALLOW
+           && (ace->a_flags == ACE_OWNER
+               || ace->a_flags == ACE_GROUP
+               || ace->a_flags == ACE_OTHER)))
+       return 1;
+     }
+   return 0;
+ }
+ 
+ # endif
+ 
  #endif
  
  
***************
*** 204,210 ****
  
  #  if HAVE_ACL_TRIVIAL
  
!       /* Solaris 10, which also has NFSv4 and ZFS style ACLs.  */
        return acl_trivial (name);
  
  #  else /* Solaris, Cygwin, general case */
--- 261,269 ----
  
  #  if HAVE_ACL_TRIVIAL
  
!       /* Solaris 10 (newer version), which has additional API declared in
!        <sys/acl.h> (acl_t) and implemented in libsec (acl_set, acl_trivial,
!        acl_fromtext, ...).  */
        return acl_trivial (name);
  
  #  else /* Solaris, Cygwin, general case */
***************
*** 212,220 ****
        /* Solaris 2.5 through Solaris 10, Cygwin, and contemporaneous versions
         of Unixware.  The acl() call returns the access and default ACL both
         at once.  */
!       int n = acl (name, GETACLCNT, 0, NULL);
!       return n < 0 ? (errno == ENOSYS ? 0 : -1) : (MIN_ACL_ENTRIES < n);
  
  #  endif
  
  # endif
--- 271,374 ----
        /* Solaris 2.5 through Solaris 10, Cygwin, and contemporaneous versions
         of Unixware.  The acl() call returns the access and default ACL both
         at once.  */
!       int count;
!       {
!       aclent_t *entries;
! 
!       for (;;)
!         {
!           count = acl (name, GETACLCNT, 0, NULL);
! 
!           if (count < 0)
!             {
!               if (errno == ENOSYS || errno == ENOTSUP)
!                 break;
!               else
!                 return -1;
!             }
! 
!           if (count == 0)
!             break;
! 
!           /* Don't use MIN_ACL_ENTRIES:  It's set to 4 on Cygwin, but Cygwin
!              returns only 3 entries for files with no ACL.  But this is safe:
!              If there are more than 4 entries, there cannot be only the
!              "user::", "group::", "other:", and "mask:" entries.  */
!           if (count > 4)
!             return 1;
! 
!           entries = (aclent_t *) malloc (count * sizeof (aclent_t));
!           if (entries == NULL)
!             {
!               errno = ENOMEM;
!               return -1;
!             }
!           if (acl (name, GETACL, count, entries) == count)
!             {
!               if (acl_nontrivial (count, entries))
!                 {
!                   free (entries);
!                   return 1;
!                 }
!               free (entries);
!               break;
!             }
!           /* Huh? The number of ACL entries changed since the last call.
!              Repeat.  */
!           free (entries);
!         }
!       }
! 
! #   ifdef ACE_GETACL
!       /* Solaris also has a different variant of ACLs, used in ZFS and NFSv4
!        file systems (whereas the other ones are used in UFS file systems).  */
!       {
!       ace_t *entries;
! 
!       for (;;)
!         {
!           count = acl (name, ACE_GETACLCNT, 0, NULL);
! 
!           if (count < 0)
!             {
!               if (errno == ENOSYS || errno == EINVAL)
!                 break;
!               else
!                 return -1;
!             }
! 
!           if (count == 0)
!             break;
! 
!           /* If there are more than 3 entries, there cannot be only the
!              ACE_OWNER, ACE_GROUP, ACE_OTHER entries.  */
!           if (count > 3)
!             return 1;
! 
!           entries = (ace_t *) malloc (count * sizeof (ace_t));
!           if (entries == NULL)
!             {
!               errno = ENOMEM;
!               return -1;
!             }
!           if (acl (name, ACE_GETACL, count, entries) == count)
!             {
!               if (acl_ace_nontrivial (count, entries))
!                 {
!                   free (entries);
!                   return 1;
!                 }
!               free (entries);
!               break;
!             }
!           /* Huh? The number of ACL entries changed since the last call.
!              Repeat.  */
!           free (entries);
!         }
!       }
! #   endif
  
+       return 0;
  #  endif
  
  # endif
*** lib/set-mode-acl.c.orig     2008-06-08 15:58:20.000000000 +0200
--- lib/set-mode-acl.c  2008-06-08 15:58:04.000000000 +0200
***************
*** 197,204 ****
    return chmod_or_fchmod (name, desc, mode);
  #  endif
  
! # elif defined ACL_NO_TRIVIAL
!   /* Solaris 10, with NFSv4 ACLs.  */
    acl_t *aclp;
    char acl_text[] = "user::---,group::---,mask:---,other:---";
  
--- 197,209 ----
    return chmod_or_fchmod (name, desc, mode);
  #  endif
  
! # elif HAVE_ACL && defined GETACLCNT /* Solaris, Cygwin, not HP-UX */
! 
! #  if defined ACL_NO_TRIVIAL
!   /* Solaris 10 (newer version), which has additional API declared in
!      <sys/acl.h> (acl_t) and implemented in libsec (acl_set, acl_trivial,
!      acl_fromtext, ...).  */
! 
    acl_t *aclp;
    char acl_text[] = "user::---,group::---,mask:---,other:---";
  
***************
*** 231,236 ****
--- 236,312 ----
  
    return chmod_or_fchmod (name, desc, mode);
  
+ #  else /* Solaris, Cygwin, general case */
+ 
+ #   ifdef ACE_GETACL
+   /* Solaris also has a different variant of ACLs, used in ZFS and NFSv4
+      file systems (whereas the other ones are used in UFS file systems).  */
+   {
+     ace_t entries[3];
+     int ret;
+ 
+     entries[0].a_type = ALLOW;
+     entries[0].a_flags = ACE_OWNER;
+     entries[0].a_who = 0; /* irrelevant */
+     entries[0].a_access_mask = (mode >> 6) & 7;
+     entries[1].a_type = ALLOW;
+     entries[1].a_flags = ACE_GROUP;
+     entries[1].a_who = 0; /* irrelevant */
+     entries[1].a_access_mask = (mode >> 3) & 7;
+     entries[2].a_type = ALLOW;
+     entries[2].a_flags = ACE_OTHER;
+     entries[2].a_who = 0;
+     entries[2].a_access_mask = mode & 7;
+ 
+     if (desc != -1)
+       ret = facl (desc, ACE_SETACL, sizeof (entries) / sizeof (aclent_t), 
entries);
+     else
+       ret = acl (name, ACE_SETACL, sizeof (entries) / sizeof (aclent_t), 
entries);
+     if (ret < 0 && errno != EINVAL && errno != ENOTSUP)
+       {
+       if (errno == ENOSYS)
+         return chmod_or_fchmod (name, desc, mode);
+       return -1;
+       }
+   }
+ #   endif
+ 
+   {
+     aclent_t entries[3];
+     int ret;
+ 
+     entries[0].a_type = USER_OBJ;
+     entries[0].a_id = 0; /* irrelevant */
+     entries[0].a_perm = (mode >> 6) & 7;
+     entries[1].a_type = GROUP_OBJ;
+     entries[1].a_id = 0; /* irrelevant */
+     entries[1].a_perm = (mode >> 3) & 7;
+     entries[2].a_type = OTHER_OBJ;
+     entries[2].a_id = 0;
+     entries[2].a_perm = mode & 7;
+ 
+     if (desc != -1)
+       ret = facl (desc, SETACL, sizeof (entries) / sizeof (aclent_t), 
entries);
+     else
+       ret = acl (name, SETACL, sizeof (entries) / sizeof (aclent_t), entries);
+     if (ret < 0)
+       {
+       if (errno == ENOSYS)
+         return chmod_or_fchmod (name, desc, mode);
+       return -1;
+       }
+   }
+   
+   if (mode & (S_ISUID | S_ISGID | S_ISVTX))
+     {
+       /* We did not call chmod so far, so the special bits have not yet
+          been set.  */
+       return chmod_or_fchmod (name, desc, mode);
+     }
+   return 0;
+ 
+ #  endif
+ 
  # else /* Unknown flavor of ACLs */
    return chmod_or_fchmod (name, desc, mode);
  # endif
*** lib/copy-acl.c.orig 2008-06-08 15:58:20.000000000 +0200
--- lib/copy-acl.c      2008-06-08 15:57:16.000000000 +0200
***************
*** 176,183 ****
  
  # endif
  
! #elif USE_ACL && defined ACL_NO_TRIVIAL
!   /* Solaris 10 NFSv4 ACLs.  */
  
    int ret;
    acl_t *aclp = NULL;
--- 176,187 ----
  
  # endif
  
! #elif USE_ACL && defined GETACL /* Solaris, Cygwin, not HP-UX */
! 
! # if defined ACL_NO_TRIVIAL
!   /* Solaris 10 (newer version), which has additional API declared in
!      <sys/acl.h> (acl_t) and implemented in libsec (acl_set, acl_trivial,
!      acl_fromtext, ...).  */
  
    int ret;
    acl_t *aclp = NULL;
***************
*** 209,214 ****
--- 213,397 ----
  
    return 0;
  
+ # else /* Solaris, Cygwin, general case */
+ 
+   /* Solaris 2.5 through Solaris 10, Cygwin, and contemporaneous versions
+      of Unixware.  The acl() call returns the access and default ACL both
+      at once.  */
+ #  ifdef ACE_GETACL
+   int ace_count;
+   ace_t *ace_entries;
+ #  endif
+   int count;
+   aclent_t *entries;
+   int did_chmod;
+   int saved_errno;
+   int ret;
+ 
+ #  ifdef ACE_GETACL
+   /* Solaris also has a different variant of ACLs, used in ZFS and NFSv4
+      file systems (whereas the other ones are used in UFS file systems).
+      There is an API
+        pathconf (name, _PC_ACL_ENABLED)
+        fpathconf (desc, _PC_ACL_ENABLED)
+      that allows to determine which of the two kinds of ACLs is supported
+      for the given file.  But some file systems may implement this call
+      incorrectly, so better not use it.
+      When fetching the source ACL, we simply fetch both ACL types.
+      When setting the destination ACL, we try either ACL types, assuming
+      that the kernel will translate the ACL from one form to the other.
+      (See in <http://docs.sun.com/app/docs/doc/819-2241/6n4huc7ia?l=en&a=view>
+      the description of ENOTSUP.)  */
+   for (;;)
+     {
+       ace_count = (source_desc != -1
+                  ? facl (source_desc, ACE_GETACLCNT, 0, NULL)
+                  : acl (src_name, ACE_GETACLCNT, 0, NULL));
+ 
+       if (ace_count < 0)
+       {
+         if (errno == ENOSYS || errno == EINVAL)
+           {
+             ace_count = 0;
+             ace_entries = NULL;
+             break;
+           }
+         else
+           return -2;
+       }
+ 
+       if (ace_count == 0)
+       {
+         ace_entries = NULL;
+         break;
+       }
+ 
+       ace_entries = (ace_t *) malloc (ace_count * sizeof (ace_t));
+       if (ace_entries == NULL)
+       {
+         errno = ENOMEM;
+         return -2;
+       }
+ 
+       if ((source_desc != -1
+          ? facl (source_desc, ACE_GETACL, ace_count, ace_entries)
+          : acl (src_name, ACE_GETACL, ace_count, ace_entries))
+         == ace_count)
+       break;
+       /* Huh? The number of ACL entries changed since the last call.
+        Repeat.  */
+     }
+ #  endif
+ 
+   for (;;)
+     {
+       count = (source_desc != -1
+              ? facl (source_desc, GETACLCNT, 0, NULL)
+              : acl (src_name, GETACLCNT, 0, NULL));
+ 
+       if (count < 0)
+       {
+         if (errno == ENOSYS || errno == ENOTSUP)
+           {
+             count = 0;
+             entries = NULL;
+             break;
+           }
+         else
+           return -2;
+       }
+ 
+       if (count == 0)
+       {
+         entries = NULL;
+         break;
+       }
+ 
+       entries = (aclent_t *) malloc (count * sizeof (aclent_t));
+       if (entries == NULL)
+       {
+         errno = ENOMEM;
+         return -2;
+       }
+ 
+       if ((source_desc != -1
+          ? facl (source_desc, GETACL, count, entries)
+          : acl (src_name, GETACL, count, entries))
+         == count)
+       break;
+       /* Huh? The number of ACL entries changed since the last call.
+        Repeat.  */
+     }
+ 
+   /* Is there an ACL of either kind?  */
+ #  ifdef ACE_GETACL
+   if (ace_count == 0)
+ #  endif
+     if (count == 0)
+       return qset_acl (dst_name, dest_desc, mode);
+ 
+   did_chmod = 0; /* set to 1 once the mode bits in 0777 have been set,
+                   set to 2 once the mode bits other than 0777 have been set */
+   saved_errno = 0; /* the first non-ignorable error code */
+ 
+   /* If both ace_entries and entries are available, try SETACL before
+      ACE_SETACL, because SETACL cannot fail with ENOTSUP whereas ACE_SETACL
+      can.  */
+ 
+   if (count > 0)
+     {
+       ret = (dest_desc != -1
+            ? facl (dest_desc, SETACL, count, entries)
+            : acl (dst_name, SETACL, count, entries));
+       if (ret < 0)
+       {
+         saved_errno = errno;
+         if (errno == ENOSYS && !acl_nontrivial (count, entries))
+           saved_errno = 0;
+       }
+       else
+       did_chmod = 1;
+     }
+   free (entries);
+ 
+ #  ifdef ACE_GETACL
+   if (ace_count > 0)
+     {
+       ret = (dest_desc != -1
+            ? facl (dest_desc, ACE_SETACL, ace_count, ace_entries)
+            : acl (dst_name, ACE_SETACL, ace_count, ace_entries));
+       if (ret < 0 && saved_errno == 0)
+       {
+         saved_errno = errno;
+         if ((errno == ENOSYS || errno == EINVAL || errno == ENOTSUP)
+             && !acl_ace_nontrivial (ace_count, ace_entries))
+           saved_errno = 0;
+       }
+     }
+   free (ace_entries);
+ #  endif
+ 
+   if (did_chmod <= ((mode & (S_ISUID | S_ISGID | S_ISVTX)) ? 1 : 0))
+     {
+       /* We did not call chmod so far, and either the mode and the ACL are
+        separate or special bits are to be set which don't fit into ACLs.  */
+ 
+       if (chmod_or_fchmod (dst_name, dest_desc, mode) != 0)
+       {
+         if (saved_errno == 0)
+           saved_errno = errno;
+       }
+     }
+ 
+   if (saved_errno)
+     {
+       errno = saved_errno;
+       return -1;
+     }
+   return 0;
+ 
+ # endif
+ 
  #else
  
    return qset_acl (dst_name, dest_desc, mode);