Re: Comparing binary files with Diff 3.2 compiled with MinGW

[Paul Eggert]

Thanks for the diffutils patch.  We're a bit better off with something
even closer to the pre-2006 approach, which took care to convert
size_t to off_t before negating it, as that is important on hosts where
sizeof (size_t) < sizeof (off_t).  Also, it's cleaner to use set_binary_mode
uniformly rather than set_binary_mode sometimes and SET_BINARY others.
Here's a revised patch that I hope catches all the issues.  I plan to
push it after Bruno's binary-io.h patches are pushed.

Use binary mode when testing for binary files.
This reverts the 2006-01-05 change and modernizes to the current API.
Idea suggested by Eli Zaretskii in:
http://lists.gnu.org/archive/html/bug-gnu-utils/2012-05/msg00066.html
* src/cmp.c (main):
* src/diff.c (main, compare_files):
Use set_binary_mode rather than SET_BINARY.
* src/diff.c (compare_files): Omit unnecessary use of O_BINARY.
* src/io.c (sip): Sample unknown files in binary mode, to see
whether they are binary.
(read_files): Read binary files in binary mode.
diff --git a/src/cmp.c b/src/cmp.c
index 1387ec1..32b25e6 100644
--- a/src/cmp.c
+++ b/src/cmp.c
@@ -293,7 +293,7 @@ main (int argc, char **argv)
        {
          file_desc[f1] = STDIN_FILENO;
          if (O_BINARY && ! isatty (STDIN_FILENO))
-           SET_BINARY (STDIN_FILENO);
+           set_binary_mode (STDIN_FILENO, O_BINARY);
        }
       else
        file_desc[f1] = open (file[f1], O_RDONLY | O_BINARY, 0);
diff --git a/src/diff.c b/src/diff.c
index b316afe..79a4726 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -524,7 +524,7 @@ main (int argc, char **argv)
 #if O_BINARY
          binary = true;
          if (! isatty (STDOUT_FILENO))
-           SET_BINARY (STDOUT_FILENO);
+           set_binary_mode (STDOUT_FILENO, O_BINARY);
 #endif
          break;

@@ -1111,8 +1111,8 @@ compare_files (struct comparison const *parent,
          else if (STREQ (cmp.file[f].name, "-"))
            {
              cmp.file[f].desc = STDIN_FILENO;
-             if (O_BINARY && binary && ! isatty (STDIN_FILENO))
-               SET_BINARY (STDIN_FILENO);
+             if (binary && ! isatty (STDIN_FILENO))
+               set_binary_mode (STDIN_FILENO, O_BINARY);
              if (fstat (STDIN_FILENO, &cmp.file[f].stat) != 0)
                cmp.file[f].desc = ERRNO_ENCODE (errno);
              else
diff --git a/src/io.c b/src/io.c
index 5a631a5..3abffb5 100644
--- a/src/io.c
+++ b/src/io.c
@@ -19,6 +19,7 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

 #include "diff.h"
+#include <binary-io.h>
 #include <cmpbuf.h>
 #include <file-type.h>
 #include <xalloc.h>
@@ -111,11 +112,24 @@ sip (struct file_data *current, bool skip_test)
        {
          /* Check first part of file to see if it's a binary file.  */

-         /* FIXME: if O_BINARY, this should revert to text mode
-            if the file is not binary.  */
-
+         int prev_mode = set_binary_mode (current->desc, O_BINARY);
+         off_t buffered;
          file_block_read (current, current->bufsize);
-         return binary_file_p (current->buffer, current->buffered);
+         buffered = current->buffered;
+
+         if (prev_mode != O_BINARY)
+           {
+             /* Revert to text mode and seek back to the start to reread
+                the file.  Use relative seek, since file descriptors
+                like stdin might not start at offset zero.  */
+             if (lseek (current->desc, - buffered, SEEK_CUR) < 0)
+               pfatal_with_name (current->name);
+             set_binary_mode (current->desc, prev_mode);
+             current->buffered = 0;
+             current->eof = false;
+           }
+
+         return binary_file_p (current->buffer, buffered);
        }
     }

@@ -761,7 +775,8 @@ read_files (struct file_data filevec[], bool pretend_binary)
     }
   if (appears_binary)
     {
-      /* FIXME: If O_BINARY, this should set both files to binary mode.  */
+      set_binary_mode (filevec[0].desc, O_BINARY);
+      set_binary_mode (filevec[1].desc, O_BINARY);
       return true;
     }