Re: Run-time dynamic linking in grep

[Jim Meyering]

Reuben Thomas wrote:
> New patch attached, with support for zlib and libbz2, and a test, and
> all relevant blurbs (I hope), and a test that passes, along with make
> distcheck.
>
> .gz and .bz2 still account for the vast majority of compressed files I
> come across. Also, unlike zlib and libbz2, liblzma does not currently
> provide a file operations interface that makes it easy to use for
> transparent decompression (I am using zlib's and libbz2's veneers for
> fdopen, close and read); on the other hand, there's the following
> paragraph on the xz home page:
>
> While liblzma has a zlib-like API, liblzma doesn't include any file
> I/O functions. A separate I/O library is planned, which would abstract
> handling of .gz, .bz2, and .xz files with an easy to use API.
>
> Since I have implemented transparent decompression as a veneer
> library, I wonder therefore whether it might not be a good idea to
> polish it to be suitable for inclusion in xz, and I thought of
> approaching the xz maintainers to ask their opinion; maybe we can kill
> two birds with one stone, and get transparent decompression for grep
> and the multi-library veneer in liblzma. Advice gratefully received.
>
> I also suggest that both for easier support for many more formats and
> to take Paul Eggert's suggestion into account I implement
> decompression via a filter, as zutils does. But I thought I'd release
> first.

Sooner is always better.  Thanks.

> One comment about the documentation: grep.texi has different
> sectioning for the flags from grep --help; perhaps this should be
> fixed?

If you think one is obviously better than the other,
please explain why along with a patch to improve things.

[This is just a quick, first-pass review.
 I don't have time for more right now. ]

I applied your patch and got a warning from git am about trailing spaces.
An attempt to build (configure with --enable-gcc-warnings) failed
due to this always-false comparison: if (zfd->fd.plain < 0 || ty < 0)
(ty < 0), since ty is unsigned (an enum).  Also, "dfd" was unused.

Also, "make syntax-check" failed due to a useless <assert.h>
and I adjusted the formatting of function definitions whose names
did not start in the first column.

I've attached the incremental, followed by full format-patch output,
with that merged in.  Please apply that to a clean check-out
(via git am) and base any subsequent changes on that.

Most importantly, non-seekable (pipe) input is not decompressed:
(that "null" looks suspicious, too)

    $ echo foo|gzip|src/grep --decompress o
    src/grep: (null): Illegal seek

It would sure be nice to fix that.

Also, the test didn't do anything, so I fixed it for you,
but you should also make it exercise the bzip2-related code.
I also tweaked spacing in your log message.

diff --git a/lib/zio.c b/lib/zio.c
index 95cddd7..5e6e975 100644
--- a/lib/zio.c
+++ b/lib/zio.c
@@ -18,7 +18,6 @@

 #include <config.h>

-#include <assert.h>
 #include <stdlib.h>
 #include <sys/types.h>
 #include <stdio.h>
@@ -84,7 +83,8 @@ static const unsigned char * const magic[] = {
 };
 #define MAX_MAGIC 5 /* Length in bytes of longest magic. */

-static enum zfile_type zfd_identify (int fd)
+static enum zfile_type
+zfd_identify (int fd)
 {
   char buf[MAX_MAGIC];
   unsigned i = ZIO_PLAIN;
@@ -102,7 +102,8 @@ static enum zfile_type zfd_identify (int fd)
                automatically gives ZIO_PLAIN */
 }

-static const char * mode_to_string (int mode)
+static const char *
+mode_to_string (int mode)
 {
   switch (mode & (O_ACCMODE | O_APPEND))
     {
@@ -126,7 +127,6 @@ static const char * mode_to_string (int mode)
 ZIO_FILE
 zio_new_raw (int fd)
 {
-  int dfd;
   ZIO_FILE zfd = XMALLOC (struct zfile);

   zfd->ty = ZIO_PLAIN;
@@ -142,7 +142,7 @@ zio_new (int fd)
   enum zfile_type ty = zfd_identify (fd);
   const char *mode;

-  if (zfd->fd.plain < 0 || ty < 0)
+  if (zfd->fd.plain < 0 || ty == -1)
     {
       free (zfd);
       return NULL;
@@ -157,7 +157,7 @@ zio_new (int fd)
         case ZIO_ZLIB:
           {
             gzFile fd_zlib = gzdopen (zfd->fd.plain, mode);
-            
+
             if (fd_zlib == NULL)
               break;

diff --git a/tests/decompress b/tests/decompress
index 12ea844..387cdbe 100644
--- a/tests/decompress
+++ b/tests/decompress
@@ -6,5 +6,9 @@
 LC_ALL=C
 export LC_ALL

-echo grep 'easter' $abs_top_srcdir/tests/compressed.gz > /dev/null
-Exit $?
+echo easter | gzip > compressed.gz || framework_failure_
+
+fail=0
+grep --decompress easter compressed.gz || fail=1
+
+Exit $fail


>From 331804557bcbe8fc7a96b4bd256b4bfb901930fb Mon Sep 17 00:00:00 2001
From: Reuben Thomas <address@hidden>
Date: Fri, 21 Jan 2011 13:21:32 +0000
Subject: [PATCH] grep: add --decompress flag for transparent decompression

* configure.ac: Add support for zlib and libbz2, including
--disable-{zlib,libbz2}.
* src/main.c: Add --decompress option.
* lib/zio.{c,h}: A new library module which provides open/close/read
veneers for compressed file formats. Starts with support for zlib and
libbz2.
* lib/Makefile.am: Add zio.{c,h}
* doc/grep.texi: Document --decompress.
* tests/decompress: New test.
* tests/Makefile.am: Add new test.
* NEWS: Add blurb for --decompress.
---
 .gitignore        |    2 +
 NEWS              |    6 ++
 configure.ac      |   27 ++++++
 doc/grep.texi     |    7 ++
 lib/Makefile.am   |    2 +-
 lib/zio.c         |  245 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 lib/zio.h         |   28 ++++++
 src/main.c        |   54 +++++++++++-
 tests/Makefile.am |    3 +-
 tests/decompress  |   14 +++
 10 files changed, 382 insertions(+), 6 deletions(-)
 create mode 100644 lib/zio.c
 create mode 100644 lib/zio.h
 create mode 100644 tests/decompress

diff --git a/.gitignore b/.gitignore
index 18f58aa..4f82be9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,8 @@
 !/lib/Makefile.am
 !/lib/savedir.c
 !/lib/savedir.h
+!/lib/zio.c
+!/lib/zio.h
 !/m4/djgpp.m4
 *.a
 *.exe
diff --git a/NEWS b/NEWS
index 9e9974a..b20315a 100644
--- a/NEWS
+++ b/NEWS
@@ -12,6 +12,12 @@ GNU grep NEWS                                    -*- outline 
-*-
   grep erroneously returned with exit status 1 on some memory allocation
   failure. [bug present since "the beginning"]

+** New features
+
+  grep now has a --decompress flag that enables transparent
+  decompression of supported compressed formats. Initial support for
+  zlib (gzip) and libbz2 (bzip2) is implemented.
+
 * Noteworthy changes in release 2.7 (2010-09-16) [stable]

 ** Bug fixes
diff --git a/configure.ac b/configure.ac
index aca529a..8b282df 100644
--- a/configure.ac
+++ b/configure.ac
@@ -73,6 +73,23 @@ AM_SILENT_RULES([yes]) # make --enable-silent-rules the 
default.

 AC_CONFIG_HEADERS([config.h:config.hin])

+dnl Check for arguments
+AC_ARG_ENABLE(zlib,
+ [  --disable-zlib          disable zlib],
+ [case "${enableval}" in
+  yes) testzlib=yes ;;
+  no)  testzlib=no ;;
+  *)   AC_MSG_ERROR(bad value ${enableval} for --disable-zlib) ;;
+ esac],[testzlib=yes])
+
+AC_ARG_ENABLE(libbz2,
+ [  --disable-libbz2        disable libbz2],
+ [case "${enableval}" in
+  yes) testlibbz2=yes ;;
+  no)  testlibbz2=no ;;
+  *)   AC_MSG_ERROR(bad value ${enableval} for --disable-libbz2) ;;
+ esac],[testlibbz2=yes])
+
 dnl Checks for programs.
 AC_CANONICAL_HOST
 AC_PROG_AWK
@@ -199,6 +216,16 @@ fi

 gl_FUNC_PCRE

+# Check for zlib
+if test x"$testzlib" = x"yes"; then
+  AC_CHECK_HEADER(zlib.h, [AC_CHECK_LIB(z, gzdopen)],)
+fi
+
+# Check for libbz2
+if test x"$testlibbz2" = x"yes"; then
+  AC_CHECK_HEADER(bzlib.h, [AC_CHECK_LIB(bz2, BZ2_bzdopen)],)
+fi
+
 AC_CONFIG_FILES([
   Makefile
   lib/Makefile
diff --git a/doc/grep.texi b/doc/grep.texi
index da27aa1..ca31e68 100644
--- a/doc/grep.texi
+++ b/doc/grep.texi
@@ -577,6 +577,13 @@ which can have nasty side effects
 if the output is a terminal and
 if the terminal driver interprets some of it as commands.

address@hidden --decompress
address@hidden --decompress
address@hidden compressed files
+If the first few bytes of a file indicate that it contains compressed
+data, match on the decompressed data.  Support for gzip and bzip2 files
+is implemented.
+
 @item -D @var{action}
 @itemx address@hidden
 @opindex -D
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 3d0c1ba..8e67922 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -21,7 +21,7 @@ INCLUDES = -I.. -I$(srcdir)
 AM_CFLAGS += $(GNULIB_WARN_CFLAGS) $(WERROR_CFLAGS)

 libgreputils_a_SOURCES += \
-  savedir.c savedir.h
+  savedir.c savedir.h zio.c zio.h

 libgreputils_a_LIBADD += $(LIBOBJS) $(ALLOCA)
 libgreputils_a_DEPENDENCIES += $(LIBOBJS) $(ALLOCA)
diff --git a/lib/zio.c b/lib/zio.c
new file mode 100644
index 0000000..5e6e975
--- /dev/null
+++ b/lib/zio.c
@@ -0,0 +1,245 @@
+/* zio.c - compression library wrapper for input only.
+   Copyright (C) 2011 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
+   02110-1301, USA.  */
+
+#include <config.h>
+
+#include <stdlib.h>
+#include <sys/types.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include "xalloc.h"
+
+#include "zio.h"
+
+#ifdef HAVE_LIBZ
+# include <zlib.h>
+#endif
+
+#ifdef HAVE_LIBBZ2
+# include <bzlib.h>
+#endif
+
+/* Compressed file types. */
+enum zfile_type {
+  ZIO_ZLIB,
+  ZIO_BZ2,
+  ZIO_LZ,
+  ZIO_XZ,
+  ZIO_PLAIN
+};
+
+struct zfile
+{
+  enum zfile_type ty;
+  union
+  {
+    int plain;
+#ifdef HAVE_LIBZ
+    gzFile zlib;
+#endif
+#ifdef HAVE_LIBBZ2
+    BZFILE *bz2;
+#endif
+#ifdef HAVE_LIBLZMA
+#error "LZMA support not implemented yet, sorry!"
+#endif
+  } fd;
+};
+
+/* File extensions and magic headers; keep in same order as enum
+   zfile_type in zinput.h */
+static const char * const magic_extension[] = {
+   ".gz",
+   ".bz2",
+   ".lz",
+   ".xz",
+   ""
+};
+
+static const unsigned char * const magic[] = {
+  "\x1F\x8B",
+  "BZh",
+  "LZIP",
+  "\xFD" "7zXZ",
+  ""
+};
+#define MAX_MAGIC 5 /* Length in bytes of longest magic. */
+
+static enum zfile_type
+zfd_identify (int fd)
+{
+  char buf[MAX_MAGIC];
+  unsigned i = ZIO_PLAIN;
+
+  if (read (fd, buf, MAX_MAGIC) == MAX_MAGIC)
+    /* Assume that no valid file can be this short. */
+    for (i = 0; i < sizeof (magic) / sizeof (magic[0]) - 1; i++)
+      if (memcmp (buf, magic[i], strlen (magic[i])) == 0)
+        break;
+
+  if (lseek (fd, -MAX_MAGIC, SEEK_CUR) < 0)
+    return -1;
+
+  return i; /* magic arrays match enum zfile_type; unrecognised file
+               automatically gives ZIO_PLAIN */
+}
+
+static const char *
+mode_to_string (int mode)
+{
+  switch (mode & (O_ACCMODE | O_APPEND))
+    {
+    case O_RDONLY:
+      return "rb";
+    case O_WRONLY:
+      return "wb";
+    case O_RDWR:
+      return "r+b";
+    case O_RDONLY | O_APPEND:
+      return "a+b";
+    case O_WRONLY | O_APPEND:
+      return "ab";
+    case O_RDWR | O_APPEND:
+      return "a+b";
+    }
+
+  return NULL; /* Invalid mode */
+}
+
+ZIO_FILE
+zio_new_raw (int fd)
+{
+  ZIO_FILE zfd = XMALLOC (struct zfile);
+
+  zfd->ty = ZIO_PLAIN;
+  zfd->fd.plain = fd;
+
+  return zfd;
+}
+
+ZIO_FILE
+zio_new (int fd)
+{
+  ZIO_FILE zfd = zio_new_raw (fd);
+  enum zfile_type ty = zfd_identify (fd);
+  const char *mode;
+
+  if (zfd->fd.plain < 0 || ty == -1)
+    {
+      free (zfd);
+      return NULL;
+    }
+
+  mode = mode_to_string (fcntl (zfd->fd.plain, F_GETFL));
+  if (mode)
+    {
+      switch (ty)
+        {
+#ifdef HAVE_LIBZ
+        case ZIO_ZLIB:
+          {
+            gzFile fd_zlib = gzdopen (zfd->fd.plain, mode);
+
+            if (fd_zlib == NULL)
+              break;
+
+            zfd->ty = ZIO_ZLIB;
+            zfd->fd.zlib = fd_zlib;
+            return zfd;
+          }
+#endif
+
+#ifdef HAVE_LIBBZ2
+        case ZIO_BZ2:
+          {
+            BZFILE *fd_bz2 = BZ2_bzdopen (zfd->fd.plain, mode);
+
+            if (fd_bz2 == NULL)
+              break;
+
+            zfd->ty = ZIO_BZ2;
+            zfd->fd.bz2 = fd_bz2;
+            return zfd;
+          }
+#endif
+
+        case ZIO_PLAIN:
+        default: /* We found a known compressed type, but it's unimplemented */
+          return zfd;
+        }
+    }
+
+  close (zfd->fd.plain);
+  free (zfd);
+  return NULL;
+}
+
+int
+zio_close (ZIO_FILE zfd)
+{
+  int ret = 0;
+
+  switch (zfd->ty)
+    {
+    case ZIO_PLAIN:
+      ret = close (zfd->fd.plain);
+      break;
+#ifdef HAVE_LIBZ
+    case ZIO_ZLIB:
+      ret = gzclose (zfd->fd.zlib);
+      break;
+#endif
+#ifdef HAVE_LIBBZ2
+    case ZIO_BZ2:
+      BZ2_bzclose (zfd->fd.bz2);
+      break;
+#endif
+    default:
+      ret = -1;
+      break;
+    }
+
+  if (ret == 0)
+    free (zfd);
+  return ret;
+}
+
+ssize_t
+zio_read (ZIO_FILE zfd, void *buf, size_t count)
+{
+  switch (zfd->ty)
+    {
+    case ZIO_PLAIN:
+      return read (zfd->fd.plain, buf, count);
+#ifdef HAVE_LIBZ
+    case ZIO_ZLIB:
+      return gzread (zfd->fd.zlib, buf, count);
+#endif
+#ifdef HAVE_LIBBZ2
+    case ZIO_BZ2:
+      return BZ2_bzread (zfd->fd.bz2, buf, count);
+#endif
+    default:
+      return -1;
+    }
+
+  abort (); /* Should not get here. */
+}
diff --git a/lib/zio.h b/lib/zio.h
new file mode 100644
index 0000000..c2e8794
--- /dev/null
+++ b/lib/zio.h
@@ -0,0 +1,28 @@
+/* zinput.h - interface to compression library wrapper for input only.
+   Copyright (C) 2011 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
+   02110-1301, USA.  */
+
+#include <config.h>
+
+#include <sys/types.h>
+
+typedef struct zfile *ZIO_FILE;
+
+ZIO_FILE zio_new_raw (int fd);
+ZIO_FILE zio_new (int fd);
+int zio_close (ZIO_FILE zfd);
+ssize_t zio_read (ZIO_FILE zfd, void *buf, size_t count);
diff --git a/src/main.c b/src/main.c
index f964adb..013f5bb 100644
--- a/src/main.c
+++ b/src/main.c
@@ -48,6 +48,7 @@
 #include "version-etc.h"
 #include "xalloc.h"
 #include "xstrtol.h"
+#include "zio.h"

 #define SEP_CHAR_SELECTED ':'
 #define SEP_CHAR_REJECTED '-'
@@ -83,6 +84,9 @@ static int color_option;
 /* If nonzero, show only the part of a line matching the expression. */
 static int only_matching;

+/* If nonzero, auto-decompress the input. */
+static int do_decompress;
+
 /* If nonzero, make sure first content char in a line is on a tab stop. */
 static int align_tabs;

@@ -286,7 +290,8 @@ enum
   LABEL_OPTION,
   EXCLUDE_DIRECTORY_OPTION,
   GROUP_SEPARATOR_OPTION,
-  MMAP_OPTION
+  MMAP_OPTION,
+  DECOMPRESS_OPTION
 };

 /* Long options equivalences. */
@@ -305,6 +310,7 @@ static struct option const long_options[] =
   {"color", optional_argument, NULL, COLOR_OPTION},
   {"colour", optional_argument, NULL, COLOR_OPTION},
   {"count", no_argument, NULL, 'c'},
+  {"decompress", no_argument, NULL, DECOMPRESS_OPTION},
   {"devices", required_argument, NULL, 'D'},
   {"directories", required_argument, NULL, 'd'},
   {"exclude", required_argument, NULL, EXCLUDE_OPTION},
@@ -428,6 +434,7 @@ static char *buffer;                /* Base of buffer. */
 static size_t bufalloc;                /* Allocated buffer size, counting 
slop. */
 #define INITIAL_BUFSIZE 32768  /* Initial buffer size, not counting slop. */
 static int bufdesc;            /* File descriptor. */
+static ZIO_FILE zbufdesc;      /* Compressed file descriptor. */
 static char *bufbeg;           /* Beginning of user-visible stuff. */
 static char *buflim;           /* Limit of user-visible stuff. */
 static size_t pagesize;                /* alignment of memory pages */
@@ -549,7 +556,7 @@ fillbuf (size_t save, struct stats const *stats)
   if (! fillsize)
     {
       ssize_t bytesread;
-      while ((bytesread = read (bufdesc, readbuf, readsize)) < 0
+      while ((bytesread = zio_read (zbufdesc, readbuf, readsize)) < 0
              && errno == EINTR)
         continue;
       if (bytesread < 0)
@@ -1090,18 +1097,36 @@ grep (int fd, char const *file, struct stats *stats)
   residue = 0;
   save = 0;

+  if (do_decompress)
+    {
+      int dupdesc = dup (bufdesc);
+      if (dupdesc < 0)
+        {
+          error (0, errno, "%s", file);
+          return 0;
+        }
+      zbufdesc = zio_new (dupdesc);
+    }
+  else
+    zbufdesc = zio_new_raw (bufdesc);
+  if (zbufdesc == NULL)
+    {
+      error (0, errno, "%s", file);
+      return 0;
+    }
+
   if (! fillbuf (save, stats))
     {
       if (! is_EISDIR (errno, file))
         suppressible_error (filename, errno);
-      return 0;
+      goto abort_grep;
     }

   not_text = (((binary_files == BINARY_BINARY_FILES && !out_quiet)
                || binary_files == WITHOUT_MATCH_BINARY_FILES)
               && memchr (bufbeg, eol ? '\0' : '\200', buflim - bufbeg));
   if (not_text && binary_files == WITHOUT_MATCH_BINARY_FILES)
-    return 0;
+    goto abort_grep;
   done_on_match += not_text;
   out_quiet += not_text;

@@ -1183,6 +1208,22 @@ grep (int fd, char const *file, struct stats *stats)
   out_quiet -= not_text;
   if ((not_text & ~out_quiet) && nlines != 0)
     printf (_("Binary file %s matches\n"), filename);
+
+ abort_grep:
+  if (do_decompress)
+    {
+      while (zio_close (zbufdesc) != 0)
+        if (errno != EINTR)
+          {
+            error (0, errno, "%s", file);
+            break;
+          }
+    }
+  else
+    {
+      free (zbufdesc);
+    }
+
   return nlines;
 }

@@ -1432,6 +1473,7 @@ Output control:\n\
   -a, --text                equivalent to --binary-files=text\n\
 "));
       printf (_("\
+      --decompress          transparently decompress compressed files\n\
   -I                        equivalent to --binary-files=without-match\n\
   -d, --directories=ACTION  how to handle directories;\n\
                             ACTION is `read', `recurse', or `skip'\n\
@@ -2036,6 +2078,10 @@ main (int argc, char **argv)
           }
         break;

+      case DECOMPRESS_OPTION:
+        do_decompress = 1;
+        break;
+
       case EXCLUDE_OPTION:
         if (!excluded_patterns)
           excluded_patterns = new_exclude ();
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 7233c01..ba060cf 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -45,6 +45,7 @@ TESTS =                                               \
   case-fold-char-range                         \
   case-fold-char-type                          \
   char-class-multibyte                         \
+  decompress                                   \
   dfaexec-multibyte                            \
   empty                                                \
   equiv-classes                                 \
@@ -104,7 +105,7 @@ TESTS_ENVIRONMENT =                         \
     if grep '^\#!/usr/bin/perl' "$$1" > /dev/null; then                        
\
       if $(PERL) -e 'use warnings' > /dev/null 2>&1; then              \
        grep '^\#!/usr/bin/perl -T' "$$1" > /dev/null && T_=T || T_=;   \
-        $(PERL) -w$$T_ -I$(srcdir) -MCoreutils                         \
+       $(PERL) -w$$T_ -I$(srcdir) -MCoreutils                          \
              -M"CuTmpdir qw($$f)" -- "$$1";    \
       else                                     \
        echo 1>&2 "$$tst: configure did not find a usable version of Perl," \
diff --git a/tests/decompress b/tests/decompress
new file mode 100644
index 0000000..387cdbe
--- /dev/null
+++ b/tests/decompress
@@ -0,0 +1,14 @@
+#!/bin/sh
+# Test that transparent decompression works.
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+
+LC_ALL=C
+export LC_ALL
+
+echo easter | gzip > compressed.gz || framework_failure_
+
+fail=0
+grep --decompress easter compressed.gz || fail=1
+
+Exit $fail
-- 
1.7.3.5.38.gb312b