[PATCH 0/6] *** SUBJECT HERE ***

[Paolo Bonzini]

Patches 1, 2 and 6 are mine.

Patches 3 (trivial) and 4 come from Fedora.  I haven't rewritten the
fedora.sh testcases to use Jim's framework for simplicity (though I used
it later for patch 6).

Patch 5 was rewritten by me, though it originally came from Fedora.

Paolo Bonzini (6):
  fix for bug 21276
  eliminate invalid "ptr += (ptr2 - ptr1)"
  convert AUTHORS file to UTF-8
  Fedora Grep regression test suite
  fix cross-line matching in PCRE backend
  allow grep -Pz

 AUTHORS           |    2 +-
 lib/savedir.c     |   35 +++++++++++--------
 src/search.c      |   25 ++++++++++----
 tests/Makefile.am |    8 +++-
 tests/fedora.sh   |   96 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/pcre-z      |   29 ++++++++++++++++
 6 files changed, 170 insertions(+), 25 deletions(-)
 create mode 100644 tests/fedora.sh
 create mode 100755 tests/pcre-z

>From 8517712583bb30e22133a6db1106b0aca564f313 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <address@hidden>
Date: Mon, 22 Feb 2010 14:36:58 +0100
Subject: [PATCH 1/6] fix for bug 21276

* lib/savedir.c (isdir1): Use realloc instead of calloc.  Remove
dead code.
(savedir): Do not leak name_space if allocation of new_name_space fails.
---
 lib/savedir.c |   34 ++++++++++++++++++++--------------
 1 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/lib/savedir.c b/lib/savedir.c
index 55c7385..45bf9c3 100644
--- a/lib/savedir.c
+++ b/lib/savedir.c
@@ -35,30 +35,35 @@
 #include <string.h>
 #include <fnmatch.h>
 #include "savedir.h"
+#include "xalloc.h"
 
-extern int isdir (const char *path);
+static char *path;
+static size_t pathlen;
 
-char *path;
-size_t pathlen;
+extern int isdir (const char *path);
 
 static int
 isdir1 (const char *dir, const char *file)
 {
-  int status;
-  int slash;
   size_t dirlen = strlen (dir);
   size_t filelen = strlen (file);
+
+  while (dirlen && path[dirlen - 1] == '/')
+    dirlen--;
+
   if ((dirlen + filelen + 2) > pathlen)
     {
-      path = calloc (dirlen + 1 + filelen + 1, sizeof (*path));
-      pathlen = dirlen + filelen + 2;
+      pathlen *= 2;
+      if ((dirlen + filelen + 2) > pathlen)
+        pathlen = dirlen + filelen + 2;
+
+      path = xrealloc (path, pathlen);
     }
-  strcpy (path, dir);
-  slash = (path[dirlen] != '/');
+
+  memcpy (path, dir, dirlen);
   path[dirlen] = '/';
-  strcpy (path + dirlen + slash , file);
-  status  = isdir (path);
-  return status;
+  strcpy (path + dirlen + 1, file);
+  return isdir (path);
 }
 
 /* Return a freshly allocated string containing the filenames
@@ -133,7 +138,7 @@ savedir (const char *dir, off_t name_size, struct exclude 
*included_patterns,
              if (new_name_space == NULL)
                {
                  closedir (dirp);
-                 return NULL;
+                 goto fail;
                }
              namep += new_name_space - name_space;
              name_space = new_name_space;
@@ -145,8 +150,9 @@ savedir (const char *dir, off_t name_size, struct exclude 
*included_patterns,
   *namep = '\0';
   if (CLOSEDIR (dirp))
     {
+     fail:
       free (name_space);
-      return NULL;
+      name_space = NULL;
     }
   if (path)
     {
-- 
1.6.6


>From 7cc3aff2c2976f85e57c134123a81a618eb5e452 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <address@hidden>
Date: Fri, 5 Mar 2010 11:51:09 +0100
Subject: [PATCH 2/6] eliminate invalid "ptr += (ptr2 - ptr1)"

* lib/savedir.c (savedir): new_name_space and name_space do not point into
the same object, so computing their difference is invalid.  Similarly,
summing the difference to namep is invalid because namep and the result
point into different objects.  Avoid this.
---
 lib/savedir.c |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/lib/savedir.c b/lib/savedir.c
index 45bf9c3..91fd77b 100644
--- a/lib/savedir.c
+++ b/lib/savedir.c
@@ -140,7 +140,7 @@ savedir (const char *dir, off_t name_size, struct exclude 
*included_patterns,
                  closedir (dirp);
                  goto fail;
                }
-             namep += new_name_space - name_space;
+             namep = new_name_space + (namep - name_space);
              name_space = new_name_space;
            }
          strcpy (namep, dp->d_name);
-- 
1.6.6


>From 201b4117ac6a5d92cf1de5dbecba6a47d3c634c8 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <address@hidden>
Date: Thu, 4 Mar 2010 18:06:22 +0100
Subject: [PATCH 3/6] convert AUTHORS file to UTF-8

* AUTHORS: Convert to UTF-8.
---
 AUTHORS |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index 3d7ab74..17ebade 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -46,7 +46,7 @@ send me email.
 
 Alain Magloire maintained GNU grep until version 2.5e.
 
-Bernhard "Bero" Rosenkränzer <address@hidden> maintained GNU grep until
+Bernhard "Bero" Rosenkränzer <address@hidden> maintained GNU grep until
 version 2.5.1, ie. from Sep 2001 till 2003.
 
 Stepan Kasal <address@hidden> maintained GNU grep since Feb 2004.
-- 
1.6.6


>From c7eb55731a1b266e6415d87b967d81fc9f145d41 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <address@hidden>
Date: Sun, 23 Nov 2008 17:24:13 +0100
Subject: [PATCH 4/6] Fedora Grep regression test suite

* tests/Makefile.am (TESTS): Add fedora.sh.
(CLEANFILES): Add several new files.
* tests/fedora.sh: New file, originally by Lubomir Rintel but somewhat
rewritten to avoid bashisms.
---
 tests/Makefile.am |    7 +++-
 tests/fedora.sh   |   88 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+), 2 deletions(-)
 create mode 100644 tests/fedora.sh

diff --git a/tests/Makefile.am b/tests/Makefile.am
index a34c38d..ea108e4 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -30,7 +30,8 @@ TESTS =               \
   spencer1.sh  \
   status.sh    \
   warning.sh   \
-  yesno.sh
+  yesno.sh     \
+  fedora.sh
 
 EXTRA_DIST =   \
   $(TESTS)     \
@@ -46,7 +47,9 @@ EXTRA_DIST =  \
 
 CLEANFILES = \
   spencer1.script bre.script ere.script khadafy.out patfile \
-  cspatfile csinput yesno.txt
+  cspatfile csinput yesno.txt 112869.out 116909.in 116909.list \
+  116909.out 123362.out 161700.out 171379.out 179698.out \
+  204255.first 204255.second
 
 TESTS_ENVIRONMENT =                            \
   tmp__=$$TMPDIR; test -d "$$tmp__" || tmp__=.;        \
diff --git a/tests/fedora.sh b/tests/fedora.sh
new file mode 100644
index 0000000..d08bfce
--- /dev/null
+++ b/tests/fedora.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# GREP Regression test suite for Fedora bugs and fixes
+# (c) 2008 Lubomir Rintel
+# Licensed under the same terms as GNU Grep itself
+
+if [ -t 1 ]
+then
+       # Colored output on terimal
+       G='\033[32m'
+       R='\033[31m'
+       D='\033[0m'
+fi
+
+ok ()  { echo -e "${G}OK${D}"; }
+fail () { echo -e "${R}FAIL${D} (See ${U})"; failures=1; }
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=116909
+echo -n "fgrep false negatives: "
+cat > 116909.list <<EOF
+a
+b
+c
+EOF
+cat > 116909.in <<EOF
+a
+barn
+c
+EOF
+cat > 116909.out <<EOF
+a
+c
+EOF
+${GREP} -F -w -f 116909.list 116909.in | diff - 116909.out && ok || fail
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=123362
+echo -n "bad handling of brackets in UTF-8: "
+echo Y > 123362.out
+echo Y | LC_ALL=de_DE.UTF-8 ${GREP} -i '[y,Y]' | diff - 123362.out && ok | fail
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=112869
+echo -n "crash with \W: "
+echo '<form>' > 112869.out
+LANG=it_IT ${GREP} -iE '\Wform\W' 112869.out | diff - 112869.out && ok || fail
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=189580
+echo -n "grep -D skip opening a special file: "
+${GREP} -D skip foo /dev/zero &
+sleep 1
+kill $! 2>/dev/null && fail || ok
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=169524
+echo -n "grep -Fw looping infinitely: "
+echo foobar | ${GREP} -Fw "" &
+sleep 1
+kill $! 2>/dev/null && fail || ok
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=140781
+echo -n "fgrep hangs on binary files: "
+${GREP} -F grep $(which ${GREP}) >/dev/null &
+sleep 1
+kill $! 2>/dev/null && fail || ok
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=161700
+echo -n "grep -Fw fails to match anything: "
+echo test > 161700.out
+${GREP} -Fw test 161700.out | diff - 161700.out && ok || fail
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=179698
+echo -n "grep -w broken in non-utf8 multibyte locales: "
+echo za a > 179698.out
+LANG=ja_JP.eucjp ${GREP} -w a 179698.out | diff - 179698.out && ok || fail
+
+# Skip the rest of tests in compiled without PCRE
+echo a |grep -P a >/dev/null || exit $failures
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=171379
+echo -n "grep -P crashes on whitespace lines: "
+echo '   ' > 171379.out
+${GREP} -P '^\s+$' 171379.out | diff - 171379.out && ok || fail
+
+U=https://bugzilla.redhat.com/show_bug.cgi?id=204255
+echo -n "-e '' does not work if not a first parameter: "
+echo test | grep -e 'HighlightThis' -e '' > 204255.first
+echo test | grep -e '' -e 'HighlightThis' > 204255.second
+diff 204255.first 204255.second && ok || fail
+
+exit $failures
-- 
1.6.6


>From f3b8db2e230c6b9ad8a7538382f55ac8f117e926 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <address@hidden>
Date: Thu, 4 Mar 2010 18:18:42 +0100
Subject: [PATCH 5/6] fix cross-line matching in PCRE backend

* search.c (Pexecute): Split the buffer in lines and match each line
separately.
* tests/fedora.sh: Add regression testsuite.
---
        I am pretty sure the start_ptr/start_ofs logic is not covered
        by the testsuite.  It was surely incorrect in the original
        Fedora patch.

 src/search.c    |   31 ++++++++++++++++++++++++++-----
 tests/fedora.sh |    8 ++++++++
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/src/search.c b/src/search.c
index cf39e0d..9d7ed98 100644
--- a/src/search.c
+++ b/src/search.c
@@ -738,9 +738,30 @@ EXECUTE_FCT(Pexecute)
      is just for performance improvement in pcre_exec.  */
   int sub[300];
 
-  int e = pcre_exec (cre, extra, buf, size,
-                    start_ptr ? (start_ptr - buf) : 0, 0,
-                    sub, sizeof sub / sizeof *sub);
+  const char *line_buf, *line_end, *line_next;
+  int e = PCRE_ERROR_NOMATCH;
+  ptrdiff_t start_ofs = start_ptr ? start_ptr - buf : 0;
+
+  /* PCRE can't limit the matching to single lines, therefore we have to
+     match each line in the buffer separately.  */
+  for (line_next = buf;
+       e == PCRE_ERROR_NOMATCH && line_next < buf + size; 
+       start_ofs -= line_next - line_buf)
+    {
+      line_buf = line_next;
+      line_end = memchr (line_buf, eolbyte, (buf + size) - line_buf);
+      if (line_end == NULL)
+        line_next = line_end = buf + size;
+      else
+        line_next = line_end + 1;
+
+      if (start_ptr && start_ptr >= line_end)
+        continue;
+
+      e = pcre_exec (cre, extra, line_buf, line_end - line_buf,
+                     start_ofs < 0 ? 0 : start_ofs, 0,
+                     sub, sizeof sub / sizeof *sub);
+    }
 
   if (e <= 0)
     {
@@ -759,8 +780,8 @@ EXECUTE_FCT(Pexecute)
   else
     {
       /* Narrow down to the line we've found.  */
-      char const *beg = buf + sub[0];
-      char const *end = buf + sub[1];
+      char const *beg = line_buf + sub[0];
+      char const *end = line_buf + sub[1];
       char const *buflim = buf + size;
       char eol = eolbyte;
       if (!start_ptr)
diff --git a/tests/fedora.sh b/tests/fedora.sh
index d08bfce..a179271 100644
--- a/tests/fedora.sh
+++ b/tests/fedora.sh
@@ -85,4 +85,12 @@ echo test | grep -e 'HighlightThis' -e '' > 204255.first
 echo test | grep -e '' -e 'HighlightThis' > 204255.second
 diff 204255.first 204255.second && ok || fail
 
+U=https://bugzilla.redhat.com/show_bug.cgi?id=324781
+echo -n "bad handling of line breaks with grep -P #1: "
+echo -ne "a\na" | ${GREP} -P '[^a]' >/dev/null && fail || ok
+
+# This is mostly a check that fix for above doesn't break -P further
+echo -n "bad handling of line breaks with grep -P #2: "
+echo -ne "a\na" | ${GREP} -P '[^b].[^b]' >/dev/null && fail || ok
+
 exit $failures
-- 
1.6.6


>From c382050feb74672151e9d5331f3cb5d4bccbbab0 Mon Sep 17 00:00:00 2001
From: Paolo Bonzini <address@hidden>
Date: Fri, 5 Mar 2010 11:42:41 +0100
Subject: [PATCH 6/6] allow grep -Pz

* src/search.c (Pcompile): Remove restriction on grep -Pz.
* tests/pcre-z: New.
* tests/Makefile.am (TESTS): Add pcre-z.
---
 src/search.c      |    2 --
 tests/Makefile.am |    3 ++-
 tests/pcre-z      |   29 +++++++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 3 deletions(-)
 create mode 100755 tests/pcre-z

diff --git a/src/search.c b/src/search.c
index 9d7ed98..c4e5149 100644
--- a/src/search.c
+++ b/src/search.c
@@ -677,8 +677,6 @@ COMPILE_FCT(Pcompile)
   char const *pnul;
 
   /* FIXME: Remove these restrictions.  */
-  if (eolbyte != '\n')
-    error (EXIT_TROUBLE, 0, _("The -P and -z options cannot be combined"));
   if (memchr(pattern, '\n', size))
     error (EXIT_TROUBLE, 0, _("The -P option only supports a single pattern"));
 
diff --git a/tests/Makefile.am b/tests/Makefile.am
index ea108e4..4b08abc 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -31,7 +31,8 @@ TESTS =               \
   status.sh    \
   warning.sh   \
   yesno.sh     \
-  fedora.sh
+  fedora.sh    \
+  pcre-z
 
 EXTRA_DIST =   \
   $(TESTS)     \
diff --git a/tests/pcre-z b/tests/pcre-z
new file mode 100755
index 0000000..967ecad
--- /dev/null
+++ b/tests/pcre-z
@@ -0,0 +1,29 @@
+#!/bin/sh
+# Test Perl regex with NUL-separated input
+: ${srcdir=.}
+. "$srcdir/init.sh"; path_prepend_ ../src
+
+echo a | grep -P a >/dev/null 2>err || skip_
+compare err /dev/null || fail_
+
+REGEX=a
+
+echo 'abc
+0def
+0ghi
+0aaa
+0gah' | tr 0 \\0 > in
+
+grep -z "$REGEX" in > exp 2>err || fail_
+compare err /dev/null || fail_
+
+# Sanity check the output
+test `grep -cz "$REGEX" in` 2>err = 3 || fail_
+compare err /dev/null || fail_
+
+fail=0
+grep -Pz "$REGEX" in > out 2>err || fail=1
+compare out exp || fail=1
+compare err /dev/null || fail=1
+
+Exit $fail
-- 
1.6.6