[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
Re: inotify back end for tail -f on linux
From: |
Giuseppe Scrivano |
Subject: |
Re: inotify back end for tail -f on linux |
Date: |
Mon, 01 Jun 2009 00:36:36 +0200 |
User-agent: |
Gnus/5.13 (Gnus v5.13) Emacs/23.0.94 (gnu/linux) |
Jim Meyering <address@hidden> writes:
> If we add inotify support, I'd like it to work for
> both tail-by-FD and tail-by-name.
>
> You can watch the parent directory for e.g., rename and creation events.
> Log rotation is so common, that it'd be a shame not to be able to take
> advantage of inotify when using "tail -F ...".
>
> For starters, you needn't worry about recovering from parent-dir-removal.
> I think that parent removal is rare enough in practice that we won't
> need to deal with it for some time.
>
> Eventually, the code could monitor the longest parent prefix that *does*
> exist.
> When the missing subdir reappears, monitor $parent/$subdir, etc.
> However, there's no need to do that now.
Considering only the parent directory makes things easier but I don't
think that in future it will be difficult to remove completely this
limit.
The version I attached is a bit more complex than the last one. Since
the watch descriptors are added at any time, I can't do any consideration
on their range and I am using a hash table to access the files
information in a costant time.
I need to stress my patch better but it seems already to cover all cases
you showed me.
> I haven't looked closely at the code below or even tried it,
> but here are a few superficial comments:
Thank you. I included them except this one:
>> + if (!len && errno == EINTR)
>> + continue;
>> +
>> + if (!len)
>> + error (EXIT_FAILURE, errno, _("error reading inotify event"));
>
> This function must not exit upon a failure that is specific
> to a single file. Just diagnose it and continue with any other files.
> Of course, if there are no other usable file descriptors, *then*
> you will need to exit the loop -- since this is currently used only
> when tailing-by-FD.
>
> You would not exit a similar loop when tailing by name.
I added some code to recover from errors when possible. I don't think
there are other recoverable cases but if we want to force it to re-try
read then we will need to avoid an end-less loop when there is an
unrecoverable situation. What do you think?
Regards,
Giuseppe
>From 706774fb68dde926343cc906dc627891e42504a9 Mon Sep 17 00:00:00 2001
From: Giuseppe Scrivano <address@hidden>
Date: Sat, 30 May 2009 13:31:58 +0200
Subject: [PATCH] tail: Use inotify if it is available.
* NEWS: Document the new feature
* configure.ac: Check if inotify is present.
* src/tail.c (main): Use the tail_forever inotify version if it
is possible.
(tail_forever_inotify): Added new function.
---
NEWS | 2 +
configure.ac | 2 +
src/tail.c | 247 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 249 insertions(+), 2 deletions(-)
diff --git a/NEWS b/NEWS
index 29b09a0..c5a2ef5 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,8 @@ GNU coreutils NEWS -*-
outline -*-
sort accepts a new option, --human-numeric-sort (-h): sort numbers
while honoring human readable suffixes like KiB and MB etc.
+ tail uses inotify if it is present.
+
* Noteworthy changes in release 7.4 (2009-05-07) [stable]
diff --git a/configure.ac b/configure.ac
index 4eb640e..a63ac0c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -410,6 +410,8 @@ AM_GNU_GETTEXT_VERSION([0.15])
# For a test of uniq: it uses the $LOCALE_FR envvar.
gt_LOCALE_FR
+AC_CHECK_FUNCS([inotify_init], [AC_DEFINE([HAVE_INOTIFY], [1], [Check for
libinotify])])
+
AC_CONFIG_FILES(
Makefile
doc/Makefile
diff --git a/src/tail.c b/src/tail.c
index fe34600..6335815 100644
--- a/src/tail.c
+++ b/src/tail.c
@@ -1,5 +1,5 @@
/* tail -- output the last part of file(s)
- Copyright (C) 1989, 90, 91, 1995-2006, 2008 Free Software Foundation, Inc.
+ Copyright (C) 1989, 90, 91, 1995-2006, 2008, 2009 Free Software Foundation,
Inc.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -45,6 +45,12 @@
#include "xstrtol.h"
#include "xstrtod.h"
+#ifdef HAVE_INOTIFY
+# include "hash.h"
+# include "dirname.h"
+# include <sys/inotify.h>
+#endif
+
/* The official name of this program (e.g., no `g' prefix). */
#define PROGRAM_NAME "tail"
@@ -124,6 +130,17 @@ struct File_spec
/* The value of errno seen last time we checked this file. */
int errnum;
+#ifdef HAVE_INOTIFY
+ /* The watch descriptor used by inotify. */
+ int wd;
+
+ /* The parent directory watch descriptor. It is used only
+ * when Follow_name is used. */
+ int parent_wd;
+
+ /* Offset in NAME of the basename part. */
+ size_t basename_start;
+#endif
};
/* Keep trying to open a file even if it is inaccessible when tail starts
@@ -1116,6 +1133,225 @@ tail_forever (struct File_spec *f, int nfiles, double
sleep_interval)
}
}
+#ifdef HAVE_INOTIFY
+
+static size_t
+wd_hasher (const void *entry, size_t tabsize)
+{
+ const struct File_spec *spec = entry;
+ return spec->wd % tabsize;
+}
+
+static bool
+wd_comparator (const void *e1, const void *e2)
+{
+ const struct File_spec *spec1 = e1;
+ const struct File_spec *spec2 = e2;
+ return spec1->wd == spec2->wd;
+}
+
+/* Tail NFILES files forever, or until killed.
+ Check modifications using the inotify events system. */
+static void
+tail_forever_inotify (int wd, struct File_spec *f, int nfiles)
+{
+ /* Create an index to access File_spec by its watch descriptor
+ * in costant time. */
+ struct File_spec **wd_index;
+ int i;
+ int max_realloc = 3;
+ Hash_table *wd_table;
+
+ int watched = 0;
+ size_t last;
+ size_t evlen = 0;
+ char *evbuff;
+ size_t evbuff_off = 0;
+ ssize_t len = 0;
+
+ wd_table = hash_initialize (nfiles, NULL, wd_hasher,
+ wd_comparator,
+ NULL);
+ if (! wd_table)
+ xalloc_die ();
+
+ for (i = 0; i < nfiles; i++)
+ {
+ if (!f[i].ignore)
+ {
+ size_t fnlen = strlen (f[i].name);
+ if (evlen < fnlen)
+ evlen = fnlen;
+
+ f[i].wd = 0;
+
+ if (follow_mode == Follow_name)
+ {
+ size_t dirlen = dir_len (f[i].name);
+ char prev = f[i].name[dirlen];
+ f[i].basename_start = last_component (f[i].name) - f[i].name;
+
+ f[i].name[dirlen] = '\0';
+
+ /* Do not care if the directory was already added, in this case
the
+ * same watch descriptor will be returned. */
+ f[i].parent_wd = inotify_add_watch (wd, f[i].name, IN_CREATE|
+ IN_MOVED_TO);
+
+ f[i].name[dirlen] = prev;
+
+ if (f[i].parent_wd < 0)
+ {
+ error (0, errno, _("cannot watch parent directory of %s"),
+ quote (f[i].name));
+ continue;
+ }
+ }
+
+ if (f[i].errnum != ENOENT)
+ {
+ f[i].wd = inotify_add_watch (wd, f[i].name, IN_MODIFY|
+ IN_ATTRIB|IN_DELETE_SELF|
+ IN_MOVE_SELF);
+
+ if (f[i].wd < 0)
+ {
+ error (0, errno, _("cannot watch %s"), quote (f[i].name));
+ continue;
+ }
+
+ hash_insert (wd_table, &(f[i]));
+ }
+
+ if (follow_mode == Follow_name || f[i].wd)
+ watched++;
+ }
+ }
+
+ if (!watched)
+ return;
+
+ last = f[nfiles - 1].wd;
+
+ evlen += sizeof (struct inotify_event) + 1;
+ evbuff = xmalloc (evlen);
+
+ while (1)
+ {
+ char inotify_buffer;
+ char const *name;
+ struct File_spec *fspec;
+ uintmax_t bytes_read;
+ struct stat stats;
+
+ struct inotify_event *ev;
+
+ evbuff_off += sizeof (*ev) + ev->len;
+
+ if (len <= evbuff_off)
+ {
+ len = read (wd, evbuff, evlen);
+ evbuff_off = 0;
+
+ if (len <= 0 && errno == EINVAL && max_realloc--)
+ {
+ len = 0;
+ evlen *= 2;
+ evbuff = xrealloc (evbuff, evlen);
+ continue;
+ }
+
+ if (len <= 0 && errno == EINTR)
+ {
+ len = 0;
+ continue;
+ }
+
+ if (len <= 0)
+ error (EXIT_FAILURE, errno, _("error reading inotify event"));
+ }
+
+ ev = (struct inotify_event*)(evbuff + evbuff_off);
+
+ if (ev->mask & (IN_CREATE|IN_MOVED_TO))
+ {
+ for (i = 0; i < nfiles; i++)
+ if (f[i].parent_wd == ev->wd &&
+ !strcmp (ev->name, f[i].name + f[i].basename_start))
+ break;
+
+ /* It is not a watched file. */
+ if (i == nfiles)
+ continue;
+
+ f[i].wd = inotify_add_watch (wd, f[i].name, IN_MODIFY|IN_ATTRIB|
+ IN_DELETE_SELF);
+
+ if (f[i].wd < 0)
+ {
+ error (0, errno, _("cannot watch %s"), quote (f[i].name));
+ continue;
+ }
+
+ fspec = &(f[i]);
+ hash_insert (wd_table, fspec);
+
+ recheck (&(f[i]), false);
+ }
+ else
+ {
+ struct File_spec key;
+ key.wd = ev->wd;
+ fspec = hash_lookup (wd_table, &key);
+ }
+
+ if (! fspec)
+ continue;
+
+ if (ev->mask & (IN_ATTRIB | IN_DELETE_SELF | IN_MOVE_SELF))
+ {
+ if (ev->mask & (IN_DELETE_SELF | IN_MOVE_SELF))
+ {
+ inotify_rm_watch (wd, f[i].wd);
+ hash_delete (wd_table, &(f[i]));
+ }
+
+ recheck (fspec, false);
+ continue;
+ }
+
+ name = pretty_name (fspec);
+
+ if (fstat (fspec->fd, &stats) != 0)
+ {
+ close_fd (fspec->fd, name);
+ fspec->fd = -1;
+ fspec->errnum = errno;
+ continue;
+ }
+
+ if (S_ISREG (fspec->mode) && stats.st_size < fspec->size)
+ {
+ error (0, 0, _("%s: file truncated"), name);
+ last = ev->wd;
+ xlseek (fspec->fd, stats.st_size, SEEK_SET, name);
+ fspec->size = stats.st_size;
+ }
+
+ if (ev->wd != last)
+ {
+ if (print_headers)
+ write_header (name);
+ last = ev->wd;
+ }
+
+ bytes_read = dump_remainder (name, fspec->fd, COPY_TO_EOF);
+ fspec->size += bytes_read;
+ }
+
+}
+#endif
+
/* Output the last N_BYTES bytes of file FILENAME open for reading in FD.
Return true if successful. */
@@ -1690,7 +1926,14 @@ main (int argc, char **argv)
ok &= tail_file (&F[i], n_units);
if (forever)
- tail_forever (F, n_files, sleep_interval);
+ {
+#ifdef HAVE_INOTIFY
+ int wd = inotify_init ();
+ if (wd > 0)
+ tail_forever_inotify (wd, F, n_files);
+#endif
+ tail_forever (F, n_files, sleep_interval);
+ }
if (have_read_stdin && close (STDIN_FILENO) < 0)
error (EXIT_FAILURE, errno, "-");
--
1.6.3.1