Re: [Bug-tar] mtime in x header

[Michael D. Adams]

Thanks.  I've compiled the git version and it seems to meet my needs.

On Wed, Oct 7, 2009 at 2:23 PM, Sergey Poznyakoff <address@hidden> wrote:
> Hi Michael,
>
> Following our discussion, I've applied the attached patches to
> the Git master. Their two main effects are:
>
> 1. Modification times in the ustar header blocks for the
> extended headers are set to the mtimes of the corresponding archive
> members. Thus, to ensure binary equivalence of the two PAX archives
> with the same contents, the following option is sufficient:
>
>  --pax-option=exthdr.name=%d/PaxHeaders/%f,atime:=0
>
> 2. Any value in the --pax-option argument may contain a
> time reference, provided that it is enclosed in curly
> braces, e.g.:
>
>  --pax-option='atime:={now -2 days},address@hidden'
>
> Thanks & regards,
> Sergey
>
>
> From 7cb84c25ee51f443c69443e217efe194d12678ea Mon Sep 17 00:00:00 2001
> From: Sergey Poznyakoff <address@hidden>
> Date: Wed, 7 Oct 2009 18:40:07 +0300
> Subject: [PATCH 2/3] Use file's mtime as mtime for its extended header.
>
> This makes two pax archives binary equivalent if they
> have the same contents and care is taken to make extended
> headers otherwise reproducible, e.g. by using:
>
>  --pax-option=exthdr.name=%d/PaxHeaders/%f,atime:=0
>
> Proposed by Michael D. Adams <address@hidden>.
>
> * src/common.h (start_private_header): Take time_t as 3rd param.
> (xheader_write): Likewise.
> * src/create.c (start_private_header): Take time_t as 3rd param.
> All callers updated.
> (write_extended): Use file's mtime as mtime for its extended header,
> Use current time stamp as mtime for global headers.
> (xheader_write): Take time_t as 3rd param.
> ---
>  src/common.h  |    4 ++--
>  src/create.c  |   25 +++++++++++++------------
>  src/xheader.c |    6 +++---
>  3 files changed, 18 insertions(+), 17 deletions(-)
>
> diff --git a/src/common.h b/src/common.h
> index c2a92d2..0020f08 100644
> --- a/src/common.h
> +++ b/src/common.h
> @@ -452,7 +452,7 @@ void finish_header (struct tar_stat_info *st, union block 
> *header,
>  void simple_finish_header (union block *header);
>  union block * write_extended (bool global, struct tar_stat_info *st,
>                              union block *old_header);
> -union block *start_private_header (const char *name, size_t size);
> +union block *start_private_header (const char *name, size_t size, time_t t);
>  void write_eot (void);
>  void check_links (void);
>  void exclusion_tag_warning (const char *dirname, const char *tagname,
> @@ -717,7 +717,7 @@ void xheader_decode_global (struct xheader *xhdr);
>  void xheader_store (char const *keyword, struct tar_stat_info *st,
>                    void const *data);
>  void xheader_read (struct xheader *xhdr, union block *header, size_t size);
> -void xheader_write (char type, char *name, struct xheader *xhdr);
> +void xheader_write (char type, char *name, time_t t, struct xheader *xhdr);
>  void xheader_write_global (struct xheader *xhdr);
>  void xheader_finish (struct xheader *hdr);
>  void xheader_destroy (struct xheader *hdr);
> diff --git a/src/create.c b/src/create.c
> index d4b9ae7..a964bc2 100644
> --- a/src/create.c
> +++ b/src/create.c
> @@ -515,9 +515,8 @@ write_eot (void)
>
>  /* Write a "private" header */
>  union block *
> -start_private_header (const char *name, size_t size)
> +start_private_header (const char *name, size_t size, time_t t)
>  {
> -  time_t t;
>   union block *header = find_next_block ();
>
>   memset (header->buffer, 0, sizeof (union block));
> @@ -525,7 +524,6 @@ start_private_header (const char *name, size_t size)
>   tar_name_copy_str (header->header.name, name, NAME_FIELD_SIZE);
>   OFF_TO_CHARS (size, header->header.size);
>
> -  time (&t);
>   TIME_TO_CHARS (t, header->header.mtime);
>   MODE_TO_CHARS (S_IFREG|S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH, 
> header->header.mode);
>   UID_TO_CHARS (getuid (), header->header.uid);
> @@ -563,13 +561,13 @@ write_gnu_long_link (struct tar_stat_info *st, const 
> char *p, char type)
>   union block *header;
>   char *tmpname;
>
> -  header = start_private_header ("././@LongLink", size);
> -  FILL(header->header.mtime, '0');
> -  FILL(header->header.mode, '0');
> -  FILL(header->header.uid, '0');
> -  FILL(header->header.gid, '0');
> -  FILL(header->header.devmajor, 0);
> -  FILL(header->header.devminor, 0);
> +  header = start_private_header ("././@LongLink", size, time (NULL));
> +  FILL (header->header.mtime, '0');
> +  FILL (header->header.mode, '0');
> +  FILL (header->header.uid, '0');
> +  FILL (header->header.gid, '0');
> +  FILL (header->header.devmajor, 0);
> +  FILL (header->header.devminor, 0);
>   uid_to_uname (0, &tmpname);
>   UNAME_TO_CHARS (tmpname, header->header.uname);
>   free (tmpname);
> @@ -712,7 +710,8 @@ write_extended (bool global, struct tar_stat_info *st, 
> union block *old_header)
>   union block *header, hp;
>   char *p;
>   int type;
> -
> +  time_t t;
> +
>   if (st->xhdr.buffer || st->xhdr.stk == NULL)
>     return old_header;
>
> @@ -722,13 +721,15 @@ write_extended (bool global, struct tar_stat_info *st, 
> union block *old_header)
>     {
>       type = XGLTYPE;
>       p = xheader_ghdr_name ();
> +      time (&t);
>     }
>   else
>     {
>       type = XHDTYPE;
>       p = xheader_xhdr_name (st);
> +      t = st->stat.st_mtime;
>     }
> -  xheader_write (type, p, &st->xhdr);
> +  xheader_write (type, p, t, &st->xhdr);
>   free (p);
>   header = find_next_block ();
>   memcpy (header, &hp.buffer, sizeof (hp.buffer));
> diff --git a/src/xheader.c b/src/xheader.c
> index c779c0a..e8fd6a2 100644
> --- a/src/xheader.c
> +++ b/src/xheader.c
> @@ -364,14 +364,14 @@ xheader_ghdr_name (void)
>  }
>
>  void
> -xheader_write (char type, char *name, struct xheader *xhdr)
> +xheader_write (char type, char *name, time_t t, struct xheader *xhdr)
>  {
>   union block *header;
>   size_t size;
>   char *p;
>
>   size = xhdr->size;
> -  header = start_private_header (name, size);
> +  header = start_private_header (name, size, t);
>   header->header.typeflag = type;
>
>   simple_finish_header (header);
> @@ -413,7 +413,7 @@ xheader_write_global (struct xheader *xhdr)
>   for (kp = keyword_global_override_list; kp; kp = kp->next)
>     code_string (kp->value, kp->pattern, xhdr);
>   xheader_finish (xhdr);
> -  xheader_write (XGLTYPE, name = xheader_ghdr_name (), xhdr);
> +  xheader_write (XGLTYPE, name = xheader_ghdr_name (), time (NULL), xhdr);
>   free (name);
>  }
>
> --
> 1.6.4.2
>
>
> From 63e092548a9b87c0be0f0b286c883e1f3d52294c Mon Sep 17 00:00:00 2001
> From: Sergey Poznyakoff <address@hidden>
> Date: Wed, 7 Oct 2009 21:08:29 +0300
> Subject: [PATCH 3/3] Provide a way to explicitly set mtime for extended 
> header ustar blocks.
>
> * src/tar.c (struct textual_date): ts is a copy of the structure,
> not a pointer to it. Date is a copy as well, hence the `const' is
> taken away.
> (get_date_or_file): Return 0/1 depending on success/failure.
> Copy timestamp to the `ts' member. Store a copy of the string
> in `date'.
> (report_textual_dates): Report only if verbose_option is set,
> but always free the list.
> (expand_pax_option): New function.
> (parse_opt): Preprocess the argument to xheader_set_option with
> expand_pax_option.
> (decode_options): Call report_textual_dates unconditionally.
> * src/xheader.c (exthdr_mtime_option, exthdr_mtime)
> (globexthdr_mtime_option, globexthdr_mtime): New statics.
> (xheader_set_keyword_equal): handle exthdr.mtime and globexthdr.mtime.
> (xheader_write): Override `t' argument if a corresponding
> exthdr.mtime or globexthdr.mtime option is set.
> * NEWS: Update
> * doc/tar.texi: Document the changes.
> ---
>  NEWS          |   34 +++++++++++++++++++++-
>  doc/tar.texi  |   46 +++++++++++++++++++++++++++--
>  src/tar.c     |   89 ++++++++++++++++++++++++++++++++++++++++++++++++--------
>  src/xheader.c |   37 +++++++++++++++++++++++
>  4 files changed, 189 insertions(+), 17 deletions(-)
>
> diff --git a/NEWS b/NEWS
> index bf02a24..7624d80 100644
> --- a/NEWS
> +++ b/NEWS
> @@ -1,4 +1,4 @@
> -GNU tar NEWS - User visible changes. 2009-09-08
> +GNU tar NEWS - User visible changes. 2009-10-07
>  Please send GNU tar bug reports to <address@hidden>
>
>
> @@ -53,11 +53,43 @@ and sets the exit code to 1, which means "some files 
> differ".
>  If the --warning=no-file-removed option is given, no warning
>  is issued and the exit code remains 0.
>
> +* Modification times of PAX extended headers.
> +
> +Modification times in the ustar header blocks for the
> +extended headers are set to the mtimes of the corresponding archive
> +members.  This can be overridden by the
> +
> +  --pax-opion='exthdr.mtime=STRING'
> +
> +command line option.  The STRING is either the number of seconds since
> +the Epoch or a `Time reference' (see below).
> +
> +Modification times in the ustar header blocks for the global
> +extended headers are set to the time when tar was invoked.
> +
> +This can be overridden by the
> +
> +  --pax-opion='globexthdr.mtime=STRING'
> +
> +command line option.  The STRING is either the number of seconds since
> +the Epoch or a `Time reference' (see below).
> +
> +* Time references in --pax-option argument.
> +
> +Any value from the --pax-option argument that is enclosed in a pair
> +of curly braces.  In that case, the string between the braces is
> +understood either as a textual time representation, as described in
> +chapter 7, "Date input formats", of the Tar manual, or as a name of
> +the existing file, starting with `/' or `.'.  In the latter
> +case, the value is replaced with the modification time of that file.
> +
>  * Bugfixes
>  ** Fix handling of hard link targets by -c --transform.
>  ** Fix hard links recognition with -c --remove-files.
>  ** Fix restoring files from backup (debian bug #508199).
>  ** Correctly restore modes and permissions on existing directories.
> +** The --remove-files option removes the files only if they were
> +succesfully stored in the archive.
>
>
>  version 1.22 - Sergey Poznyakoff, 2009-03-05
> diff --git a/doc/tar.texi b/doc/tar.texi
> index e609368..1102e32 100644
> --- a/doc/tar.texi
> +++ b/doc/tar.texi
> @@ -3055,8 +3055,8 @@ This option does not affect extraction from archives.
>
> address@hidden
> address@hidden address@hidden
> -This option is meaningful only with @acronym{POSIX.1-2001} archives
> -(@pxref{posix}).  It modifies the way @command{tar} handles the
> +This option enables creation of the archive in @acronym{POSIX.1-2001}
> +format (@pxref{posix}) and modifies the way @command{tar} handles the
>  extended header keywords. address@hidden is a comma-separated
>  list of keyword options. address@hidden keywords}, for a detailed
>  discussion.
> @@ -5400,7 +5400,7 @@ Name of the file owner group.
> address@hidden TAR_ATIME, to-command environment
> address@hidden TAR_ATIME
>  Time of last access. It is a decimal number, representing seconds
> -since the epoch.  If the archive provides times with nanosecond
> +since the Epoch.  If the archive provides times with nanosecond
>  precision, the nanoseconds are appended to the timestamp after a
>  decimal point.
>
> @@ -9409,6 +9409,13 @@ will use the following default value:
>  %d/PaxHeaders.%p/%f
> address@hidden smallexample
>
> address@hidden address@hidden
> +
> +This keyword defines the value of the @samp{mtime} field that
> +is written into the ustar header blocks for the extended headers.
> +By default, the @samp{mtime} field is set to the modification time
> +of the archive member described by that extended headers.
> +
> address@hidden address@hidden
>  This keyword allows user control over the name that is written into
>  the ustar header blocks for global extended header records.  The name
> @@ -9438,6 +9445,13 @@ where @samp{$TMPDIR} represents the value of the 
> @var{TMPDIR}
>  environment variable.  If @var{TMPDIR} is not set, @command{tar}
>  uses @samp{/tmp}.
>
> address@hidden address@hidden
> +
> +This keyword defines the value of the @samp{mtime} field that
> +is written into the ustar header blocks for the global extended headers.
> +By default, the @samp{mtime} field is set to the time when
> address@hidden was invoked.
> +
> address@hidden @address@hidden
>  When used with one of archive-creation commands, these keyword/value pairs
>  will be included at the beginning of the archive in a global extended
> @@ -9467,6 +9481,32 @@ the group name will be forced to a new value for all 
> files
>  stored in the archive.
> address@hidden table
>
> +In any of the forms described above, the @var{value} may be
> +a string enclosed in curly braces.  In that case, the string
> +between the braces is understood either as a textual time
> +representation, as described in @ref{Date input formats}, or a name of
> +the existing file, starting with @samp{/} or @samp{.}.  In the latter
> +case, the modification time of that file is used.
> +
> +For example, to set all modification times to the current date, you
> +use the following option:
> +
> address@hidden
> +--pax-option='mtime:address@hidden@}'
> address@hidden smallexample
> +
> +Note quoting of the option's argument.
> +
> address@hidden archives, binary equivalent
> address@hidden binary equivalent archives, creating
> +As another example, here is the option that ensures that any two
> +archives created using it, will be binary equivalent if they have the
> +same contents:
> +
> address@hidden
> +--pax-option=exthdr.name=%d/PaxHeaders/%f,atime:=0
> address@hidden smallexample
> +
> address@hidden Checksumming
> address@hidden Checksumming Problems
>
> diff --git a/src/tar.c b/src/tar.c
> index 0c59352..a639974 100644
> --- a/src/tar.c
> +++ b/src/tar.c
> @@ -1002,12 +1002,12 @@ set_stat_signal (const char *name)
>  struct textual_date
>  {
>   struct textual_date *next;
> -  struct timespec *ts;
> +  struct timespec ts;
>   const char *option;
> -  const char *date;
> +  char *date;
>  };
>
> -static void
> +static int
>  get_date_or_file (struct tar_args *args, const char *option,
>                  const char *str, struct timespec *ts)
>  {
> @@ -1030,17 +1030,19 @@ get_date_or_file (struct tar_args *args, const char 
> *option,
>          WARN ((0, 0, _("Substituting %s for unknown date format %s"),
>                 tartime (*ts, false), quote (str)));
>          ts->tv_nsec = 0;
> +         return 1;
>        }
>       else
>        {
>          struct textual_date *p = xmalloc (sizeof (*p));
> -         p->ts = ts;
> +         p->ts = *ts;
>          p->option = option;
> -         p->date = str;
> +         p->date = xstrdup (str);
>          p->next = args->textual_date;
>          args->textual_date = p;
>        }
>     }
> +  return 0;
>  }
>
>  static void
> @@ -1050,10 +1052,14 @@ report_textual_dates (struct tar_args *args)
>   for (p = args->textual_date; p; )
>     {
>       struct textual_date *next = p->next;
> -      char const *treated_as = tartime (*p->ts, true);
> -      if (strcmp (p->date, treated_as) != 0)
> -       WARN ((0, 0, _("Option %s: Treating date `%s' as %s"),
> -              p->option, p->date, treated_as));
> +      if (verbose_option)
> +       {
> +         char const *treated_as = tartime (p->ts, true);
> +         if (strcmp (p->date, treated_as) != 0)
> +           WARN ((0, 0, _("Option %s: Treating date `%s' as %s"),
> +                  p->option, p->date, treated_as));
> +       }
> +      free (p->date);
>       free (p);
>       p = next;
>     }
> @@ -1273,6 +1279,60 @@ tar_help_filter (int key, const char *text, void 
> *input)
>   return s;
>  }
>
> +static char *
> +expand_pax_option (struct tar_args *targs, const char *arg)
> +{
> +  struct obstack stk;
> +  char *res;
> +
> +  obstack_init (&stk);
> +  while (*arg)
> +    {
> +      size_t seglen = strcspn (arg, ",");
> +      char *p = memchr (arg, '=', seglen);
> +      if (p)
> +       {
> +         size_t len = p - arg + 1;
> +         obstack_grow (&stk, arg, len);
> +         len = seglen - len;
> +         for (++p; *p && isspace ((unsigned char) *p); p++)
> +           len--;
> +         if (*p == '{' && p[len-1] == '}')
> +           {
> +             struct timespec ts;
> +             char *tmp = xmalloc (len);
> +             memcpy (tmp, p + 1, len-2);
> +             tmp[len-2] = 0;
> +             if (get_date_or_file (targs, "--pax-option", tmp, &ts) == 0)
> +               {
> +                 char buf[UINTMAX_STRSIZE_BOUND], *s;
> +                 s = umaxtostr (ts.tv_sec, buf);
> +                 obstack_grow (&stk, s, strlen (s));
> +               }
> +             else
> +               obstack_grow (&stk, p, len);
> +             free (tmp);
> +           }
> +         else
> +           obstack_grow (&stk, p, len);
> +       }
> +      else
> +       obstack_grow (&stk, arg, seglen);
> +
> +      arg += seglen;
> +      if (*arg)
> +       {
> +         obstack_1grow (&stk, *arg);
> +         arg++;
> +       }
> +    }
> +  obstack_1grow (&stk, 0);
> +  res = xstrdup (obstack_finish (&stk));
> +  obstack_free (&stk, NULL);
> +  return res;
> +}
> +
> +
>  static error_t
>  parse_opt (int key, char *arg, struct argp_state *state)
>  {
> @@ -1840,8 +1900,12 @@ parse_opt (int key, char *arg, struct argp_state 
> *state)
>       break;
>
>     case PAX_OPTION:
> -      args->pax_option = true;
> -      xheader_set_option (arg);
> +      {
> +       char *tmp = expand_pax_option (args, arg);
> +       args->pax_option = true;
> +       xheader_set_option (tmp);
> +       free (tmp);
> +      }
>       break;
>
>     case POSIX_OPTION:
> @@ -2440,8 +2504,7 @@ decode_options (int argc, char **argv)
>
>   checkpoint_finish_compile ();
>
> -  if (verbose_option)
> -    report_textual_dates (&args);
> +  report_textual_dates (&args);
>  }
>
>
> diff --git a/src/xheader.c b/src/xheader.c
> index e8fd6a2..5eabdfb 100644
> --- a/src/xheader.c
> +++ b/src/xheader.c
> @@ -96,9 +96,15 @@ static struct keyword_list *global_header_override_list;
>  /* Template for the name field of an 'x' type header */
>  static char *exthdr_name;
>
> +static char *exthdr_mtime_option;
> +static time_t exthdr_mtime;
> +
>  /* Template for the name field of a 'g' type header */
>  static char *globexthdr_name;
>
> +static char *globexthdr_mtime_option;
> +static time_t globexthdr_mtime;
> +
>  bool
>  xheader_keyword_deleted_p (const char *kw)
>  {
> @@ -157,6 +163,21 @@ xheader_set_single_keyword (char *kw)
>  }
>
>  static void
> +assign_time_option (char **sval, time_t *tval, const char *input)
> +{
> +  uintmax_t u;
> +  char *p;
> +  time_t t = u = strtoumax (input, &p, 10);
> +  if (t != u || *p || errno == ERANGE)
> +    ERROR ((0, 0, _("Time stamp is out of allowed range")));
> +  else
> +    {
> +      *tval = t;
> +      assign_string (sval, input);
> +    }
> +}
> +
> +static void
>  xheader_set_keyword_equal (char *kw, char *eq)
>  {
>   bool global = true;
> @@ -186,6 +207,10 @@ xheader_set_keyword_equal (char *kw, char *eq)
>     assign_string (&exthdr_name, p);
>   else if (strcmp (kw, "globexthdr.name") == 0)
>     assign_string (&globexthdr_name, p);
> +  else if (strcmp (kw, "exthdr.mtime") == 0)
> +    assign_time_option (&exthdr_mtime_option, &exthdr_mtime, p);
> +  else if (strcmp (kw, "globexthdr.mtime") == 0)
> +    assign_time_option (&globexthdr_mtime_option, &globexthdr_mtime, p);
>   else
>     {
>       if (xheader_protected_keyword_p (kw))
> @@ -371,6 +396,18 @@ xheader_write (char type, char *name, time_t t, struct 
> xheader *xhdr)
>   char *p;
>
>   size = xhdr->size;
> +  switch (type)
> +    {
> +    case XGLTYPE:
> +      if (globexthdr_mtime_option)
> +       t = globexthdr_mtime;
> +      break;
> +
> +    case XHDTYPE:
> +      if (exthdr_mtime_option)
> +       t = exthdr_mtime;
> +      break;
> +    }
>   header = start_private_header (name, size, t);
>   header->header.typeflag = type;
>
> --
> 1.6.4.2
>
>
>