[Automake-ng] [PATCH 1/4] [ng] dist: simplify VPATH handling

[Stefano Lattarini]

Now that we don't need to cater to Sun and Tru64 make, we can simplify
and rationalize our VPATH support in the creation of the distribution
directory.  Since we are at it, we clarify some existing comments, add
some new ones, and remove the obsolete ones.

This change has a little collateral effect in that after it the use of
"bare" wildcards in EXTRA_DIST does not work as intended anymore:

    # Won't distribute all test files in the builddir anymore.
    EXTRA_DIST = *.test

One will have to use the $(wildcard) GNU make builtin instead, as in:

    EXTRA_DIST = $(wildcard *.test)

This new limitation is deemed acceptable, especially because "bare"
wildcards suffer of a number of issues and inconsistencies, and their
use is mostly deprecated in favor of the $(wildcard) builtin (refer
to the GNU make manual for more details).

* lib/am/distdir.am (distdir): Extend the comments explaining why, while
filling up the dist directory, we need to check whether any distributed
file already exists in there; in particular, refer to some relevant bug
numbers and past commit IDs.  Remove workarounds and hack required to
support automatic VPATH rewrites with Sun and Tru64 make.  Refactor the
recipe using ...
(am__dist_files_1, am__dist_files_2, am__dist_files,
am__dist_parent_dirs): This new internal custom GNU make macros.
* NG-NEWS: Document that "bare" wildcards are not supported anymore
in EXTRA_DIST definition.
* t/extra12.sh: Adjust accordingly.
* t/nodep.sh: Remove, it's giving too much false positives to be
usable.
* t/dist-srcdir.sh: New test.
* t/dist-srcdir2.sh: New test, xfailing.

Signed-off-by: Stefano Lattarini <address@hidden>
---
 Makefile.am       |    1 +
 NG-NEWS           |   15 ++++++
 lib/am/distdir.am |  142 +++++++++++++++++++++++------------------------------
 t/dist-srcdir.sh  |   92 ++++++++++++++++++++++++++++++++++
 t/dist-srcdir2.sh |   49 ++++++++++++++++++
 t/extra12.sh      |    3 +-
 t/nodep.sh        |   40 ---------------
 7 files changed, 220 insertions(+), 122 deletions(-)
 create mode 100755 t/dist-srcdir.sh
 create mode 100755 t/dist-srcdir2.sh
 delete mode 100755 t/nodep.sh

diff --git a/Makefile.am b/Makefile.am
index 4509e00..754bcaf 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -376,6 +376,7 @@ XFAIL_TESTS = \
   t/yacc-bison-skeleton-cxx.sh \
   t/yacc-bison-skeleton.sh \
   t/cond17.sh \
+  t/dist-srcdir2.sh \
   t/gcj6.sh \
   t/override-conditional-2.sh \
   t/dist-pr109765.sh \
diff --git a/NG-NEWS b/NG-NEWS
index 99cb0bc..452ecd9 100644
--- a/NG-NEWS
+++ b/NG-NEWS
@@ -104,6 +104,21 @@ Parallel testsuite harness
   work as expected:
     TESTS = $(wildcard $(srcdir)/t[0-9][0-9]*.sh)
 
+Distribution
+============
+
+* "Bare" wildcards are not usable anymore in $(EXTRA_DIST):
+
+     EXTRA_DIST = $(srcdir)/doc/*.txt  # This won't work anymore.
+
+   You'll have to use the $(wildcard) GNU make builtin instead, as in:
+
+     EXTRA_DIST = $(wildcard $(srcdir)/doc/*.txt)  # Good.
+
+   or even (for a better support of VPATH builds):
+
+     EXTRA_DIST = $(wildcard doc/*.txt $(srcdir)/doc/*.txt)  # Better.
+
 Miscellaneous
 =============
 
diff --git a/lib/am/distdir.am b/lib/am/distdir.am
index 2c7307f..90ba095 100644
--- a/lib/am/distdir.am
+++ b/lib/am/distdir.am
@@ -16,6 +16,49 @@
 
 DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
 
+## Try to avoid repeated slashed in the $(DISTFILES) entries, to make the
+## filtering in the 'am__dist_files_1' definition below more reliable.
+## This idiom should compress up to four consecutive '/' characters in
+## each $(DISTFILES) entry.
+am__dist_files_1 = $(subst //,/,$(subst //,/,$(DISTFILES)))
+
+am__dist_files_2 = \
+## Files filtered out here require an ad-hoc "munging"; see the two
+## following 'patsubst's.
+  $(filter-out $(srcdir)/% $(top_srcdir)/%, $(am__dist_files_1)) \
+## Let's strip leading $(srcdir) (which might appears in EXTRA_DIST,
+## especially if one want to use the $(wildcard) built-in in there),
+## so that in our 'distdir' recipe below we can loop on the list of
+## distributed files and copy them in the distribution directory with
+## a simple "cp $file $(distdir)/$file" -- which would break if $file
+## contained a leading $(srcdir) component.
+## However, it should be noted that this filtering has the undesirable
+## side effect of triggering a VPATH search also for files specified
+## *explicitly* with a $(srcdir) prefix; but this limitation is also
+## present in mainstream Automake, and concerns only such corner-case
+## situations that it's probably not worth worrying about.
+  $(patsubst $(srcdir)/%, %, \
+             $(filter $(srcdir)/%, $(am__dist_files_1))) \
+## Also rewrite $(top_srcdir) (which sometimes appears in DISTFILES, and
+## can be absolute) by $(top_builddir) (which is always relative).  If
+## needed, $(srcdir) will be prepended later by our VPATH-aware rules.
+## The same caveats reported above apply.
+  $(patsubst $(top_srcdir)/%, $(top_builddir)/%, \
+             $(filter $(top_srcdir)/%, $(am__dist_files_1)))
+
+## Strip extra whitespaces, for more safety.
+am__dist_files = $(strip $(am__dist_files_2))
+
+## Given the pro-processing done above to 'am__dist_files', this
+## definition ensures that we won't try to create the wrong directories
+## when $(top_srcdir) or $(srcdir) appears in DISTFILES.  For example,
+## with EXTRA_DIST containing "$(srcdir)/subdir/file", this will allow
+## our rules to correctly create "$(distdir)/subdir", and not
+## "$(distdir)/$(srcdir)/subdir" -- which, in a VPATH build where
+## $(subdir) = .., would be the build directory!
+am__dist_parent_dirs = $(call am__uniq, \
+  $(filter-out ., $(patsubst ./%,%,$(dir $(am__dist_files)))))
+
 if %?TOPDIR_P%
 distdir = $(PACKAGE)-$(VERSION)
 top_distdir = $(distdir)
@@ -122,92 +165,27 @@ if %?TOPDIR_P%
        $(am__remove_distdir)
        test -d "$(distdir)" || mkdir "$(distdir)"
 endif %?TOPDIR_P%
-##
-##
-       @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
-       topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
-##
-## Yet another hack to support SUN make.
-##
-## Let's assume 'foo' appears in DISTFILES and is not a built file.
-## When building with VPATH=$(srcdir), SUN make and OSF1/Tru64 will
-## rewrite 'foo' as '$(srcdir)/foo'.  An attempt to install the file
-## with
-##    cp $file $(distdir)/$file
-## will thus install $(srcdir)/foo as $(distdir)/$(srcdir)/foo
-## instead of $(distdir)/foo.
-##
-## So let's strip this leading $(srcdir)/ when it exists.  (As far we
-## know, only SUN make and OSF1/Tru64 make add it.)  Searching whether
-## the file is to be found in the source or build directory will be
-## done later.
-##
-## In case we are _not_ using SUN or OSF1/Tru64 make, how can we be sure
-## we are not stripping a legitimate filename that starts with the
-## same pattern as $(srcdir)?
-## Well, it can't happen without the Makefile author distributing
-## something out of the distribution (which is bad).  As an example,
-## consider "EXTRA_DIST = ../bar".  This is an issue if $srcdir is
-## '..', however getting this value for srcdir is impossible:
-## "EXTRA_DIST = ../bar" implies we are in a subdirectory (so '../bar'
-## is within the package), hence '$srcdir' is something like
-## '../../subdir'.
-##
-## There is more to say about files which are above the current directory,
-## like '../bar' in the previous example.  The OSF1/Tru64 make
-## implementation can simplify filenames resulting from a VPATH lookup.
-## For instance if "VPATH = ../../subdir" and '../bar' is found in that
-## VPATH directory, then occurrences of '../bar' will be replaced by
-## '../../bar' (instead of '../../subdir/../bar').  This obviously defeats
-## any attempt to strip a leading $srcdir.  Presently we have no workaround
-## for this.  We avoid this issue by writing "EXTRA_DIST = $(srcdir)/../bar"
-## instead of "EXTRA_DIST = ../bar".  This prefixing is needed only for files
-## above the current directory.  Fortunately, apart from auxdir files which
-## can be located in .. or ../.., this situation hardly occurs in practice.
-##
-## Also rewrite $(top_srcdir) (which sometimes appears in DISTFILES, and can
-## be absolute) by $(top_builddir) (which is always relative).  $(srcdir) will
-## be prepended later.
-       list='$(DISTFILES)'; \
-         dist_files=`for file in $$list; do echo $$file; done | \
-         sed -e "s|^$$srcdirstrip/||;t" \
-             -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
-## (The second 't' command clears the flag for the next round.)
-##
 ## Make the subdirectories for the files.
-##
-       case $$dist_files in \
-         */*) $(MKDIR_P) `echo "$$dist_files" | \
-                          sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
-                          sort -u` ;; \
-       esac; \
-##
-##
-       for file in $$dist_files; do \
-##
-## Always look for the file in the build directory first.  That way
-## for something like yacc output we will correctly pick up the latest
-## version.  Also check for directories in the build directory first,
-## so one can ship generated directories.
-##
+       @$(if $(am__dist_parent_dirs),\
+          $(MKDIR_P) $(patsubst %,"$(distdir)"/%,$(am__dist_parent_dirs)))
+## Install the files and directories, applying a "VPATH rewrite"
+## by hand where needed.
+## To get the files in the distribution directory, use 'cp', not 'ln'.
+## There are situations in which 'ln' can fail.  For instance a file to
+## distribute could actually be a cross-filesystem symlink -- this can
+## easily happen if "gettextize" was run on the distribution.
+       @lst='$(am__dist_files)'; for file in $$lst; do \
+## Always look for the file or directory to distribute in the build
+## directory first, in VPATH spirit.
          if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
-##
-## Use cp, not ln.  There are situations in which "ln" can fail.  For
-## instance a file to distribute could actually be a cross-filesystem
-## symlink -- this can easily happen if "gettextize" was run on the
-## distribution.
-##
          if test -d $$d/$$file; then \
 ## Don't mention $$file in the destination argument, since this fails if
 ## the destination directory already exists.  Also, use '-R' and not '-r'.
 ## '-r' is almost always incorrect.
-##
 ## If a directory exists both in '.' and $(srcdir), then we copy the
 ## files from $(srcdir) first and then install those from '.'.  This
 ## can help people who distribute directories made of source files
-## *and* generated files.  It is also important when the directory
-## exists only in $(srcdir), because some vendor Make (such as Tru64)
-## will magically create an empty directory in '.'.
+## *and* generated files.
            dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
 ## If the destination directory already exists, it may contain read-only
 ## files, e.g., during "make distcheck".
@@ -220,9 +198,13 @@ endif %?TOPDIR_P%
            fi; \
            cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
          else \
-## Test for file existence because sometimes a file gets included in
-## DISTFILES twice.  For example this happens when a single source
-## file is used in building more than one program.
+## Test for file existence because sometimes:
+##  - a file gets included in DISTFILES twice (for example, this happens
+##    when a single source file is used in building more than one
+##    program), or
+##  - a single auxiliary file is distributed from several Makefiles at
+##    once (see automake bug#9546 and bug#9651, and the follow-up commits
+##    v1.11-1219-g326ecba, v1.11-1220-g851b1ae and v1.11-1221-gdccae6a).
 ## See also test 'dist-repeated.test'.
            test -f "$(distdir)/$$file" \
            || cp -p $$d/$$file "$(distdir)/$$file" \
diff --git a/t/dist-srcdir.sh b/t/dist-srcdir.sh
new file mode 100755
index 0000000..88d8d4a
--- /dev/null
+++ b/t/dist-srcdir.sh
@@ -0,0 +1,92 @@
+#! /bin/sh
+# Copyright (C) 2012 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# We use EXTRA_DIST to distribute stuff *explicitly* from the srcdir.
+
+am_create_testdir=empty
+. ./defs || Exit 1
+
+ocwd=`pwd` || fatal_ "cannot get current working directory"
+
+mkdir src
+cd src
+
+cat >> configure.ac <<END
+AC_INIT([$me], [1.0])
+AM_INIT_AUTOMAKE
+AC_CONFIG_FILES([Makefile sub/Makefile])
+AC_OUTPUT
+END
+
+cat > Makefile.am <<'END'
+SUBDIRS = sub
+
+EXTRA_DIST = \
+  $(srcdir)/one \
+  @srcdir@/two \
+  $(top_srcdir)/three \
+  @top_srcdir@/four
+
+.PHONY: test
+check-local: test
+test: distdir
+       find $(distdir) # For debugging.
+       test -f $(distdir)/one
+       test -f $(distdir)/two
+       test -f $(distdir)/three
+       test -f $(distdir)/four
+       test -f $(distdir)/sub/five
+       test -f $(distdir)/sub/six
+       test ! -f $(distdir)/five
+       test ! -f $(distdir)/six
+       test -f $(distdir)/seven
+       test -f $(distdir)/eight
+       test ! -f $(distdir)/sub/seven
+       test ! -f $(distdir)/sub/eight
+END
+
+mkdir sub
+cat > sub/Makefile.am <<'END'
+EXTRA_DIST  = $(srcdir)/five @srcdir@/six
+EXTRA_DIST += $(top_srcdir)/seven @top_srcdir@/eight
+END
+
+touch one two three four sub/five sub/six seven eight
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+mkdir build
+cd build
+../configure
+$MAKE test
+$MAKE distcheck
+cd "$ocwd"
+
+mkdir a a/b a/b/c
+cd a/b/c
+../../../src/configure
+$MAKE test
+cd "$ocwd"
+
+mkdir build2
+cd build2
+"$ocwd"/src/configure
+$MAKE test
+cd "$ocwd"
+
+:
diff --git a/t/dist-srcdir2.sh b/t/dist-srcdir2.sh
new file mode 100755
index 0000000..e64d412
--- /dev/null
+++ b/t/dist-srcdir2.sh
@@ -0,0 +1,49 @@
+#! /bin/sh
+# Copyright (C) 2012 Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# If we distribute a file whose name that starts with $(srcdir),
+# then the distribution rules should not try to instead distribute
+# a file with the same name from the builddir.
+# Currently, this doesn't work (the reasons and details for this
+# limitation should be explained in depth in comments in file
+# 'lib/am/distdir.am').
+
+. ./defs || Exit 1
+
+echo AC_OUTPUT >> configure.ac
+
+cat > Makefile.am <<'END'
+EXTRA_DIST = $(srcdir)/filename-that-is-easy-to-grep
+END
+
+$ACLOCAL
+$AUTOCONF
+$AUTOMAKE -a
+
+mkdir build
+cd build
+../configure
+
+echo bad > filename-that-is-easy-to-grep
+$MAKE distdir 2>stderr && { cat stderr >&2; Exit 1; }
+cat stderr >&2
+grep 'filename-that-is-easy-to-grep' stderr
+
+echo good > ../filename-that-is-easy-to-grep
+$MAKE distdir
+test "`cat $distdir/filename-that-is-easy-to-grep`" = good
+
+:
diff --git a/t/extra12.sh b/t/extra12.sh
index d499ced..0745850 100755
--- a/t/extra12.sh
+++ b/t/extra12.sh
@@ -22,9 +22,8 @@
 
 echo AC_OUTPUT >> configure.ac
 
-
 cat > Makefile.am <<'END'
-EXTRA_DIST = *.foo $(srcdir)/*.foo $(builddir)/*.bar $(srcdir)/*.bar
+EXTRA_DIST = $(wildcard *.foo $(srcdir)/*.foo $(builddir)/*.bar 
$(srcdir)/*.bar)
 
 .PHONY: test
 test: distdir
diff --git a/t/nodep.sh b/t/nodep.sh
deleted file mode 100755
index 1002c54..0000000
--- a/t/nodep.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#! /bin/sh
-# Copyright (C) 1999-2012 Free Software Foundation, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-# Test to make sure no-dependencies option does the right thing.
-# Bug report from Greg A. Woods.
-
-. ./defs || Exit 1
-
-cat > Makefile.am << 'END'
-AUTOMAKE_OPTIONS = no-dependencies
-bin_PROGRAMS = zardoz
-zardoz_SOURCES = y.c
-END
-
-cat >> configure.ac << 'END'
-AC_PROG_CC
-END
-
-mkdir x
-
-: > y.c
-
-$ACLOCAL
-$AUTOMAKE
-
-$FGREP -v '$(filter --%,' Makefile.in | grep '%' && Exit 1
-Exit 0
-- 
1.7.9.5