Parallel automake --add-missing: serialized file installs. [4/4]

[Ralf Wildenhues]

The final piece.  A bit ugly because the changes in require_conf_file
depend on nonlocal semantics in require_file_internal and in
maybe_push_required_file.  Oh well.

Cheers,
Ralf

        Parallel automake --add-missing: serialized file installs.
        * automake.in (QUEUE_CONF_FILE, QUEUE_LOCATION, QUEUE_STRING):
        New serialization keys.
        ($required_conf_file_queue): New file global.
        (queue_required_conf_file, require_queued_conf_file): New
        functions, to queue and dequeue requirements for aux dir files.
        (require_conf_file): Enqueue if needed.
        (get_number_of_threads): Can do threads with --add-missing now.
        (handle_makefiles_threaded): Let worker threads enqueue, let
        master attend to queued requirements at the right time.
        * tests/parallel-am.test: Explain the purpose of the include
        chain used here.
        * tests/parallel-am2.test: Also cope with --add-missing.
        * tests/parallel-am3.test: New test, test absence of races with
        concurrent same-file installs stemming from --add-missing.
        * tests/Makefile.am: Adjust.

diff --git a/automake.in b/automake.in
index d7db627..0815773 100755
--- a/automake.in
+++ b/automake.in
@@ -280,6 +280,9 @@ use constant INTERNAL => new Automake::Location;
 # Serialization keys for message queues.
 use constant {
   QUEUE_MESSAGE   => "msg",
+  QUEUE_CONF_FILE => "conf file",
+  QUEUE_LOCATION  => "location",
+  QUEUE_STRING    => "string"
 };
 
 
@@ -7475,13 +7478,80 @@ sub require_libsource_with_macro ($$$@)
       }
 }
 
+# Queue to push require_conf_file requirements to.
+my $required_conf_file_queue;
+
+# &queue_required_conf_file ($QUEUE, $KEY, $DIR, $WHERE, $MYSTRICT, @FILES)
+# -------------------------------------------------------------------------
+sub queue_required_conf_file ($$$$@)
+{
+    my ($queue, $key, $dir, $where, $mystrict, @files) = @_;
+    my @serial_loc;
+    if (ref $where)
+      {
+        @serial_loc = (QUEUE_LOCATION, $where->serialize ());
+      }
+    else
+      {
+        @serial_loc = (QUEUE_STRING, $where);
+      }
+    $queue->enqueue ($key, $dir, @serial_loc, $mystrict, 0 + @files, @files);
+}
+
+# &require_queued_conf_file ($QUEUE)
+# ----------------------------------
+sub require_queued_conf_file ($)
+{
+    my ($queue) = @_;
+    my $where;
+    my $dir = $queue->dequeue ();
+    my $loc_key = $queue->dequeue ();
+    if ($loc_key eq QUEUE_LOCATION)
+      {
+       $where = Automake::Location::deserialize ($queue);
+      }
+    elsif ($loc_key eq QUEUE_STRING)
+      {
+       $where = $queue->dequeue ();
+      }
+    else
+      {
+       prog_error "unexpected key $loc_key";
+      }
+    my $mystrict = $queue->dequeue ();
+    my $nfiles = $queue->dequeue ();
+    my @files;
+    push @files, $queue->dequeue ()
+      foreach (1 .. $nfiles);
+
+    # Dequeuing happens outside of per-makefile context, so we have to
+    # set the variables used by require_file_internal and the functions
+    # it calls.  Gross!
+    $relative_dir = $dir;
+    require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+}
+
 # &require_conf_file ($WHERE, $MYSTRICT, @FILES)
 # ----------------------------------------------
-# Looks in configuration path, as specified by AC_CONFIG_AUX_DIR.
+# Looks in configuration path, as specified by AC_CONFIG_AUX_DIR;
+# worker threads may queue up the action to be serialized by the master.
+#
+# FIXME: this seriously relies on the semantics of require_file_internal
+# and maybe_push_required_file, in that we exploit the fact that only the
+# contents of the last handled output file may be impacted (which in turn
+# is dealt with by the master thread).
 sub require_conf_file ($$@)
 {
     my ($where, $mystrict, @files) = @_;
-    require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+    if (defined $required_conf_file_queue)
+      {
+       queue_required_conf_file ($required_conf_file_queue, QUEUE_CONF_FILE,
+                                 $relative_dir, $where, $mystrict, @files);
+      }
+    else
+      {
+       require_file_internal ($where, $mystrict, $config_aux_dir, @files);
+      }
 }
 
 
@@ -8051,11 +8121,6 @@ sub get_number_of_threads
     {
       $nthreads = $max_threads;
     }
-
-  # We cannot deal with --add-missing (yet).
-  $nthreads = 0
-    if ($add_missing);
-
   return $nthreads;
 }
 
@@ -8066,6 +8131,7 @@ sub get_number_of_threads
 # worker threads push back everything that needs serialization:
 # * warning and (normal) error messages, for stable stderr output
 #   order and content (avoiding duplicates, for example),
+# * races when installing aux files (and respective messages),
 # * races when collecting aux files for distribution.
 #
 # The latter requires that the makefile that deals with the aux dir
@@ -8101,9 +8167,11 @@ sub handle_makefiles_threaded ($)
              verb "handling $file";
              my $queue = $msg_queues{$file};
              setup_channel_queue ($queue, QUEUE_MESSAGE);
+             $required_conf_file_queue = $queue;
              handle_makefile ($file);
              $queue->enqueue (undef);
              setup_channel_queue (undef, undef);
+             $required_conf_file_queue = undef;
            }
          return $exit_code;
        });
@@ -8125,6 +8193,10 @@ sub handle_makefiles_threaded ($)
            {
              pop_channel_queue ($queue);
            }
+         elsif ($key eq QUEUE_CONF_FILE)
+           {
+             require_queued_conf_file ($queue);
+           }
          else
            {
              prog_error "unexpected key $key";
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 7c6d4a2..2526acf 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -451,6 +451,7 @@ output-order.test \
 overrid.test \
 parallel-am.test \
 parallel-am2.test \
+parallel-am3.test \
 parse.test \
 percent.test \
 percent2.test \
diff --git a/tests/parallel-am.test b/tests/parallel-am.test
index d57e014..de86271 100755
--- a/tests/parallel-am.test
+++ b/tests/parallel-am.test
@@ -60,6 +60,8 @@ for i in $list; do
   mkdir sub$i
   echo > sub$i/Makefile.am
 done
+# Use an include chain to cause a nontrivial location object to be
+# serialized through a thread queue.
 echo 'include foo.am' >> sub7/Makefile.am
 echo 'include bar.am' > sub7/foo.am
 echo 'python_PYTHON = foo.py' > sub7/bar.am
diff --git a/tests/parallel-am2.test b/tests/parallel-am2.test
index b5a4ac7..b92b79e 100755
--- a/tests/parallel-am2.test
+++ b/tests/parallel-am2.test
@@ -62,15 +62,16 @@ $ACLOCAL
 
 # Generate expected output using non-threaded code.
 unset AUTOMAKE_JOBS
+rm -f install-sh missing depcomp
 AUTOMAKE_fails --add-missing
-AUTOMAKE_fails
 mv stderr expected
 
 AUTOMAKE_JOBS=5
 export AUTOMAKE_JOBS
 
 for i in 1 2 3 4 5 6 7 8; do
-  AUTOMAKE_fails
+  rm -f install-sh missing depcomp
+  AUTOMAKE_fails --add-missing
   diff expected stderr
 done
 
diff --git a/tests/parallel-am3.test b/tests/parallel-am3.test
new file mode 100755
index 0000000..b3b0aac
--- /dev/null
+++ b/tests/parallel-am3.test
@@ -0,0 +1,75 @@
+#! /bin/sh
+# Copyright (C) 2008  Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Test parallel automake execution.
+
+# This tests:
+# 3) normal automake output should be identical and ordered in the same way
+#    with --add-missing, even with concurrent file requirements, and the
+#    installation of aux files should be race-free,
+
+. ./defs || Exit 1
+
+set -e
+
+cat > configure.in << 'END'
+AC_INIT([parallel-am], [1.0])
+AC_CONFIG_AUX_DIR([build-aux])
+AM_INIT_AUTOMAKE
+AC_PROG_CC
+AM_PATH_LISPDIR
+AM_PATH_PYTHON
+AC_CONFIG_FILES([Makefile])
+END
+
+cat > Makefile.am << 'END'
+SUBDIRS =
+END
+
+list='1 2 3'
+for i in $list; do
+  echo "AC_CONFIG_FILES([sub$i/Makefile])" >> configure.in
+  echo "SUBDIRS += sub$i" >> Makefile.am
+  mkdir sub$i
+  cat > sub$i/Makefile.am <<END
+python_PYTHON = foo$i.py
+lisp_LISP = foo$i.el
+bin_PROGRAMS = p$i
+END
+done
+
+rm -f install-sh missing depcomp
+mkdir build-aux
+
+$ACLOCAL
+
+# Generate expected output using the non-threaded code.
+unset AUTOMAKE_JOBS
+AUTOMAKE_run 0 --add-missing
+mv stderr expected
+mv Makefile.in Makefile.in.exp
+
+AUTOMAKE_JOBS=3
+export AUTOMAKE_JOBS
+
+for run in 1 2 3 4 5 6 7; do
+  rm -f build-aux/* sub*/Makefile.in
+  AUTOMAKE_run 0 --add-missing
+  diff stderr expected
+  diff Makefile.in Makefile.in.exp
+done
+
+: