[Emacs-diffs] trunk r117404: Simplify and parallize test/automated Makefile

[Glenn Morris]

------------------------------------------------------------
revno: 117404
revision-id: address@hidden
parent: address@hidden
fixes bug: http://debbugs.gnu.org/15991
committer: Glenn Morris <address@hidden>
branch nick: trunk
timestamp: Wed 2014-06-25 22:47:10 -0700
message:
  Simplify and parallize test/automated Makefile
  
  * Makefile.in (mostlyclean, clean): Maybe clean test/automated.
  
  * lisp/emacs-lisp/ert.el (ert-summarize-tests-batch-and-exit): New.
  
  * test/automated/Makefile.in: Simplify and parallelize. 
  (XARGS_LIMIT, BYTE_COMPILE_EXTRA_FLAGS)
  (setwins, compile-targets, compile-main, compile-clean): Remove.
  (GREP_OPTIONS): Unexport.
  (.el.elc): Replace with pattern rule.
  (%.elc, %.log): New pattern rules.
  (ELFILES, LOGFILES): New variables.
  (check): Depend on LOGFILES.  Call ert-summarize-tests-batch-and-exit.
  (clean, mostlyclean): New rules.
  (bootstrap-clean): Simplify.
  (bootstrap-clean, distclean): Depend on clean.
  
  * .bzrignore: Ignore test/automated/*.log.
modified:
  .bzrignore                     bzrignore-20091227212532-2lrlpwjsfm9i221p-1
  ChangeLog                      changelog-20091113204419-o5vbwnq5f7feedwu-1538
  Makefile.in                    makefile.in-20091113204419-o5vbwnq5f7feedwu-446
  lisp/ChangeLog                 changelog-20091113204419-o5vbwnq5f7feedwu-1432
  lisp/emacs-lisp/ert.el         ert.el-20110112160650-056hnl9qhpjvjicy-2
  test/ChangeLog                 changelog-20091113204419-o5vbwnq5f7feedwu-8588
  test/automated/Makefile.in     makefile.in-20110112160650-056hnl9qhpjvjicy-6

=== modified file '.bzrignore'
--- a/.bzrignore        2014-06-10 01:44:11 +0000
+++ b/.bzrignore        2014-06-26 05:47:10 +0000
@@ -227,3 +227,4 @@
 admin/charsets/eucjp-ms.el
 admin/charsets/jisx2131-filter
 test/automated/flymake/warnpred/a.out
+test/automated/*.log

=== modified file 'ChangeLog'
--- a/ChangeLog 2014-06-21 19:45:59 +0000
+++ b/ChangeLog 2014-06-26 05:47:10 +0000
@@ -1,3 +1,7 @@
+2014-06-26  Glenn Morris  <address@hidden>
+
+       * Makefile.in (mostlyclean, clean): Maybe clean test/automated.
+
 2014-06-21  Paul Eggert  <address@hidden>
 
        * configure.ac: Warn about --enable-link-time-optimization's issues

=== modified file 'Makefile.in'
--- a/Makefile.in       2014-06-20 16:05:10 +0000
+++ b/Makefile.in       2014-06-26 05:47:10 +0000
@@ -798,7 +798,9 @@
 $(foreach dir,$(mostlyclean_dirs),$(eval $(call 
submake_template,$(dir),mostlyclean)))
 
 mostlyclean: $(mostlyclean_dirs:=_mostlyclean)
-
+       for dir in test/automated; do \
+         [ ! -d $$dir ] || $(MAKE) -C $$dir mostlyclean; \
+       done
 
 ### `clean'
 ###      Delete all files from the current directory that are normally
@@ -813,6 +815,9 @@
 $(foreach dir,$(clean_dirs),$(eval $(call submake_template,$(dir),clean)))
 
 clean: $(clean_dirs:=_clean)
+       for dir in test/automated; do \
+         [ ! -d $$dir ] || $(MAKE) -C $$dir clean; \
+       done
        -rm -f etc/emacs.tmpdesktop
 
 ### `bootclean'

=== modified file 'lisp/ChangeLog'
--- a/lisp/ChangeLog    2014-06-25 23:51:10 +0000
+++ b/lisp/ChangeLog    2014-06-26 05:47:10 +0000
@@ -1,3 +1,7 @@
+2014-06-26  Glenn Morris  <address@hidden>
+
+       * emacs-lisp/ert.el (ert-summarize-tests-batch-and-exit): New.
+
 2014-06-25  Glenn Morris  <address@hidden>
 
        * Makefile.in ($(lisp)/progmodes/cc-defs.elc)

=== modified file 'lisp/emacs-lisp/ert.el'
--- a/lisp/emacs-lisp/ert.el    2014-06-22 05:43:58 +0000
+++ b/lisp/emacs-lisp/ert.el    2014-06-26 05:47:10 +0000
@@ -1463,6 +1463,65 @@
       (kill-emacs 2))))
 
 
+(defun ert-summarize-tests-batch-and-exit ()
+  "Summarize the results of testing.
+Expects to be called in batch mode, with logfiles as command-line arguments.
+The logfiles should have the `ert-run-tests-batch' format.  When finished,
+this exits Emacs, with status as per `ert-run-tests-batch-and-exit'."
+  (or noninteractive
+      (user-error "This function is only for use in batch mode"))
+  (let ((nlogs (length command-line-args-left))
+        (ntests 0) (nrun 0) (nexpected 0) (nunexpected 0) (nskipped 0)
+        nnotrun logfile notests badtests unexpected)
+    (with-temp-buffer
+      (while (setq logfile (pop command-line-args-left))
+        (erase-buffer)
+        (insert-file-contents logfile)
+        (if (not (re-search-forward "^Running \\([0-9]+\\) tests" nil t))
+            (push logfile notests)
+          (setq ntests (+ ntests (string-to-number (match-string 1))))
+          (if (not (re-search-forward "^\\(Aborted: \\)?\
+Ran \\([0-9]+\\) tests, \\([0-9]+\\) results as expected\
+\\(?:, \\([0-9]+\\) unexpected\\)?\
+\\(?:, \\([0-9]+\\) skipped\\)?" nil t))
+              (push logfile badtests)
+            (if (match-string 1) (push logfile badtests))
+            (setq nrun (+ nrun (string-to-number (match-string 2)))
+                  nexpected (+ nexpected (string-to-number (match-string 3))))
+            (when (match-string 4)
+              (push logfile unexpected)
+              (setq nunexpected (+ nunexpected
+                                   (string-to-number (match-string 4)))))
+            (if (match-string 5)
+                (setq nskipped (+ nskipped
+                                  (string-to-number (match-string 5)))))))))
+    (setq nnotrun (- ntests nrun))
+    (message "\nSUMMARY OF TEST RESULTS")
+    (message "-----------------------")
+    (message "Files examined: %d" nlogs)
+    (message "Ran %d tests%s, %d results as expected%s%s"
+             nrun
+             (if (zerop nnotrun) "" (format ", %d failed to run" nnotrun))
+             nexpected
+             (if (zerop nunexpected)
+                 ""
+               (format ", %d unexpected" nunexpected))
+             (if (zerop nskipped)
+                 ""
+               (format ", %d skipped" nskipped)))
+    (when notests
+      (message "%d files did not contain any tests:" (length notests))
+      (mapc (lambda (l) (message "  %s" l)) notests))
+    (when badtests
+      (message "%d files did not finish:" (length badtests))
+      (mapc (lambda (l) (message "  %s" l)) badtests))
+    (when unexpected
+      (message "%d files contained unexpected results:" (length unexpected))
+      (mapc (lambda (l) (message "  %s" l)) unexpected))
+    (kill-emacs (cond ((or notests badtests (not (zerop nnotrun))) 2)
+                      (unexpected 1)
+                      (t 0)))))
+
 ;;; Utility functions for load/unload actions.
 
 (defun ert--activate-font-lock-keywords ()

=== modified file 'test/ChangeLog'
--- a/test/ChangeLog    2014-06-25 19:12:14 +0000
+++ b/test/ChangeLog    2014-06-26 05:47:10 +0000
@@ -1,3 +1,17 @@
+2014-06-26  Glenn Morris  <address@hidden>
+
+       * automated/Makefile.in: Simplify and parallelize.  (Bug#15991)
+       (XARGS_LIMIT, BYTE_COMPILE_EXTRA_FLAGS)
+       (setwins, compile-targets, compile-main, compile-clean): Remove.
+       (GREP_OPTIONS): Unexport.
+       (.el.elc): Replace with pattern rule.
+       (%.elc, %.log): New pattern rules.
+       (ELFILES, LOGFILES): New variables.
+       (check): Depend on LOGFILES.  Call ert-summarize-tests-batch-and-exit.
+       (clean, mostlyclean): New rules.
+       (bootstrap-clean): Simplify.
+       (bootstrap-clean, distclean): Depend on clean.
+
 2014-06-25  Glenn Morris  <address@hidden>
 
        * automated/flymake-tests.el (flymake-tests--current-face):

=== modified file 'test/automated/Makefile.in'
--- a/test/automated/Makefile.in        2014-06-13 23:05:00 +0000
+++ b/test/automated/Makefile.in        2014-06-26 05:47:10 +0000
@@ -24,10 +24,6 @@
 
 SEPCHAR = @SEPCHAR@
 
-# Empty for all systems except MinGW, where xargs needs an explicit
-# limitation.
-XARGS_LIMIT = @XARGS_LIMIT@
-
 # We never change directory before running Emacs, so a relative file
 # name is fine, and makes life easier.  If we need to change
 # directory, we can use emacs --chdir.
@@ -38,87 +34,73 @@
 # but we might as well be explicit.
 EMACSOPT = -batch --no-site-file --no-site-lisp -L "$(SEPCHAR)$(srcdir)"
 
-# Extra flags to pass to the byte compiler.
-BYTE_COMPILE_EXTRA_FLAGS =
-
 # Prevent any settings in the user environment causing problems.
-unexport EMACSDATA EMACSDOC EMACSPATH
+unexport EMACSDATA EMACSDOC EMACSPATH GREP_OPTIONS
 
 # The actual Emacs command run in the targets below.
 # Prevent any setting of EMACSLOADPATH in user environment causing problems.
 emacs = EMACSLOADPATH= LC_ALL=C EMACS_TEST_DIRECTORY=$(srcdir) "$(EMACS)" 
$(EMACSOPT)
 
-# Common command to find subdirectories
-setwins=for file in `find $(srcdir) -type d -print`; do \
-          case $$file in $(srcdir)*/data* | $(srcdir)*/flymake* ) ;; \
-               *) wins="$$wins$${wins:+ }$$file" ;; \
-          esac; \
-        done
-
 .PHONY: all check
 
 all: check
 
-# The compilation stuff is copied from lisp/Makefile - see comments there.
-
-.SUFFIXES: .elc .el
-
-.el.elc:
+%.elc: %.el
        @echo Compiling $<
-       @$(emacs) $(BYTE_COMPILE_EXTRA_FLAGS) -f batch-byte-compile $<
-
-
-.PHONY: compile-targets compile-main compile-clean
-
-# TARGETS is set dynamically in the recursive call from `compile-main'.
-compile-targets: $(TARGETS)
-
-# Compile all the Elisp files that need it.  Beware: it approximates
-# `no-byte-compile', so watch out for false-positives!
-compile-main: compile-clean
-       @$(setwins); \
-       els=`echo "$$wins " | sed -e 's|/\./|/|g' -e 's|/\. | |g' -e 's| |/*.el 
|g'`; \
-       for el in $$els; do \
-         test -f $$el || continue; \
-         test ! -f $${el}c && GREP_OPTIONS= grep '^;.*no-byte-compile: t' $$el 
> /dev/null && continue; \
-         echo "$${el}c"; \
-       done | xargs $(XARGS_LIMIT) echo | \
-       while read chunk; do \
-         $(MAKE) compile-targets EMACS="$(EMACS)" TARGETS="$$chunk"; \
-       done
-
-# Erase left-over .elc files that do not have a corresponding .el file.
-compile-clean:
-       @$(setwins); \
-       elcs=`echo "$$wins " | sed -e 's|/\./|/|g' -e 's|/\. | |g' -e 's| 
|/*.elc |g'`; \
-       for el in $$(echo $$elcs | sed -e 's/\.elc/\.el/g'); do \
-         if test -f "$$el" -o \! -f "$${el}c"; then :; else \
-           echo rm "$${el}c"; \
-           rm "$${el}c"; \
-         fi \
-       done
-
-
-.PHONY: bootstrap-clean distclean maintainer-clean
-
-bootstrap-clean:
-       -cd $(srcdir) && rm -f *.elc */*.elc */*/*.elc */*/*/*.elc
-
-distclean:
+       @$(emacs) -f batch-byte-compile $<
+
+## Ignore any test errors so we can continue to test other files.
+## (It would be nice if we could get an error when running an
+## individual test, but not when running check.)
+## But compilation errors are always fatal.
+##
+## I'd prefer to use -emacs -f ert-run-tests-batch-and-exit rather
+## than || true, since the former makes problems more obvious.
+## I'd also prefer to @-hide the grep part and not the
+## ert-run-tests-batch-and-exit part.
+##
+## We need to use $loadfile because:
+## i) -L :$srcdir -l basename does not work, because we have files whose
+## basename duplicates a file in lisp/ (eg eshell.el).
+## ii) Although -l basename will automatically load .el or .elc,
+## -l ./basename treats basename as a literal file (it would be nice
+## to change this).
+##
+## Beware: it approximates `no-byte-compile', so watch out for false-positives!
+%.log: ${srcdir}/%.el
+       @if grep '^;.*no-byte-compile: t' $< > /dev/null; then \
+         loadfile=$<; \
+       else \
+         loadfile=$<c; \
+         ${MAKE} $$loadfile; \
+       fi; \
+       echo Testing $$loadfile; \
+       stat=OK ; \
+       $(emacs) -l ert -l $$loadfile \
+         -f ert-run-tests-batch-and-exit >& $@ || stat=ERROR; \
+       echo $$stat: $@
+
+ELFILES = $(wildcard ${srcdir}/*.el)
+LOGFILES = $(patsubst %.el,%.log,$(notdir ${ELFILES}))
+
+## If we have to interrupt a hanging test, preserve the log so we can
+## see what the problem was.
+.PRECIOUS: %.log
+
+check: ${LOGFILES}
+       $(emacs) -l ert -f ert-summarize-tests-batch-and-exit $^
+
+.PHONY: mostlyclean clean bootstrap-clean distclean maintainer-clean
+
+clean mostlyclean:
+       -rm -f *.log
+
+bootstrap-clean: clean
+       -rm -f ${srcdir}/*.elc
+
+distclean: clean
        rm -f Makefile
 
 maintainer-clean: distclean bootstrap-clean
 
-
-check: compile-main
-       @$(setwins); \
-       pattern=`echo "$$wins " | sed -e 's|/\./|/|g' -e 's|/\. | |g' -e 's| 
|/*.el |g'`; \
-       for el in $$pattern; do \
-         test -f $$el || continue; \
-         args="$$args -l $$el"; \
-         els="$$els $$el"; \
-       done; \
-       echo Testing $$els; \
-       $(emacs) $$args -f ert-run-tests-batch-and-exit
-
 # Makefile ends here.