[Emacs-diffs] emacs-26 a9b884c: Tag some unstable tests, and skip by default (bug#24503)

[Glenn Morris]

branch: emacs-26
commit a9b884c60f5d17d55e18039b73020a1c726dd048
Author: Glenn Morris <address@hidden>
Commit: Glenn Morris <address@hidden>

    Tag some unstable tests, and skip by default (bug#24503)
    
    * Makefile.in (check-all): New phony target.
    * test/Makefile.in (SELECTOR_DEFAULT, SELECTOR_EXPENSIVE):
    Also skip unstable tests.
    (SELECTOR_ALL): New variable.
    (check-all): New phony target.
    * test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el
    (eieio-test-method-order-list-6):
    * test/lisp/emacs-lisp/eieio-tests/eieio-tests.el
    (eieio-test-37-obsolete-name-in-constructor):
    Mark as unstable rather than skipping on hydra.nixos.org.
---
 Makefile.in                                                 | 13 ++++++++-----
 test/Makefile.in                                            | 10 ++++++++--
 test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el |  2 +-
 test/lisp/emacs-lisp/eieio-tests/eieio-tests.el             |  4 ++--
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/Makefile.in b/Makefile.in
index 89277e2..238df40 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -58,9 +58,11 @@
 # make docs
 #      Make Emacs documentation files from their sources; requires makeinfo.
 #
-# make check  or  make check-expensive
-#      Run Emacs test suite.  check-expensive runs also tests which
-#      take more time to perform.
+# make check (or check-expensive or check-all)
+#      Run the Emacs test suite.
+#      check-expensive includes additional tests that can be slow.
+#      check-all runs all tests, including ones that can be slow, or
+#        fail unpredictably
 
 SHELL = @SHELL@
 
@@ -938,7 +940,8 @@ have-tests:
         exit 1; \
        fi
 
-check check-maybe check-expensive: have-tests all
+.PHONY: check check-maybe check-expensive check-all
+check check-maybe check-expensive check-all: have-tests all
        $(MAKE) -C test $@
 
 dist:
@@ -955,7 +958,7 @@ $(DOCS):
        $(MAKE) -C doc/$(subst -, ,$@)
 
 .PHONY: $(DOCS) docs pdf ps
-.PHONY: info dvi dist check check-maybe check-expensive html info-real 
info-dir check-info
+.PHONY: info dvi dist html info-real info-dir check-info
 
 ## TODO add etc/refcards.
 docs: $(DOCS)
diff --git a/test/Makefile.in b/test/Makefile.in
index 7bc99fb..e03ffc5 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -125,8 +125,9 @@ test_module_dir := $(srcdir)/data/emacs-module
 
 all: check
 
-SELECTOR_DEFAULT = (quote (not (tag :expensive-test)))
-SELECTOR_EXPENSIVE = nil
+SELECTOR_DEFAULT = (quote (not (or (tag :expensive-test) (tag :unstable))))
+SELECTOR_EXPENSIVE = (quote (not (tag :unstable)))
+SELECTOR_ALL = nil
 ifdef SELECTOR
 SELECTOR_ACTUAL=$(SELECTOR)
 else ifndef MAKECMDGOALS
@@ -241,6 +242,11 @@ check: mostlyclean check-no-automated-subdir
 check-expensive: mostlyclean check-no-automated-subdir
        @${MAKE} check-doit SELECTOR="${SELECTOR_EXPENSIVE}"
 
+## Run all tests, regardless of tag.
+.PHONY: check-all
+check-all: mostlyclean check-no-automated-subdir
+       @${MAKE} check-doit SELECTOR="${SELECTOR_ALL}"
+
 ## Re-run all tests which are outdated. A test is outdated if its
 ## logfile is out-of-date with either the test file, or the source
 ## files that the tests depend on.  See test_template.
diff --git a/test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el 
b/test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el
index 84661e8..c6da9e1 100644
--- a/test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el
+++ b/test/lisp/emacs-lisp/eieio-tests/eieio-test-methodinvoke.el
@@ -192,7 +192,7 @@
 (ert-deftest eieio-test-method-order-list-6 ()
   ;; FIXME repeated intermittent failures on hydra (bug#24503)
   ;; ((:STATIC C) (:STATIC C-base1) (:STATIC C-base2)) != ((:STATIC C))")
-  (skip-unless (not (getenv "EMACS_HYDRA_CI")))
+  :tags '(:unstable)
   (let ((eieio-test-method-order-list nil)
        (ans '(
               (:STATIC C)
diff --git a/test/lisp/emacs-lisp/eieio-tests/eieio-tests.el 
b/test/lisp/emacs-lisp/eieio-tests/eieio-tests.el
index ee73974..5ba094c 100644
--- a/test/lisp/emacs-lisp/eieio-tests/eieio-tests.el
+++ b/test/lisp/emacs-lisp/eieio-tests/eieio-tests.el
@@ -893,8 +893,8 @@ Subclasses to override slot attributes.")
   (list newname 2))
 
 (ert-deftest eieio-test-37-obsolete-name-in-constructor ()
-  ;; FIXME repeated intermittent failures on hydra (bug#24503)
-  (skip-unless (not (getenv "EMACS_HYDRA_CI")))
+  ;; FIXME repeated intermittent failures on hydra and elsewhere (bug#24503).
+  :tags '(:unstable)
   (should (equal (eieio--testing "toto") '("toto" 2))))
 
 (ert-deftest eieio-autoload ()