>From e5ed5248fe4fa5fed521e16929694c9d169540b2 Mon Sep 17 00:00:00 2001 From: Bruno Haible Date: Fri, 7 Jun 2024 15:31:26 +0200 Subject: [PATCH] automake: Allow omitting the logs of skipped tests from test-suite.log. * doc/automake.texi (Parallel Test Harness): Describe the contents of test-suite.log. Mention IGNORE_SKIPPED_LOGS. (Command-line arguments for test drivers): Document the --collect-skipped-logs option. * lib/test-driver (print_usage): Document --collect-skipped-logs. (collect_skipped_logs): New variable. (gcopy): Use collect_skipped_logs. * lib/am/check.am (am__common_driver_flags): Add the contents of am__collect_skipped_logs. (am__check_pre): Set am__collect_skipped_logs. * t/parallel-tests-without-skipped-logs.sh: New file, based on t/parallel-tests-no-color-in-log.sh. * t/list-of-tests.mk (handwritten_TESTS): Add it. * NEWS: Mention the change. --- NEWS | 4 ++ doc/automake.texi | 14 ++++-- lib/am/check.am | 6 +++ lib/test-driver | 7 ++- t/list-of-tests.mk | 1 + t/parallel-tests-without-skipped-logs.sh | 63 ++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 t/parallel-tests-without-skipped-logs.sh diff --git a/NEWS b/NEWS index 2da4cc769..55f692695 100644 --- a/NEWS +++ b/NEWS @@ -48,6 +48,10 @@ New in 1.17: console message about bug reporting on failure has a bit more detail. (bug#68746) + - When using the (default) "parallel" test driver, you can now choose to + omit the output of skipped tests from test-suite.log, by defining the + variable IGNORE_SKIPPED_LOGS to a non-empty value. + * Bugs fixed - Generated file timestamp checks handle filesystems with sub-second diff --git a/doc/automake.texi b/doc/automake.texi index 4f7029fff..fc009ae4d 100644 --- a/doc/automake.texi +++ b/doc/automake.texi @@ -9742,14 +9742,13 @@ versions. @node Parallel Test Harness @subsection Parallel Test Harness -By default, Automake generated a parallel (concurrent) test harness. It +By default, Automake generates a parallel (concurrent) test harness. It features automatic collection of the test scripts output in @file{.log} files, concurrent execution of tests with @code{make -j}, specification of inter-test dependencies, lazy reruns of tests that have not completed in a prior run, and hard errors for exceptional failures. @anchor{Basics of test metadata} -@vindex TEST_SUITE_LOG @vindex TESTS @cindex @file{.log} files @cindex @file{.trs} files @@ -9768,10 +9767,15 @@ its standard output and its standard error. The @file{.trs} file will contain, among the other things, the results of the test cases run by the script. +@vindex TEST_SUITE_LOG +@vindex IGNORE_SKIPPED_LOGS The parallel test harness will also create a summary log file, @code{TEST_SUITE_LOG}, which defaults to @file{test-suite.log} and requires a @file{.log} suffix. This file depends upon all the @file{.log} and @file{.trs} files created for the test scripts listed in @code{TESTS}. +It contains the output of all tests that failed, encountered a hard error, +succeeded unexpectedly, or -- unless the variable @code{IGNORE_SKIPPED_LOGS} +is set to a non-empty value -- were skipped. @vindex VERBOSE As with the serial harness above, by default one status line is printed @@ -10162,6 +10166,9 @@ exists @emph{before} the test driver is called. Whether the console output should be colorized or not (@pxref{Simple tests and color-tests}, to learn when this option gets activated and when it doesn't). +@item --collect-skipped-logs @{yes|no@} +Whether to include the logs of skipped tests in the global +@file{test-suite.log} file. @item --expect-failure @{yes|no@} Whether the tested program is expected to fail. @item --enable-hard-errors @{yes|no@} @@ -10179,7 +10186,8 @@ be run, and all the following ones are command-line options and arguments for this program. Note that the exact semantics attached to the @option{--color-tests}, -@option{--expect-failure} and @option{--enable-hard-errors} options are +@option{--collect-skipped-logs}, @option{--expect-failure}, and +@option{--enable-hard-errors} options are left up to the individual test drivers. Still, having a behaviour compatible or at least similar to that provided by the default driver is advised, as that would offer a better consistency and a more pleasant diff --git a/lib/am/check.am b/lib/am/check.am index b6fa3aaad..e35ff1e8b 100644 --- a/lib/am/check.am +++ b/lib/am/check.am @@ -172,6 +172,7 @@ am__sh_e_setup = case $$- in *e*) set +e;; esac # Default flags passed to test drivers. am__common_driver_flags = \ --color-tests "$$am__color_tests" \ + $$am__collect_skipped_logs \ --enable-hard-errors "$$am__enable_hard_errors" \ --expect-failure "$$am__expect_failure" @@ -197,6 +198,11 @@ if test -f "./$$f"; then dir=./; \ elif test -f "$$f"; then dir=; \ else dir="$(srcdir)/"; fi; \ tst=$$dir$$f; log='$@'; \ +if test -n '$(IGNORE_SKIPPED_LOGS)'; then \ + am__collect_skipped_logs='--collect-skipped-logs no'; \ +else \ + am__collect_skipped_logs=''; \ +fi; \ if test -n '$(DISABLE_HARD_ERRORS)'; then \ am__enable_hard_errors=no; \ else \ diff --git a/lib/test-driver b/lib/test-driver index 2d45c984a..e04b5a210 100755 --- a/lib/test-driver +++ b/lib/test-driver @@ -1,7 +1,7 @@ #! /bin/sh # test-driver - basic testsuite driver script. -scriptversion=2018-03-07.03; # UTC +scriptversion=2024-06-07.11; # UTC # Copyright (C) 2011-2024 Free Software Foundation, Inc. # @@ -44,6 +44,7 @@ print_usage () Usage: test-driver --test-name NAME --log-file PATH --trs-file PATH [--expect-failure {yes|no}] [--color-tests {yes|no}] + [--collect-skipped-logs {yes|no}] [--enable-hard-errors {yes|no}] [--] TEST-SCRIPT [TEST-SCRIPT-ARGUMENTS] @@ -57,6 +58,7 @@ log_file= # Where to save the output of the test script. trs_file= # Where to save the metadata of the test run. expect_failure=no color_tests=no +collect_skipped_logs=yes enable_hard_errors=yes while test $# -gt 0; do case $1 in @@ -66,6 +68,7 @@ while test $# -gt 0; do --log-file) log_file=$2; shift;; --trs-file) trs_file=$2; shift;; --color-tests) color_tests=$2; shift;; + --collect-skipped-logs) collect_skipped_logs=$2; shift;; --expect-failure) expect_failure=$2; shift;; --enable-hard-errors) enable_hard_errors=$2; shift;; --) shift; break;; @@ -121,7 +124,7 @@ fi case $tweaked_estatus:$expect_failure in 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; 0:*) col=$grn res=PASS recheck=no gcopy=no;; - 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; + 77:*) col=$blu res=SKIP recheck=no gcopy=$collect_skipped_logs;; 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;; *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;; *:*) col=$red res=FAIL recheck=yes gcopy=yes;; diff --git a/t/list-of-tests.mk b/t/list-of-tests.mk index 9f6a08685..bfca492f9 100644 --- a/t/list-of-tests.mk +++ b/t/list-of-tests.mk @@ -815,6 +815,7 @@ t/parallel-tests-fork-bomb.sh \ t/parallel-tests-empty-testlogs.sh \ t/parallel-tests-driver-install.sh \ t/parallel-tests-no-color-in-log.sh \ +t/parallel-tests-without-skipped-logs.sh \ t/parallel-tests-no-spurious-summary.sh \ t/parallel-tests-exit-statuses.sh \ t/parallel-tests-console-output.sh \ diff --git a/t/parallel-tests-without-skipped-logs.sh b/t/parallel-tests-without-skipped-logs.sh new file mode 100644 index 000000000..86c7d9c53 --- /dev/null +++ b/t/parallel-tests-without-skipped-logs.sh @@ -0,0 +1,63 @@ +#! /bin/sh +# Copyright (C) 2011-2024 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Colorized output from the testsuite report shouldn't end up in log files. + +. test-init.sh + +cat >>configure.ac <Makefile.am <<'END' +LOG_COMPILER = $(SHELL) +AUTOMAKE_OPTIONS = color-tests +TESTS = pass fail saltata xpass xfail error +XFAIL_TESTS = xpass xfail +END + +echo 'exit 0' > pass +echo 'exit 0' > xpass +echo 'exit 1' > fail +echo 'exit 1' > xfail +echo 'exit 77' > saltata +echo 'exit 99' > error + +$ACLOCAL +$AUTOCONF +$AUTOMAKE --add-missing + +# Check that IGNORE_SKIPPED_LOGS works when given on the command line. + +./configure +run_make -e FAIL check IGNORE_SKIPPED_LOGS=true +grep saltata test-suite.log && exit 1 + +rm -f test-suite.log + +# Check that IGNORE_SKIPPED_LOGS works when given in the Makefile. + +cat >>Makefile.am <<'END' +IGNORE_SKIPPED_LOGS = true +END + +$AUTOMAKE --add-missing + +./configure +run_make -e FAIL check +grep saltata test-suite.log && exit 1 + +: -- 2.34.1