[RFC PATCH v2 1/7] tests: create optional tests/venv dependency groups

[John Snow]

This patch uses a dummy package and setup.cfg/setup.py files to manage
optional dependency groups for the test venv specification. Now, there's
a core set of dependencies which for now includes just "qemu" (but soon,
qemu.qmp) and a separate, optional 'avocado' group that includes
avocado-framework and pycdlib.

Practical upshot: We install only a minimum of things for the majority
of check-* targets, but allow optional add-ons to be processed when
running avocado tests. This will spare downstreams from having to add
more dependencies than is necessary as a build dependencies when
invoking "make check".

(We also keep both sets of dependencies in one file, which is helpful
for review to ensure that different option groups don't conflict with
one another.)

NOTE: There is a non-fatal caveat introduced by this patch on Ubuntu
20.04 systems; see the subsequent commit "tests: Remove spurious pip
warnings on Ubuntu20.04" for more information.

Signed-off-by: John Snow <jsnow@redhat.com>
---
 tests/Makefile.include | 21 ++++++++++++++-------
 tests/requirements.txt |  6 ------
 tests/setup.cfg        | 20 ++++++++++++++++++++
 tests/setup.py         | 16 ++++++++++++++++
 4 files changed, 50 insertions(+), 13 deletions(-)
 delete mode 100644 tests/requirements.txt
 create mode 100644 tests/setup.cfg
 create mode 100644 tests/setup.py

diff --git a/tests/Makefile.include b/tests/Makefile.include
index 3accb83b132..82c697230e0 100644
--- a/tests/Makefile.include
+++ b/tests/Makefile.include
@@ -81,13 +81,13 @@ clean-tcg: $(CLEAN_TCG_TARGET_RULES)
 
 # Python venv for running tests
 
-.PHONY: check-venv check-avocado check-acceptance 
check-acceptance-deprecated-warning
+.PHONY: check-venv check-venv-avocado check-avocado check-acceptance \
+        check-acceptance-deprecated-warning
 
 # Build up our target list from the filtered list of ninja targets
 TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets)))
 
 TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv
-TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt
 TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results
 TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python3
 ifndef AVOCADO_TESTS
@@ -108,10 +108,16 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
     $(TESTS_PYTHON) -m pip -q --disable-pip-version-check $1, \
     "VENVPIP","$1")
 
-$(TESTS_VENV_DIR): $(TESTS_VENV_REQ)
+# Core dependencies for tests/venv
+$(TESTS_VENV_DIR): $(SRC_PATH)/tests/setup.cfg $(SRC_PATH)/python/setup.cfg
        $(call quiet-command, $(PYTHON) -m venv $@, VENV, $@)
        $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
-       $(call quiet-venv-pip,install -r $(TESTS_VENV_REQ))
+       $(call quiet-venv-pip,install "$(SRC_PATH)/tests/")
+       $(call quiet-command, touch $@)
+
+# Optional avocado dependencies for tests/venv
+$(TESTS_VENV_DIR)/avocado: $(TESTS_VENV_DIR)
+       $(call quiet-venv-pip,install "$(SRC_PATH)/tests/[avocado]")
        $(call quiet-command, touch $@)
 
 $(TESTS_RESULTS_DIR):
@@ -119,6 +125,7 @@ $(TESTS_RESULTS_DIR):
             MKDIR, $@)
 
 check-venv: $(TESTS_VENV_DIR)
+check-venv-avocado: $(TESTS_VENV_DIR)/avocado
 
 FEDORA_31_ARCHES_TARGETS=$(patsubst %-softmmu,%, $(filter 
%-softmmu,$(TARGETS)))
 FEDORA_31_ARCHES_CANDIDATES=$(patsubst 
ppc64,ppc64le,$(FEDORA_31_ARCHES_TARGETS))
@@ -126,16 +133,16 @@ FEDORA_31_ARCHES := x86_64 aarch64 ppc64le s390x
 FEDORA_31_DOWNLOAD=$(filter $(FEDORA_31_ARCHES),$(FEDORA_31_ARCHES_CANDIDATES))
 
 # download one specific Fedora 31 image
-get-vm-image-fedora-31-%: check-venv
+get-vm-image-fedora-31-%: check-venv-avocado
        $(call quiet-command, \
              $(TESTS_PYTHON) -m avocado vmimage get \
              --distro=fedora --distro-version=31 --arch=$*, \
        "AVOCADO", "Downloading avocado tests VM image for $*")
 
 # download all vm images, according to defined targets
-get-vm-images: check-venv $(patsubst %,get-vm-image-fedora-31-%, 
$(FEDORA_31_DOWNLOAD))
+get-vm-images: check-venv-avocado $(patsubst %,get-vm-image-fedora-31-%, 
$(FEDORA_31_DOWNLOAD))
 
-check-avocado: check-venv $(TESTS_RESULTS_DIR) get-vm-images
+check-avocado: check-venv-avocado $(TESTS_RESULTS_DIR) get-vm-images
        $(call quiet-command, \
             $(TESTS_PYTHON) -m avocado \
             --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \
diff --git a/tests/requirements.txt b/tests/requirements.txt
deleted file mode 100644
index 0ba561b6bdf..00000000000
--- a/tests/requirements.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-# Add Python module requirements, one per line, to be installed
-# in the tests/venv Python virtual environment. For more info,
-# refer to: https://pip.pypa.io/en/stable/user_guide/#id1
-# Note that qemu.git/python/ is always implicitly installed.
-avocado-framework==88.1
-pycdlib==1.11.0
diff --git a/tests/setup.cfg b/tests/setup.cfg
new file mode 100644
index 00000000000..263a5de01af
--- /dev/null
+++ b/tests/setup.cfg
@@ -0,0 +1,20 @@
+# This file represents a "dummy" package that expresses
+# the dependencies necessary to run the various python-based
+# test suites in the qemu.git tree.
+[metadata]
+name = qemu.dummy_tests
+version = 0.0.0
+description = qemu.git testing dummy package
+
+[options]
+packages =
+python_requires = >= 3.6
+install_requires =
+    qemu  # Note: A special setup script will always install 'qemu' from
+          # qemu.git/python. It doesn't have a meaningful/real version.
+
+[options.extras_require]
+# Extra dependencies required by the avocado tests.
+avocado =
+    avocado-framework<89.0,>=88.1
+    pycdlib<2.0,>=1.11.0
diff --git a/tests/setup.py b/tests/setup.py
new file mode 100644
index 00000000000..c99fe5d1725
--- /dev/null
+++ b/tests/setup.py
@@ -0,0 +1,16 @@
+# This setup file is just-enough-config to allow pip to bootstrap a
+# testing environment. It is not meant to be executed directly.
+# See also: setup.cfg
+
+import setuptools
+import pkg_resources
+
+
+def main():
+    # 
https://medium.com/@daveshawley/safely-using-setup-cfg-for-metadata-1babbe54c108
+    pkg_resources.require('setuptools>=39.2')
+    setuptools.setup()
+
+
+if __name__ == '__main__':
+    main()
-- 
2.34.3