>From 9a120ae0f1791410e41951982ea3f44ad602dfec Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Sat, 23 Apr 2022 11:36:55 +0200 Subject: [PATCH v3 003/150] python-build-system: Use PEP 517-compatible builds. This is effectively an entire rewrite of python-build-system. It supports all PEP 517-compatible build backends. * gnu/packages/python-commencement.scm: New file containing new Python toolchain package(s). * gnu/local.mk (GNU_SYSTEM_MODULES): Add it. * gnu/packages/python-build.scm (python-setuptools-bootstrap): New variable. (python2-setuptools-bootstrap): New variable. (python-wheel): Break bootstrap cycle. (python-wheel-bootstrap): New variable. (python2-wheel-bootstrap): New variable. * gnu/packages/python.scm (python-2.7): Do not install setuptools and pip. * guix/build-system/python.scm (%python-build-system-modules): Use (guix build json). (default-python): Default to python-toolchain-for-build. (lower): Add default wheel output, remove test-target and use-setuptools? flags, add build-backend, test-backend and test-flags. * guix/build/python-build-system.scm: Rewrite build system. --- gnu/local.mk | 1 + gnu/packages/python-build.scm | 104 ++++++- gnu/packages/python.scm | 2 +- guix/build-system/python.scm | 26 +- guix/build/python-build-system.scm | 436 +++++++++++++++++++---------- 5 files changed, 397 insertions(+), 172 deletions(-) diff --git a/gnu/local.mk b/gnu/local.mk index 0e721236d9..4298a2621b 100644 --- a/gnu/local.mk +++ b/gnu/local.mk @@ -495,6 +495,7 @@ GNU_SYSTEM_MODULES = \ %D%/packages/python.scm \ %D%/packages/python-build.scm \ %D%/packages/python-check.scm \ + %D%/packages/python-commencement.scm \ %D%/packages/python-compression.scm \ %D%/packages/python-crypto.scm \ %D%/packages/python-science.scm \ diff --git a/gnu/packages/python-build.scm b/gnu/packages/python-build.scm index f167c3953f..e23382972c 100644 --- a/gnu/packages/python-build.scm +++ b/gnu/packages/python-build.scm @@ -29,7 +29,9 @@ (define-module (gnu packages python-build) #:use-module (guix gexp) #:use-module (guix download) #:use-module (guix git-download) - #:use-module (guix packages)) + #:use-module (guix packages) + #:use-module (gnu packages) + #:use-module (gnu packages python)) ;;; Commentary: ;;; @@ -58,11 +60,23 @@ (define-public python-setuptools ;; TODO: Find some way to build them ourself so we can include them. (for-each delete-file (find-files "setuptools" "^(cli|gui).*\\.exe$")) #t)))) + (outputs '("out" "wheel")) (build-system python-build-system) ;; FIXME: Tests require pytest, which itself relies on setuptools. ;; One could bootstrap with an internal untested setuptools. (arguments - `(#:tests? #f)) + `(#:tests? #f + #:python ,python-wrapper ; Break cycle with default build system’s setuptools dependency. + #:phases (modify-phases %standard-phases + ;; Use this setuptools’ sources to bootstrap themselves. + (add-before 'build 'set-PYTHONPATH + (lambda _ + (format #t "current working dir ~s~%" (getcwd)) + (setenv "GUIX_PYTHONPATH" + (string-append ".:" (getenv "GUIX_PYTHONPATH"))) + #t))))) + ;; Required to build wheels. + (propagated-inputs `(("python-wheel" ,python-wheel))) (home-page "https://pypi.org/project/setuptools/") (synopsis "Library designed to facilitate packaging Python projects") @@ -83,6 +97,14 @@ (define-public python-setuptools license:bsd-2)) (properties `((python2-variant . ,(delay python2-setuptools)))))) +;; Break loop between python-setuptools and python-wheel. +(define-public python-setuptools-bootstrap + (package + (inherit python-setuptools) + (name "python-setuptools-bootstrap") + (propagated-inputs `(("python-wheel" ,python-wheel-bootstrap))) + (properties `((python2-variant . ,(delay python2-setuptools-bootstrap)))))) + ;; Newer versions of setuptools no longer support Python 2. (define-public python2-setuptools (package @@ -107,9 +129,10 @@ (define-public python2-setuptools ;; FIXME: Tests require pytest, which itself relies on setuptools. ;; One could bootstrap with an internal untested setuptools. (arguments - `(#:tests? #f)) - (native-inputs - (list unzip)) + `(#:tests? #f + #:python ,python-2 ; Break loop to python2-toolchain-for-build + )) + (propagated-inputs `(("python2-wheel" ,python2-wheel))) (home-page "https://pypi.org/project/setuptools/") (synopsis "Library designed to facilitate packaging Python projects") @@ -129,6 +152,12 @@ (define-public python2-setuptools license:asl2.0 ; packaging is dual ASL2/BSD-2 license:bsd-2)))) +(define-public python2-setuptools-bootstrap + (package + (inherit python2-setuptools) + (name "python2-setuptools-bootstrap") + (propagated-inputs `(("python2-wheel" ,python2-wheel-bootstrap))))) + (define-public python-wheel (package (name "python-wheel") @@ -142,10 +171,8 @@ (define-public python-wheel "1bbga5i49rj1cwi4sjpkvfhl1f8vl9lfky2lblsy768nk4wp5vz2")))) (build-system python-build-system) (arguments - ;; FIXME: The test suite runs "python setup.py bdist_wheel", which in turn - ;; fails to find the newly-built bdist_wheel library, even though it is - ;; available on PYTHONPATH. What search path is consulted by setup.py? - '(#:tests? #f)) + `(#:python ,python-wrapper)) ; Break cycle with python-toolchain-for-build. + (native-inputs `(("python-setuptools" ,python-setuptools-bootstrap))) (home-page "https://bitbucket.org/pypa/wheel/") (synopsis "Format for built Python packages") (description @@ -158,8 +185,65 @@ (define-public python-wheel installed with a newer @code{pip} or with wheel's own command line utility.") (license license:expat))) +(define-public python-wheel-bootstrap + (package + (inherit python-wheel) + (name "python-wheel-bootstrap") + (build-system copy-build-system) + (native-inputs '()) ; Break cycle to setuptools. + (arguments + `(#:install-plan + ;; XXX: Do not hard-code Python version. + '(("src/wheel" "lib/python3.9/site-packages/wheel")) + #:phases + (modify-phases %standard-phases + ;; Add metadata for setuptools, so it will find the wheel-building code. + (add-after 'install 'install-metadata + (lambda* (#:key outputs #:allow-other-keys) + (let* ((out (assoc-ref outputs "out")) + (site-dir (string-append out "/lib/python3.9/site-packages")) + (metadata-dir (string-append site-dir "/wheel.egg-info"))) + (mkdir-p metadata-dir) + (call-with-output-file (string-append metadata-dir "/entry_points.txt") + (lambda (port) + (format port "~ + [distutils.commands]~@ + bdist_wheel = wheel.bdist_wheel:bdist_wheel~%"))))))))) + (properties `((python2-variant . ,(delay python2-wheel-bootstrap)))))) + (define-public python2-wheel - (package-with-python2 python-wheel)) + (package + (inherit (package-with-python2 python-wheel)) + (arguments `(#:python ,python-2)))) + +(define-public python2-wheel-bootstrap + (package + (inherit python2-wheel) + (name "python2-wheel-bootstrap") + (build-system copy-build-system) + (native-inputs '()) ; Break cycle to setuptools. + (arguments + `(#:install-plan + ;; XXX: Do not hard-code Python version. + '(("src/wheel" "lib/python2.7/site-packages/wheel")) + #:phases + (modify-phases %standard-phases + ;; Add metadata for setuptools, so it will find the wheel-building code. + (add-after 'install 'install-metadata + (lambda* (#:key outputs #:allow-other-keys) + (let* ((out (assoc-ref outputs "out")) + (site-dir (string-append out "/lib/python2.7/site-packages")) + (metadata-dir (string-append site-dir "/wheel.egg-info"))) + (mkdir-p metadata-dir) + (call-with-output-file (string-append metadata-dir "/entry_points.txt") + (lambda (port) + (format port "~ + [distutils.commands]~@ + bdist_wheel = wheel.bdist_wheel:bdist_wheel~%"))) + (call-with-output-file (string-append metadata-dir "/PKG-INFO") + (lambda (port) + (format port "~ + Version: ~a" (version)))))))))))) ;;; XXX: Not really at home, but this seems the best place to prevent circular ;;; module dependencies. diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm index 3bc3346c21..4399d30aad 100644 --- a/gnu/packages/python.scm +++ b/gnu/packages/python.scm @@ -183,7 +183,7 @@ (define-public python-2.7 (list "--enable-shared" ;allow embedding "--with-system-expat" ;for XML support "--with-system-ffi" ;build ctypes - "--with-ensurepip=install" ;install pip and setuptools + "--with-ensurepip=no" ;do not install pip and setuptools "--with-computed-gotos" ;main interpreter loop optimization "--enable-unicode=ucs4" diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm index efade6f74b..aad861d278 100644 --- a/guix/build-system/python.scm +++ b/guix/build-system/python.scm @@ -62,18 +62,19 @@ (define* (pypi-uri name version #:optional (extension ".tar.gz")) (define %python-build-system-modules ;; Build-side modules imported by default. `((guix build python-build-system) + (guix build json) ,@%gnu-build-system-modules)) (define (default-python) "Return the default Python package." ;; Lazily resolve the binding to avoid a circular dependency. - (let ((python (resolve-interface '(gnu packages python)))) - (module-ref python 'python-wrapper))) + (let ((python (resolve-interface '(gnu packages python-commencement)))) + (module-ref python 'python-toolchain-for-build))) (define (default-python2) "Return the default Python 2 package." - (let ((python (resolve-interface '(gnu packages python)))) - (module-ref python 'python-2))) + (let ((python (resolve-interface '(gnu packages python-commencement)))) + (module-ref python 'python2-toolchain-for-build))) (define sanity-check.py ;; The script used to validate the installation of a Python package. @@ -165,26 +166,26 @@ (define private-keywords (build-inputs `(("python" ,python) ("sanity-check.py" ,(local-file sanity-check.py)) ,@native-inputs)) - (outputs outputs) + (outputs (append outputs '(wheel))) (build python-build) (arguments (strip-keyword-arguments private-keywords arguments))))) (define* (python-build name inputs #:key source (tests? #t) - (test-target "test") - (use-setuptools? #t) (configure-flags ''()) + (build-backend #f) + (test-backend #f) + (test-flags #f) (phases '%standard-phases) - (outputs '("out")) + (outputs '("out" "wheel")) (search-paths '()) (system (%current-system)) (guile #f) (imported-modules %python-build-system-modules) (modules '((guix build python-build-system) (guix build utils)))) - "Build SOURCE using PYTHON, and with INPUTS. This assumes that SOURCE -provides a 'setup.py' file as its build system." + "Build SOURCE using PYTHON, and with INPUTS." (define build (with-imported-modules imported-modules #~(begin @@ -194,9 +195,10 @@ (define build #~(python-build #:name #$name #:source #+source #:configure-flags #$configure-flags - #:use-setuptools? #$use-setuptools? #:system #$system - #:test-target #$test-target + #:build-backend #$build-backend + #:test-backend #$test-backend + #:test-flags #$test-flags #:tests? #$tests? #:phases #$(if (pair? phases) (sexp->gexp phases) diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 08871f60cd..15cbdd4e7c 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -30,11 +30,16 @@ (define-module (guix build python-build-system) #:use-module ((guix build gnu-build-system) #:prefix gnu:) #:use-module (guix build utils) + #:use-module (guix build json) #:use-module (ice-9 match) #:use-module (ice-9 ftw) #:use-module (ice-9 format) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 regex) #:use-module (srfi srfi-1) #:use-module (srfi srfi-26) + #:use-module (srfi srfi-34) + #:use-module (srfi srfi-35) #:export (%standard-phases add-installed-pythonpath site-packages @@ -43,96 +48,45 @@ (define-module (guix build python-build-system) ;; Commentary: ;; -;; Builder-side code of the standard Python package build procedure. +;; PEP 517-compatible build system for Python packages. ;; +;; PEP 517 mandates the use of a TOML file called pyproject.toml at the +;; project root, describing build and runtime dependencies, as well as the +;; build system, which can be different from setuptools. This module uses +;; that file to extract the build system used and call its wheel-building +;; entry point build_wheel (see 'build). setuptools’ wheel builder is +;; used as a fallback if either no pyproject.toml exists or it does not +;; declare a build-system. It supports config_settings through the +;; standard #:configure-flags argument. ;; -;; Backgound about the Python installation methods +;; This wheel, which is just a ZIP file with a file structure defined +;; by PEP 427 (https://www.python.org/dev/peps/pep-0427/), is then unpacked +;; and its contents are moved to the appropriate locations in 'install. ;; -;; In Python there are different ways to install packages: distutils, -;; setuptools, easy_install and pip. All of these are sharing the file -;; setup.py, introduced with distutils in Python 2.0. The setup.py file can be -;; considered as a kind of Makefile accepting targets (or commands) like -;; "build" and "install". As of autumn 2016 the recommended way to install -;; Python packages is using pip. +;; Then entry points, as defined by the PyPa Entry Point Specification +;; (https://packaging.python.org/specifications/entry-points/) are read +;; from a file called entry_points.txt in the package’s site-packages +;; subdirectory and scripts are written to bin/. These are not part of a +;; wheel and expected to be created by the installing utility. ;; -;; For both distutils and setuptools, running "python setup.py install" is the -;; way to install Python packages. With distutils the "install" command -;; basically copies all packages into /lib/pythonX.Y/site-packages. -;; -;; Some time later "setuptools" was established to enhance distutils. To use -;; setuptools, the developer imports setuptools in setup.py. When importing -;; setuptools, the original "install" command gets overwritten by setuptools' -;; "install" command. -;; -;; The command-line tools easy_install and pip are both capable of finding and -;; downloading the package source from PyPI (the Python Package Index). Both -;; of them import setuptools and execute the "setup.py" file under their -;; control. Thus the "setup.py" behaves as if the developer had imported -;; setuptools within setup.py - even is still using only distutils. -;; -;; Setuptools' "install" command (to be more precise: the "easy_install" -;; command which is called by "install") will put the path of the currently -;; installed version of each package and it's dependencies (as declared in -;; setup.py) into an "easy-install.pth" file. In Guix each packages gets its -;; own "site-packages" directory and thus an "easy-install.pth" of its own. -;; To avoid conflicts, the python build system renames the file to -;; .pth in the phase rename-pth-file. To ensure that Python will -;; process the .pth file, easy_install also creates a basic "site.py" in each -;; "site-packages" directory. The file is the same for all packages, thus -;; there is no need to rename it. For more information about .pth files and -;; the site module, please refere to -;; https://docs.python.org/3/library/site.html. -;; -;; The .pth files contain the file-system paths (pointing to the store) of all -;; dependencies. So the dependency is hidden in the .pth file but is not -;; visible in the file-system. Now if packages A and B both required packages -;; P, but in different versions, Guix will not detect this when installing -;; both A and B to a profile. (For details and example see -;; https://lists.gnu.org/archive/html/guix-devel/2016-10/msg01233.html.) -;; -;; Pip behaves a bit different then easy_install: it always executes -;; "setup.py" with the option "--single-version-externally-managed" set. This -;; makes setuptools' "install" command run the original "install" command -;; instead of the "easy_install" command, so no .pth file (and no site.py) -;; will be created. The "site-packages" directory only contains the package -;; and the related .egg-info directory. -;; -;; This is exactly what we need for Guix and this is what we mimic in the -;; install phase below. -;; -;; As a draw back, the magic of the .pth file of linking to the other required -;; packages is gone and these packages have now to be declared as -;; "propagated-inputs". -;; -;; Note: Importing setuptools also adds two sub-commands: "install_egg_info" -;; and "install_scripts". These sub-commands are executed even if -;; "--single-version-externally-managed" is set, thus the .egg-info directory -;; and the scripts defined in entry-points will always be created. - - -(define setuptools-shim - ;; Run setup.py with "setuptools" being imported, which will patch - ;; "distutils". This is needed for packages using "distutils" instead of - ;; "setuptools" since the former does not understand the - ;; "--single-version-externally-managed" flag. - ;; Python code taken from pip 9.0.1 pip/utils/setuptools_build.py - (string-append - "import setuptools, tokenize;__file__='setup.py';" - "f=getattr(tokenize, 'open', open)(__file__);" - "code=f.read().replace('\\r\\n', '\\n');" - "f.close();" - "exec(compile(code, __file__, 'exec'))")) - -(define (call-setuppy command params use-setuptools?) - (if (file-exists? "setup.py") - (begin - (format #t "running \"python setup.py\" with command ~s and parameters ~s~%" - command params) - (if use-setuptools? - (apply invoke "python" "-c" setuptools-shim - command params) - (apply invoke "python" "./setup.py" command params))) - (error "no setup.py found"))) +;; Caveats: +;; - There is no support for in-tree build backends. + +;; Base error type. +(define-condition-type &python-build-error &error + python-build-error?) + +;; Raised when 'check cannot find a valid test system in the inputs. +(define-condition-type &test-system-not-found &python-build-error + test-system-not-found?) + +;; Raised when multiple wheels are created by 'build. +(define-condition-type &cannot-extract-multiple-wheels &python-build-error + cannot-extract-multiple-wheels?) + +;; Raised, when no wheel has been built by the build system. +(define-condition-type &no-wheels-built &python-build-error + no-wheels-built?) (define* (sanity-check #:key tests? inputs outputs #:allow-other-keys) "Ensure packages depending on this package via setuptools work properly, @@ -143,25 +97,83 @@ (define* (sanity-check #:key tests? inputs outputs #:allow-other-keys) (with-directory-excursion "/tmp" (invoke "python" sanity-check.py (site-packages inputs outputs))))) -(define* (build #:key use-setuptools? #:allow-other-keys) +(define* (build #:key outputs build-backend configure-flags #:allow-other-keys) "Build a given Python package." - (call-setuppy "build" '() use-setuptools?) - #t) -(define* (check #:key tests? test-target use-setuptools? #:allow-other-keys) + (define (pyproject.toml->build-backend file) + "Look up the build backend in a pyproject.toml file." + (call-with-input-file file + (lambda (in) + (let loop ((line (read-line in 'concat))) + (if (eof-object? line) + #f + (let ((m (string-match "build-backend = [\"'](.+)[\"']" line))) + (if m (match:substring m 1) + (loop (read-line in 'concat))))))))) + + (let* ((wheel-output (assoc-ref outputs "wheel")) + (wheel-dir (if wheel-output wheel-output "dist")) + ;; There is no easy way to get data from Guile into Python via + ;; s-expressions, but we have JSON serialization already, which Python + ;; also supports out-of-the-box. + (config-settings (call-with-output-string (cut write-json configure-flags <>))) + ;; python-setuptools’ default backend supports setup.py *and* + ;; pyproject.toml. Allow overriding this automatic detection via + ;; build-backend. + (auto-build-backend (if (file-exists? "pyproject.toml") + (pyproject.toml->build-backend "pyproject.toml") + #f)) + ;; Use build system detection here and not in importer, because a) we + ;; have alot of legacy packages and b) the importer cannot update arbitrary + ;; fields in case a package switches its build system. + (use-build-backend (or + build-backend + auto-build-backend + "setuptools.build_meta"))) + (format #t "Using '~a' to build wheels, auto-detected '~a', override '~a'.~%" + use-build-backend auto-build-backend build-backend) + (mkdir-p wheel-dir) + ;; Call the PEP 517 build function, which drops a .whl into wheel-dir. + (invoke "python" "-c" "import sys, importlib, json +config_settings = json.loads (sys.argv[3]) +builder = importlib.import_module(sys.argv[1]) +builder.build_wheel(sys.argv[2], config_settings=config_settings)" + use-build-backend wheel-dir config-settings))) + +(define* (check #:key inputs outputs tests? test-backend test-flags #:allow-other-keys) "Run the test suite of a given Python package." (if tests? - ;; Running `setup.py test` creates an additional .egg-info directory in - ;; build/lib in some cases, e.g. if the source is in a sub-directory - ;; (given with `package_dir`). This will by copied to the output, too, - ;; so we need to remove. - (let ((before (find-files "build" "\\.egg-info$" #:directories? #t))) - (call-setuppy test-target '() use-setuptools?) - (let* ((after (find-files "build" "\\.egg-info$" #:directories? #t)) - (inter (lset-difference string=? after before))) - (for-each delete-file-recursively inter))) - (format #t "test suite not run~%")) - #t) + ;; Unfortunately with PEP 517 there is no common method to specify test + ;; systems. Guess test system based on inputs instead. + (let* ((pytest (which "pytest")) + (nosetests (which "nosetests")) + (nose2 (which "nose2")) + (have-setup-py (file-exists? "setup.py")) + (use-test-backend + (or + test-backend + ;; Prefer pytest + (if pytest 'pytest #f) + (if nosetests 'nose #f) + (if nose2 'nose2 #f) + ;; But fall back to setup.py, which should work for most + ;; packages. XXX: would be nice not to depend on setup.py here? fails + ;; more often than not to find any tests at all. Maybe we can run + ;; `python -m unittest`? + (if have-setup-py 'setup.py #f)))) + (format #t "Using ~a~%" use-test-backend) + (match use-test-backend + ('pytest + (apply invoke (cons pytest (or test-flags '("-vv"))))) + ('nose + (apply invoke (cons nosetests (or test-flags '("-v"))))) + ('nose2 + (apply invoke (cons nose2 (or test-flags '("-v" "--pretty-assert"))))) + ('setup.py + (apply invoke (append '("python" "setup.py") (or test-flags '("test" "-v"))))) + ;; The developer should explicitly disable tests in this case. + (else (raise (condition (&test-system-not-found)))))) + (format #t "test suite not run~%"))) (define (python-version python) (let* ((version (last (string-split python #\-))) @@ -196,33 +208,175 @@ (define* (add-install-to-path #:key outputs #:allow-other-keys) "/bin:" (getenv "PATH")))) -(define* (install #:key inputs outputs (configure-flags '()) use-setuptools? - #:allow-other-keys) - "Install a given Python package." - (let* ((out (python-output outputs)) +(define* (install #:key inputs outputs (configure-flags '()) #:allow-other-keys) + "Install a wheel file according to PEP 427" + ;; See https://www.python.org/dev/peps/pep-0427/#installing-a-wheel-distribution-1-0-py32-none-any-whl + (let* ((site-dir (site-packages inputs outputs)) + (python (assoc-ref inputs "python")) + (out (assoc-ref outputs "out"))) + (define (extract file) + "Extract wheel (ZIP file) into site-packages directory" + ;; Use Python’s zipfile to avoid extra dependency + (invoke "python" "-m" "zipfile" "-e" file site-dir)) + + (define python-hashbang + (string-append "#!" python "/bin/python")) + + (define* (merge-directories source destination #:optional (post-move #f)) + "Move all files in SOURCE into DESTINATION, merging the two directories." + (format #t "Merging directory ~a into ~a~%" source destination) + (for-each + (lambda (file) + (format #t "~a/~a -> ~a/~a~%" source file destination file) + (mkdir-p destination) + (rename-file + (string-append source "/" file) + (string-append destination "/" file)) + (when post-move + (post-move file))) + (scandir source (negate (cut member <> '("." ".."))))) + (rmdir source)) + + (define (expand-data-directory directory) + "Move files from all .data subdirectories to their respective +destinations." + ;; Python’s distutils.command.install defines this mapping from source to + ;; destination mapping. + (let ((source (string-append directory "/scripts")) + (destination (string-append out "/bin"))) + (when (file-exists? source) + (merge-directories + source + destination + (lambda (f) + (let ((dest-path (string-append destination "/" f))) + (chmod dest-path #o755) + (substitute* dest-path (("#!python") python-hashbang))))))) + ;; data can create arbitrary directory structures. Most commonly + ;; it is used for share/. + (let ((source (string-append directory "/data")) + (destination out)) + (when (file-exists? source) + (merge-directories source destination))) + (let* ((distribution (car (string-split (basename directory) #\-))) + (source (string-append directory "/headers")) + (destination (string-append out "/include/python" (python-version python) "/" distribution))) + (when (file-exists? source) + (merge-directories source destination)))) + + (define (list-directories base predicate) + ;; Cannot use find-files here, because it’s recursive. + (scandir + base + (lambda (name) + (let ((stat (lstat (string-append base "/" name)))) + (and + (not (member name '("." ".."))) + (eq? (stat:type stat) 'directory) + (predicate name stat)))))) + + (let* ((wheel-output (assoc-ref outputs "wheel")) + (wheel-dir (if wheel-output wheel-output "dist")) + (wheels (map (cut string-append wheel-dir "/" <>) + (scandir wheel-dir (cut string-suffix? ".whl" <>))))) + (cond + ((> (length wheels) 1) ; This code does not support multiple wheels + ; yet, because their outputs would have to be + ; merged properly. + (raise (condition (&cannot-extract-multiple-wheels)))) + ((= (length wheels) 0) + (raise (condition (&no-wheels-built))))) + (for-each extract wheels)) + (let ((datadirs (map + (cut string-append site-dir "/" <>) + (list-directories site-dir (file-name-predicate "\\.data$"))))) + (for-each (lambda (directory) + (expand-data-directory directory) + (rmdir directory)) + datadirs)))) + +(define* (compile-bytecode #:key inputs outputs (configure-flags '()) #:allow-other-keys) + "Compile installed byte-code in site-packages." + (let* ((site-dir (site-packages inputs outputs)) (python (assoc-ref inputs "python")) (major-minor (map string->number (take (string-split (python-version python) #\.) 2))) (<3.7? (match major-minor ((major minor) - (or (< major 3) (and (= major 3) (< minor 7)))))) - (params (append (list (string-append "--prefix=" out) - "--no-compile") - (if use-setuptools? - ;; distutils does not accept these flags - (list "--single-version-externally-managed" - "--root=/") - '()) - configure-flags))) - (call-setuppy "install" params use-setuptools?) - ;; Rather than produce potentially non-reproducible .pyc files on Pythons - ;; older than 3.7, whose 'compileall' module lacks the - ;; '--invalidation-mode' option, do not generate any. - (unless <3.7? - (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash" - out)))) - -(define* (wrap #:key inputs outputs #:allow-other-keys) + (or (< major 3) (and (= major 3) (< minor 7))))))) + (if <3.7? + ;; These versions don’t have the hash invalidation modes and do + ;; not produce reproducible bytecode files. + (format #t "Skipping bytecode compilation for Python version ~a < 3.7~%" (python-version python)) + (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash" site-dir)))) + +(define* (create-entrypoints #:key inputs outputs (configure-flags '()) #:allow-other-keys) + "Implement Entry Points Specification +(https://packaging.python.org/specifications/entry-points/) by PyPa, +which creates runnable scripts in bin/ from entry point specification +file entry_points.txt. This is necessary, because wheels do not contain +these binaries and installers are expected to create them." + + (define (entry-points.txt->entry-points file) + "Specialized parser for Python configfile-like files, in particular +entry_points.txt. Returns a list of console_script and gui_scripts +entry points." + (call-with-input-file file + (lambda (in) + (let loop ((line (read-line in)) + (inside #f) + (result '())) + (if (eof-object? line) + result + (let* ((group-match (string-match "^\\[(.+)\\]$" line)) + (group-name (if group-match (match:substring group-match 1) #f)) + (next-inside + (if (not group-name) + inside + (or + (string=? group-name "console_scripts") + (string=? group-name "gui_scripts")))) + (item-match (string-match "^([^ =]+)\\s*=\\s*([^:]+):(.+)$" line))) + (if (and inside item-match) + (loop (read-line in) next-inside (cons (list + (match:substring item-match 1) + (match:substring item-match 2) + (match:substring item-match 3)) + result)) + (loop (read-line in) next-inside result)))))))) + + (define (create-script path name module function) + "Create a Python script from an entry point’s NAME, MODULE and + FUNCTION and return write it to PATH/NAME." + (let ((interpreter (which "python")) + (file-path (string-append path "/" name))) + (format #t "Creating entry point for '~a.~a' at '~a'.~%" module function + file-path) + (call-with-output-file file-path + (lambda (port) + ;; Technically the script could also include search-paths, + ;; but having a generic 'wrap phases also handles manually + ;; written entry point scripts. + (format port "#!~a +# Auto-generated entry point script. +import sys +import ~a as mod +sys.exit (mod.~a ())~%" interpreter module function))) + (chmod file-path #o755))) + + (let* ((site-dir (site-packages inputs outputs)) + (out (assoc-ref outputs "out")) + (bin-dir (string-append out "/bin")) + (entry-point-files (find-files site-dir "^entry_points.txt$"))) + (mkdir-p bin-dir) + (for-each + (lambda (f) + (for-each + (lambda (ep) (apply create-script (cons bin-dir ep))) + (entry-points.txt->entry-points f))) + entry-point-files))) + +(define* (wrap #:key inputs outputs search-paths #:allow-other-keys) (define (list-of-files dir) (find-files dir (lambda (file stat) (and (eq? 'regular (stat:type stat)) @@ -250,29 +404,11 @@ (define (sh) (force %sh)) files))) bindirs))) -(define* (rename-pth-file #:key name inputs outputs #:allow-other-keys) - "Rename easy-install.pth to NAME.pth to avoid conflicts between packages -installed with setuptools." - ;; Even if the "easy-install.pth" is not longer created, we kept this phase. - ;; There still may be packages creating an "easy-install.pth" manually for - ;; some good reason. - (let* ((site-packages (site-packages inputs outputs)) - (easy-install-pth (string-append site-packages "/easy-install.pth")) - (new-pth (string-append site-packages "/" name ".pth"))) - (when (file-exists? easy-install-pth) - (rename-file easy-install-pth new-pth)))) - -(define* (ensure-no-mtimes-pre-1980 #:rest _) - "Ensure that there are no mtimes before 1980-01-02 in the source tree." - ;; Rationale: patch-and-repack creates tarballs with timestamps at the POSIX - ;; epoch, 1970-01-01 UTC. This causes problems with Python packages, - ;; because Python eggs are ZIP files, and the ZIP format does not support - ;; timestamps before 1980. - (let ((early-1980 315619200)) ; 1980-01-02 UTC - (ftw "." (lambda (file stat flag) - (unless (<= early-1980 (stat:mtime stat)) - (utime file early-1980 early-1980)) - #t)))) +(define* (set-SOURCE-DATE-EPOCH #:rest _) + "Set the 'SOURCE_DATE_EPOCH' environment variable. This is used by tools +that incorporate timestamps as a way to tell them to use a fixed timestamp. +See https://reproducible-builds.org/specs/source-date-epoch/." + (setenv "SOURCE_DATE_EPOCH" "315619200")) ;; python-wheel respects this variable and sets pre-1980 times on files in zip files, which is unsupported (define* (enable-bytecode-determinism #:rest _) "Improve determinism of pyc files." @@ -299,11 +435,11 @@ (define %standard-phases ;; prefix directory. The check phase is moved after the installation phase ;; to ease testing the built package. (modify-phases gnu:%standard-phases - (add-after 'unpack 'ensure-no-mtimes-pre-1980 ensure-no-mtimes-pre-1980) - (add-after 'ensure-no-mtimes-pre-1980 'enable-bytecode-determinism + (add-after 'unpack 'enable-bytecode-determinism enable-bytecode-determinism) (add-after 'enable-bytecode-determinism 'ensure-no-cythonized-files ensure-no-cythonized-files) + (replace 'set-SOURCE-DATE-EPOCH set-SOURCE-DATE-EPOCH) (delete 'bootstrap) (delete 'configure) ;not needed (replace 'build build) @@ -313,9 +449,11 @@ (define %standard-phases (add-after 'add-install-to-pythonpath 'add-install-to-path add-install-to-path) (add-after 'add-install-to-path 'wrap wrap) + ;; must be before tests, so they can use installed packages’ entry points. + (add-before 'wrap 'create-entrypoints create-entrypoints) (add-after 'wrap 'check check) (add-after 'check 'sanity-check sanity-check) - (add-before 'strip 'rename-pth-file rename-pth-file))) + (add-before 'check 'compile-bytecode compile-bytecode))) (define* (python-build #:key inputs (phases %standard-phases) #:allow-other-keys #:rest args) -- 2.35.1