From cb72f9a773e0931ee3758c851d96007ded034e4c Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Thu, 21 Jan 2021 23:26:01 -0500 Subject: gnu: python: Replace PYTHONPATH by GUIX_PYTHONPATH. Using PYTHONPATH as a mean to discover the Python packages had the following issues: 1. It is not versioned, so different versions of Python would clash if installed in a shared profile. 2. It would interfere with the host Python site on foreign distributions, sometimes preventing a a user to login their GDM session (!). 3. It would take precedence over user installed Python packages installed through pip. 4. It would leak into Python virtualenvs, which are supposed to create isolated Python environments. This changes fixes the above issues by making use of a sitecustomize.py module. The newly introduced GUIX_PYTHONPATH environment variable is read from the environment, filtered for the current Python version of the interpreter, and spliced in 'sys.path' just before Python's own site location, which provides the expected behavior. * gnu/packages/aux-files/python/sitecustomize.py: New file. * Makefile.am: Register it. * gnu/packages/python.scm (customize-site) (guix-pythonpath-search-path): New procedures. (python-2.7)[phases]{install-sitecustomize.py}: New phase. [native-inputs]{sitecustomize.py}: New input. [native-search-paths]: Replace PYTHONPATH with GUIX_PYTHONPATH. (python-3.9)[native-search-paths]: Likewise. [phases]{install-sitecustomize}: Override with correct version. [native-search-paths]: Replace PYTHONPATH with GUIX_PYTHONPATH. * gnu/packages/commencement.scm (python-boot0): [phases]{install-sitecustomize}: Likewise. [native-inputs]{sitecustomize.py}: New input. [native-search-paths]: Replace PYTHONPATH with GUIX_PYTHONPATH. * guix/build/python-build-system.scm (site-packages): Do not add a trailing '/'. squash! gnu: python: Replace PYTHONPATH by GUIX_PYTHONPATH. --- gnu/packages/aux-files/python/sitecustomize.py | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 gnu/packages/aux-files/python/sitecustomize.py (limited to 'gnu/packages/aux-files/python') diff --git a/gnu/packages/aux-files/python/sitecustomize.py b/gnu/packages/aux-files/python/sitecustomize.py new file mode 100644 index 00000000000..65d3c7d5546 --- /dev/null +++ b/gnu/packages/aux-files/python/sitecustomize.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# GNU Guix --- Functional package management for GNU +# Copyright © 2021 Maxim Cournoyer +# +# This file is part of GNU Guix. +# +# GNU Guix is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or (at +# your option) any later version. +# +# GNU Guix is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNU Guix. If not, see . + +import os +import sys + +python_root = os.path.realpath(sys.executable).split('/bin/')[0] +major_minor = '{}.{}'.format(sys.version_info[0], sys.version_info[1]) +site_packages_prefix = 'lib/python' + major_minor + '/site-packages' +python_site = python_root + '/' + site_packages_prefix + +try: + all_sites_raw = os.environ['GUIX_PYTHONPATH'].split(':') +except KeyError: + all_sites_raw = [] +# Normalize paths, otherwise a trailing slash would cause it to not match. +all_sites_norm = [os.path.normpath(p) for p in all_sites_raw] +matching_sites = [p for p in all_sites_norm + if p.endswith(site_packages_prefix)] + +# Insert sites matching the current version into sys.path, right before +# Python's own site. +sys_path_absolute = [os.path.realpath(p) for p in sys.path] +index = sys_path_absolute.index(python_site) +sys.path = sys.path[:index] + matching_sites + sys.path[index:] -- cgit v1.3 From 09448c0994390697e876db235a3b773311795238 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Sun, 3 Jan 2021 10:30:29 +0100 Subject: build/python: Add a sanity check phase. Add a new phase validating the usability of installed Python packages. * gnu/packages/aux-files/python/sanity-check.py: New file. * Makefile.am (AUX_FILES): Register it. * guix/build-system/python.scm (sanity-check.py): New variable. (lower): Add the script as an implicit input. * guix/build/python-build-system.scm: Remove trailing #t. (sanity-check): New phase. (%standard-phases): Use it. * tests/builders.scm: (make-python-dummy) (dummy-ok, dummy-dummy-nosetuptools, dummy-fail-requirements) (dummy-fail-import, dummy-fail-console-script): New variables. ("python-build-system: dummy-ok") ("python-build-system: dummy-dummy-nosetuptools") ("python-build-system: dummy-fail-requirements") ("python-build-system: dummy-fail-import") ("python-build-system: dummy-fail-console-script"): Add tests. --- Makefile.am | 1 + gnu/packages/aux-files/python/sanity-check.py | 91 +++++++++++++++++++++++++++ guix/build-system/python.scm | 8 +++ guix/build/python-build-system.scm | 26 +++++--- tests/builders.scm | 86 ++++++++++++++++++++++++- 5 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 gnu/packages/aux-files/python/sanity-check.py (limited to 'gnu/packages/aux-files/python') diff --git a/Makefile.am b/Makefile.am index 7ef8d4d61eb..e60c5f45002 100644 --- a/Makefile.am +++ b/Makefile.am @@ -380,6 +380,7 @@ AUX_FILES = \ gnu/packages/aux-files/linux-libre/4.4-i686.conf \ gnu/packages/aux-files/linux-libre/4.4-x86_64.conf \ gnu/packages/aux-files/pack-audit.c \ + gnu/packages/aux-files/python/sanity-check.py \ gnu/packages/aux-files/python/sitecustomize.py \ gnu/packages/aux-files/run-in-namespace.c diff --git a/gnu/packages/aux-files/python/sanity-check.py b/gnu/packages/aux-files/python/sanity-check.py new file mode 100644 index 00000000000..355e55b2402 --- /dev/null +++ b/gnu/packages/aux-files/python/sanity-check.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +# GNU Guix --- Functional package management for GNU +# Copyright © 2021 Lars-Dominik Braun +# +# This file is part of GNU Guix. +# +# GNU Guix is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or (at +# your option) any later version. +# +# GNU Guix is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GNU Guix. If not, see . + +from __future__ import print_function # Python 2 support. +import importlib +import pkg_resources +import sys +import traceback + +try: + from importlib.machinery import PathFinder +except ImportError: + PathFinder = None + +ret = 0 + +# Only check site-packages installed by this package, but not dependencies +# (which pkg_resources.working_set would include). Path supplied via argv. +ws = pkg_resources.find_distributions(sys.argv[1]) + +for dist in ws: + print('validating', repr(dist.project_name), dist.location) + try: + print('...checking requirements: ', end='') + req = str(dist.as_requirement()) + # dist.activate() is not enough to actually check requirements, we + # have to .require() it. + pkg_resources.require(req) + print('OK') + except Exception as e: + print('ERROR:', req, e) + ret = 1 + continue + + # Try to load top level modules. This should not have any side-effects. + try: + metalines = dist.get_metadata_lines('top_level.txt') + except KeyError: + # distutils (i.e. #:use-setuptools? #f) will not install any metadata. + print('WARNING: cannot determine top-level modules') + continue + for name in metalines: + # Only available on Python 3. + if PathFinder and PathFinder.find_spec(name) is None: + # Ignore unavailable modules, often C modules, which were not + # installed at the top-level. Cannot use ModuleNotFoundError, + # because it is raised by failed imports too. + continue + try: + print('...trying to load module', name, end=': ') + importlib.import_module(name) + print('OK') + except Exception: + print('ERROR:') + traceback.print_exc(file=sys.stdout) + ret = 1 + continue + + # Try to load entry points of console scripts too, making sure they + # work. They should be removed if they don't. Other groups may not be + # safe, as they can depend on optional packages. + for group, v in dist.get_entry_map().items(): + if group not in {'console_scripts', 'gui_scripts'}: + continue + for name, ep in v.items(): + try: + print('...trying to load endpoint', group, name, end=': ') + ep.load() + print('OK') + except Exception: + print('ERROR:') + traceback.print_exc(file=sys.stdout) + ret = 1 + +sys.exit(ret) diff --git a/guix/build-system/python.scm b/guix/build-system/python.scm index e39c06528ef..2bb6fa87ca0 100644 --- a/guix/build-system/python.scm +++ b/guix/build-system/python.scm @@ -2,6 +2,7 @@ ;;; Copyright © 2013, 2014, 2015, 2016, 2017 Ludovic Courtès ;;; Copyright © 2013 Andreas Enge ;;; Copyright © 2013 Nikita Karetnikov +;;; Copyright © 2021 Lars-Dominik Braun ;;; ;;; This file is part of GNU Guix. ;;; @@ -19,6 +20,8 @@ ;;; along with GNU Guix. If not, see . (define-module (guix build-system python) + #:use-module ((gnu packages) #:select (search-auxiliary-file)) + #:use-module (guix gexp) #:use-module (guix store) #:use-module (guix utils) #:use-module (guix memoization) @@ -70,6 +73,10 @@ extension, such as '.tar.gz'." (let ((python (resolve-interface '(gnu packages python)))) (module-ref python 'python-2))) +(define sanity-check.py + ;; The script used to validate the installation of a Python package. + (search-auxiliary-file "python/sanity-check.py")) + (define* (package-with-explicit-python python old-prefix new-prefix #:key variant-property) "Return a procedure of one argument, P. The procedure creates a package with @@ -156,6 +163,7 @@ pre-defined variants." ;; Keep the standard inputs of 'gnu-build-system'. ,@(standard-packages))) (build-inputs `(("python" ,python) + ("sanity-check.py" ,(local-file sanity-check.py)) ,@native-inputs)) (outputs outputs) (build python-build) diff --git a/guix/build/python-build-system.scm b/guix/build/python-build-system.scm index 0f2402d3a3b..1fc97d13986 100644 --- a/guix/build/python-build-system.scm +++ b/guix/build/python-build-system.scm @@ -9,6 +9,7 @@ ;;; Copyright © 2019, 2020, 2021 Maxim Cournoyer ;;; Copyright © 2020 Jakub Kądziołka ;;; Copyright © 2020 Efraim Flashner +;;; Copyright © 2021 Lars-Dominik Braun ;;; ;;; This file is part of GNU Guix. ;;; @@ -132,6 +133,15 @@ (apply invoke "python" "./setup.py" command params))) (error "no setup.py found"))) +(define* (sanity-check #:key tests? inputs outputs #:allow-other-keys) + "Ensure packages depending on this package via setuptools work properly, +their advertised endpoints work and their top level modules are importable +without errors." + (let ((sanity-check.py (assoc-ref inputs "sanity-check.py"))) + ;; Make sure the working directory is empty (i.e. no Python modules in it) + (with-directory-excursion "/tmp" + (invoke "python" sanity-check.py (site-packages inputs outputs))))) + (define* (build #:key use-setuptools? #:allow-other-keys) "Build a given Python package." (call-setuppy "build" '() use-setuptools?) @@ -209,8 +219,7 @@ running checks after installing the package." ;; '--invalidation-mode' option, do not generate any. (unless <3.7? (invoke "python" "-m" "compileall" "--invalidation-mode=unchecked-hash" - out)) - #t)) + out)))) (define* (wrap #:key inputs outputs #:allow-other-keys) (define (list-of-files dir) @@ -244,8 +253,7 @@ installed with setuptools." (easy-install-pth (string-append site-packages "/easy-install.pth")) (new-pth (string-append site-packages "/" name ".pth"))) (when (file-exists? easy-install-pth) - (rename-file easy-install-pth new-pth)) - #t)) + (rename-file easy-install-pth new-pth)))) (define* (ensure-no-mtimes-pre-1980 #:rest _) "Ensure that there are no mtimes before 1980-01-02 in the source tree." @@ -257,8 +265,7 @@ installed with setuptools." (ftw "." (lambda (file stat flag) (unless (<= early-1980 (stat:mtime stat)) (utime file early-1980 early-1980)) - #t)) - #t)) + #t)))) (define* (enable-bytecode-determinism #:rest _) "Improve determinism of pyc files." @@ -266,8 +273,7 @@ installed with setuptools." (setenv "PYTHONHASHSEED" "0") ;; Prevent Python from creating .pyc files when loading modules (such as ;; when running a test suite). - (setenv "PYTHONDONTWRITEBYTECODE" "1") - #t) + (setenv "PYTHONDONTWRITEBYTECODE" "1")) (define* (ensure-no-cythonized-files #:rest _) "Check the source code for @code{.c} files which may have been pre-generated @@ -278,8 +284,7 @@ by Cython." (string-append (string-drop-right file 3) "c"))) (when (file-exists? generated-file) (format #t "Possible Cythonized file found: ~a~%" generated-file)))) - (find-files "." "\\.pyx$")) - #t) + (find-files "." "\\.pyx$"))) (define %standard-phases ;; The build phase only builds C extensions and copies the Python sources, @@ -301,6 +306,7 @@ by Cython." (add-after 'install 'wrap wrap) (add-before 'check 'add-install-to-pythonpath add-install-to-pythonpath) (add-before 'check 'add-install-to-path add-install-to-path) + (add-after 'check 'sanity-check sanity-check) (add-before 'strip 'rename-pth-file rename-pth-file))) (define* (python-build #:key inputs (phases %standard-phases) diff --git a/tests/builders.scm b/tests/builders.scm index 624547500a4..2143c0738b8 100644 --- a/tests/builders.scm +++ b/tests/builders.scm @@ -1,5 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2012, 2013, 2014, 2015, 2019 Ludovic Courtès +;;; Copyright © 2021 Lars-Dominik Braun ;;; ;;; This file is part of GNU Guix. ;;; @@ -23,15 +24,15 @@ #:use-module (guix build-system gnu) #:use-module (guix build gnu-build-system) #:use-module (guix build utils) + #:use-module (guix build-system python) #:use-module (guix store) + #:use-module (guix monads) #:use-module (guix utils) #:use-module (guix base32) #:use-module (guix derivations) #:use-module (gcrypt hash) #:use-module (guix tests) - #:use-module ((guix packages) - #:select (package? - package-derivation package-native-search-paths)) + #:use-module (guix packages) #:use-module (gnu packages bootstrap) #:use-module (ice-9 match) #:use-module (ice-9 textual-ports) @@ -111,4 +112,83 @@ (call-with-input-file name get-string-all)))))))) compressors) + +;;; +;;; Test the sanity-check phase of the Python build system. +;;; + +(define* (make-python-dummy name #:key (setup-py-extra "") + (init-py "") (use-setuptools? #t)) + (dummy-package (string-append "python-dummy-" name) + (version "0.1") + (build-system python-build-system) + (arguments + `(#:tests? #f + #:use-setuptools? ,use-setuptools? + #:phases + (modify-phases %standard-phases + (replace 'unpack + (lambda _ + (mkdir-p "dummy") + (with-output-to-file "dummy/__init__.py" + (lambda _ + (display ,init-py))) + (with-output-to-file "setup.py" + (lambda _ + (format #t "\ +~a +setup( + name='dummy-~a', + version='0.1', + packages=['dummy'], + ~a + )" + (if ,use-setuptools? + "from setuptools import setup" + "from distutils.core import setup") + ,name ,setup-py-extra)))))))))) + +(define python-dummy-ok + (make-python-dummy "ok")) + +;; distutil won't install any metadata, so make sure our script does not fail +;; on a otherwise fine package. +(define python-dummy-no-setuptools + (make-python-dummy + "no-setuptools" #:use-setuptools? #f)) + +(define python-dummy-fail-requirements + (make-python-dummy "fail-requirements" + #:setup-py-extra "install_requires=['nonexistent'],")) + +(define python-dummy-fail-import + (make-python-dummy "fail-import" #:init-py "import nonexistent")) + +(define python-dummy-fail-console-script + (make-python-dummy "fail-console-script" + #:setup-py-extra (string-append "entry_points={'console_scripts': " + "['broken = dummy:nonexistent']},"))) + +(define (check-build-success store p) + (unless store (test-skip 1)) + (test-assert (string-append "python-build-system: " (package-name p)) + (let* ((drv (package-derivation store p))) + (build-derivations store (list drv))))) + +(define (check-build-failure store p) + (unless store (test-skip 1)) + (test-assert (string-append "python-build-system: " (package-name p)) + (not (false-if-exception (package-derivation store python-dummy-fail-requirements))))) + +(with-external-store store + (for-each (lambda (p) (check-build-success store p)) + (list + python-dummy-ok + python-dummy-no-setuptools)) + (for-each (lambda (p) (check-build-failure store p)) + (list + python-dummy-fail-requirements + python-dummy-fail-import + python-dummy-fail-console-script))) + (test-end "builders") -- cgit v1.3 From bb557c9ac19ab01cdf621c3fac887750981feb77 Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Wed, 3 Feb 2021 21:56:53 -0500 Subject: build/python: Handle FileNotFoundError exception in sanity-check.py. This exception was thrown while running the sanity-check on the python-isort package, which doesn't make use of a setuptools-based build system but rather of a PEP 517 compliant one. * gnu/packages/aux-files/python/sanity-check.py: Handle the FileNotFoundError whe attempting to read the 'top_level.txt' metadata file of the package. Remove extraneous 'continue' statement. --- gnu/packages/aux-files/python/sanity-check.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'gnu/packages/aux-files/python') diff --git a/gnu/packages/aux-files/python/sanity-check.py b/gnu/packages/aux-files/python/sanity-check.py index 355e55b2402..83b6d583ca7 100644 --- a/gnu/packages/aux-files/python/sanity-check.py +++ b/gnu/packages/aux-files/python/sanity-check.py @@ -51,8 +51,10 @@ for dist in ws: # Try to load top level modules. This should not have any side-effects. try: metalines = dist.get_metadata_lines('top_level.txt') - except KeyError: + except (KeyError, FileNotFoundError): # distutils (i.e. #:use-setuptools? #f) will not install any metadata. + # This file is also missing for packages built using a PEP 517 builder + # such as poetry. print('WARNING: cannot determine top-level modules') continue for name in metalines: @@ -70,7 +72,6 @@ for dist in ws: print('ERROR:') traceback.print_exc(file=sys.stdout) ret = 1 - continue # Try to load entry points of console scripts too, making sure they # work. They should be removed if they don't. Other groups may not be -- cgit v1.3 From f8458a228224942298e0967a099f872c13713ede Mon Sep 17 00:00:00 2001 From: Marius Bakke Date: Fri, 30 Jul 2021 15:54:31 +0200 Subject: build-system/python: Handle missing metadata on Python 2. * gnu/packages/aux-files/python/sanity-check.py: Catch the less specific EnvironmentError rather than FileNotFoundError as the latter is Python 3 only. --- gnu/packages/aux-files/python/sanity-check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gnu/packages/aux-files/python') diff --git a/gnu/packages/aux-files/python/sanity-check.py b/gnu/packages/aux-files/python/sanity-check.py index 83b6d583ca7..a84f8f03c09 100644 --- a/gnu/packages/aux-files/python/sanity-check.py +++ b/gnu/packages/aux-files/python/sanity-check.py @@ -51,7 +51,7 @@ for dist in ws: # Try to load top level modules. This should not have any side-effects. try: metalines = dist.get_metadata_lines('top_level.txt') - except (KeyError, FileNotFoundError): + except (KeyError, EnvironmentError): # distutils (i.e. #:use-setuptools? #f) will not install any metadata. # This file is also missing for packages built using a PEP 517 builder # such as poetry. -- cgit v1.3 From 874188c0ab2955412519dc917402fefc6dd723fc Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer Date: Tue, 3 Aug 2021 20:41:20 -0400 Subject: aux-files: sitecustomize: Cleanup and add explanatory comments. Fixes . * gnu/packages/aux-files/python/sitecustomize.py: Add a comment explaining the general idea, and use sys.prefix instead of sys.executable. (major_minor): Use the unpacking operator (*) to provide the arguments. (site_packages_prefix): Use os.path.join to form the path. (python_site): Likewise. Use sys.prefix instead of sys.executable. (all_sites_raw): Split on os.path.pathsep. (sys.path): Directly splice the result in the list. Suggested-by: Hartmut Goebel Reported-by: Mathieu Othacehe Signed-off-by: Maxim Cournoyer --- gnu/packages/aux-files/python/sitecustomize.py | 28 +++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'gnu/packages/aux-files/python') diff --git a/gnu/packages/aux-files/python/sitecustomize.py b/gnu/packages/aux-files/python/sitecustomize.py index 65d3c7d5546..71e328b9ac4 100644 --- a/gnu/packages/aux-files/python/sitecustomize.py +++ b/gnu/packages/aux-files/python/sitecustomize.py @@ -20,13 +20,26 @@ import os import sys -python_root = os.path.realpath(sys.executable).split('/bin/')[0] -major_minor = '{}.{}'.format(sys.version_info[0], sys.version_info[1]) -site_packages_prefix = 'lib/python' + major_minor + '/site-packages' -python_site = python_root + '/' + site_packages_prefix +# Commentary: +# +# Site-specific customization for Guix. +# +# The program below honors the GUIX_PYTHONPATH environment variable to +# discover Python packages. File names appearing in this variable that match +# a predefined versioned installation prefix are added to the sys.path. To be +# considered, a Python package must be installed under the +# 'lib/pythonX.Y/site-packages' directory, where X and Y are the major and +# minor version numbers of the Python interpreter. +# +# Code: + +major_minor = '{}.{}'.format(*sys.version_info) +site_packages_prefix = os.path.join( + 'lib', 'python' + major_minor, 'site-packages') +python_site = os.path.join(sys.prefix, site_packages_prefix) try: - all_sites_raw = os.environ['GUIX_PYTHONPATH'].split(':') + all_sites_raw = os.environ['GUIX_PYTHONPATH'].split(os.path.pathsep) except KeyError: all_sites_raw = [] # Normalize paths, otherwise a trailing slash would cause it to not match. @@ -35,7 +48,8 @@ matching_sites = [p for p in all_sites_norm if p.endswith(site_packages_prefix)] # Insert sites matching the current version into sys.path, right before -# Python's own site. +# Python's own site. This way, the user can override the libraries provided +# by Python itself. sys_path_absolute = [os.path.realpath(p) for p in sys.path] index = sys_path_absolute.index(python_site) -sys.path = sys.path[:index] + matching_sites + sys.path[index:] +sys.path[index:index] = matching_sites -- cgit v1.3