import: pypi: Move tests to tests/import/pypi.scm

* tests/pypi.scm: Move to tests/import/pypi.scm
* CODEOWNERS: Refresh it.
* Makefile.am: Refresh it.
* etc/teams.scm: Refresh it.

Signed-off-by: Ludovic Courtès <ludo@gnu.org>

1 files changed, 571 insertions, 0 deletions

diff --git a/tests/import/pypi.scm b/tests/import/pypi.scm
new file mode 100644
index 00000000000..2f6ec650a2d
--- /dev/null
+++ b/tests/import/pypi.scm
@@ -0,0 +1,571 @@
+;;; GNU Guix --- Functional package management for GNU
+;;; Copyright © 2014 David Thompson <davet@gnu.org>
+;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net>
+;;; Copyright © 2019 Maxim Cournoyer <maxim@guixotic.coop>
+;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz>
+;;; Copyright © 2022 Vivien Kraus <vivien@planete-kraus.eu>
+;;;
+;;; This file is part of GNU Guix.
+;;;
+;;; GNU Guix is free software; you can redistribute it and/or modify it
+;;; under the terms of the GNU General Public License as published by
+;;; the Free Software Foundation; either version 3 of the License, or (at
+;;; your option) any later version.
+;;;
+;;; GNU Guix is distributed in the hope that it will be useful, but
+;;; WITHOUT ANY WARRANTY; without even the implied warranty of
+;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;; GNU General Public License for more details.
+;;;
+;;; You should have received a copy of the GNU General Public License
+;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
+
+(define-module (test-pypi)
+  #:use-module (guix import pypi)
+  #:use-module (guix base32)
+  #:use-module (guix memoization)
+  #:use-module (guix utils)
+  #:use-module ((guix base16) #:select (base16-string->bytevector))
+  #:use-module (guix upstream)
+  #:use-module (gcrypt hash)
+  #:use-module (guix tests)
+  #:use-module (guix tests http)
+  #:use-module ((guix download) #:select (url-fetch))
+  #:use-module (guix build-system python)
+  #:use-module ((guix build utils)
+                #:select (delete-file-recursively
+                          which mkdir-p dump-port
+                          with-directory-excursion))
+  #:use-module ((guix diagnostics) #:select (guix-warning-port))
+  #:use-module ((guix build syscalls) #:select (mkdtemp!))
+  #:use-module (json)
+  #:use-module (srfi srfi-26)
+  #:use-module (srfi srfi-34)
+  #:use-module (srfi srfi-35)
+  #:use-module (srfi srfi-64)
+  #:use-module (ice-9 match)
+  #:use-module (ice-9 optargs))
+
+(define default-sha256
+  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+(define default-sha256/base32
+  (bytevector->nix-base32-string
+   (base16-string->bytevector default-sha256)))
+
+(define* (foo-json #:key (name "foo") (name-in-url #f))
+  "Create a JSON description of an example pypi package, named @var{name},
+optionally using a different @var{name in its URL}."
+  (scm->json-string
+   `((info
+      . ((version . "1.0.0")
+         (name . ,name)
+         (license . "GNU LGPL")
+         (summary . "summary")
+         (home_page . "http://example.com")
+         (classifiers . #())
+         (download_url . "")))
+     (urls . #())
+     (releases
+      . ((1.0.0
+          . #(((url . ,(format #f "~a/~a-1.0.0.egg"
+                               (%local-url #:path "")
+                               (or name-in-url name)))
+               (packagetype . "bdist_egg"))
+              ((url . ,(format #f "~a/~a-1.0.0.tar.gz"
+                               (%local-url #:path "")
+                               (or name-in-url name)))
+               (packagetype . "sdist")
+               (digests . (("sha256" . ,default-sha256))))
+              ((url . ,(format #f "~a/~a-1.0.0-py2.py3-none-any.whl"
+                               (%local-url #:path "")
+                               (or name-in-url name)))
+               (packagetype . "bdist_wheel")))))))))
+
+(define test-specifications
+  '("Fizzy [foo, bar]"
+    "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1"
+    "SomethingWithMarker[foo]>1.0;python_version<\"2.7\""
+    "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\""
+    "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\
+sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"))
+
+(define test-requires.txt "\
+# A comment
+ # A comment after a space
+foo ~= 3
+bar != 2
+
+[test]
+pytest (>=2.5.0)
+")
+
+;; Beaker contains only optional dependencies.
+(define test-requires.txt-beaker "\
+[crypto]
+pycryptopp>=0.5.12
+
+[cryptography]
+cryptography
+
+[testsuite]
+Mock
+coverage
+")
+
+(define test-pyproject.toml "\
+[build-system]
+requires = [\"dummy-build-dep-a\", \"dummy-build-dep-b\"]
+
+[project]
+dependencies = [
+  \"dummy-dep-a\",
+  \"dummy-dep-b\",
+]
+
+[project.optional-dependencies]
+test = [\"dummy-test-dep-a\", \"dummy-test-dep-b\"]
+")
+
+(define test-metadata "\
+Classifier: Programming Language :: Python :: 3.7
+Requires-Dist: baz ~= 3
+Requires-Dist: bar != 2
+Provides-Extra: test
+Requires-Dist: pytest (>=2.5.0) ; extra == 'test'
+")
+
+(define test-metadata-with-extras "
+Classifier: Programming Language :: Python :: 3.7
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Requires-Dist: wrapt (<2,>=1)
+Requires-Dist: bar
+
+Provides-Extra: dev
+Requires-Dist: tox ; extra == 'dev'
+Requires-Dist: bumpversion (<1) ; extra == 'dev'
+")
+
+;;; Provides-Extra can appear before Requires-Dist.
+(define test-metadata-with-extras-jedi "\
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Provides-Extra: testing
+Requires-Dist: parso (>=0.3.0)
+Provides-Extra: testing
+Requires-Dist: pytest (>=3.1.0); extra == 'testing'
+")
+
+(define sample-directory
+  ;; Directory containing tarballs and .whl files for this test.
+  (let ((template (string-append (or (getenv "TMPDIR") "/tmp")
+                                 "/guix-pypi-test-XXXXXX")))
+    (mkdtemp! template)))
+
+(define (pypi-tarball name specs)
+  "Return a PyPI tarball called NAME suffixed with '.tar.gz' and containing
+the files specified in SPECS.  Return its file name."
+  (let ((directory (in-vicinity sample-directory name))
+        (tarball (in-vicinity sample-directory (string-append name ".tar.gz"))))
+    (false-if-exception (delete-file tarball))
+    (mkdir-p directory)
+    (for-each (match-lambda
+                ((file content)
+                 (mkdir-p (in-vicinity directory (dirname file)))
+                 (call-with-output-file (in-vicinity directory file)
+                   (lambda (port)
+                     (display content port)))))
+              specs)
+    (parameterize ((current-output-port (%make-void-port "w0")))
+      (system* "tar" "-C" sample-directory "-czvf" tarball
+               (basename directory)))
+    (delete-file-recursively directory)
+    tarball))
+
+(define (wheel-file name specs)
+  "Return a Wheel file called NAME suffixed with '.whl' and containing the
+files specified by SPECS.  Return its file name."
+  (let* ((directory (in-vicinity sample-directory
+                                 (string-append name ".dist-info")))
+         (zip-file (in-vicinity sample-directory
+                                (string-append name ".zip")))
+         (whl-file (in-vicinity sample-directory
+                                (string-append name ".whl"))))
+    (false-if-exception (delete-file whl-file))
+    (mkdir-p directory)
+    (for-each (match-lambda
+                ((file content)
+                 (mkdir-p (in-vicinity directory (dirname file)))
+                 (call-with-output-file (in-vicinity directory file)
+                   (lambda (port)
+                     (display content port)))))
+              specs)
+    ;; zip always adds a "zip" extension to the file it creates,
+    ;; so we need to rename it.
+    (with-directory-excursion (dirname directory)
+      (system* "zip" "-qr" zip-file (basename directory)))
+    (rename-file zip-file whl-file)
+    (delete-file-recursively directory)
+    whl-file))
+
+(define (file-dump file)
+  "Return a procedure that dumps FILE to the given port."
+  (lambda (output)
+    (call-with-input-file file
+      (lambda (input)
+        (dump-port input output)))))
+
+(define-syntax-rule (with-pypi responses body ...)
+  (with-http-server responses
+    (parameterize ((%pypi-base-url (%local-url #:path "/")))
+      body ...)))
+
+
+(test-begin "pypi")
+
+(test-equal "guix-package->pypi-name, old URL style"
+  "psutil"
+  (guix-package->pypi-name
+   (dummy-package "foo"
+                  (source (dummy-origin
+                           (uri
+                            "https://pypi.org/packages/source/p/psutil/psutil-4.3.0.tar.gz"))))))
+
+(test-equal "guix-package->pypi-name, new URL style"
+  "certbot"
+  (guix-package->pypi-name
+   (dummy-package "foo"
+                  (source (dummy-origin
+                           (uri
+                            "https://pypi.org/packages/a2/3b/4756e6a0ceb14e084042a2a65c615d68d25621c6fd446d0fc10d14c4ce7d/certbot-0.8.1.tar.gz"))))))
+
+(test-equal "guix-package->pypi-name, several URLs"
+  "cram"
+  (guix-package->pypi-name
+   (dummy-package "foo"
+                  (source
+                   (dummy-origin
+                    (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz"
+                               (pypi-uri "cram" "0.7"))))))))
+
+(test-equal "guix-package->pypi-name, honor 'upstream-name'"
+  "bar-3"
+  (guix-package->pypi-name
+   (dummy-package "foo"
+                  (properties
+                   '((upstream-name . "bar-3"))))))
+
+(test-equal "specification->requirement-name"
+  '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip")
+  (map specification->requirement-name test-specifications))
+
+(test-equal "parse-requires.txt"
+  (list '("foo" "bar") '("pytest"))
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-requires.txt test-requires.txt)))
+
+(test-equal "parse-requires.txt - Beaker"
+  (list '() '("Mock" "coverage"))
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-requires.txt test-requires.txt-beaker)))
+
+(test-equal "parse-wheel-metadata, with extras"
+  (list '("wrapt" "bar") '("tox" "bumpversion"))
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-wheel-metadata test-metadata-with-extras)))
+
+(test-equal "parse-wheel-metadata, with extras - Jedi"
+  (list '("parso") '("pytest"))
+  (mock ((ice-9 ports) call-with-input-file
+         call-with-input-string)
+        (parse-wheel-metadata test-metadata-with-extras-jedi)))
+
+(test-equal "find-project-url, with numpy"
+  "numpy"
+  (find-project-url
+   "numpy"
+   "https://files.pythonhosted.org/packages/0a/c8/a62767a6b374a0dfb02d2a0456e5f56a372cdd1689dbc6ffb6bf1ddedbc0/numpy-1.22.1.zip"))
+
+(test-equal "find-project-url, uWSGI"
+  "uwsgi"
+  (find-project-url
+   "uWSGI"
+   "https://files.pythonhosted.org/packages/24/fd/93851e4a076719199868d4c918cc93a52742e68370188c1c570a6e42a54f/uwsgi-2.0.20.tar.gz"))
+
+(test-equal "find-project-url, flake8-array-spacing"
+  "flake8_array_spacing"
+  (find-project-url
+   "flake8-array-spacing"
+   "https://files.pythonhosted.org/packages/a4/21/ff29b901128b681b7de7a2787b3aeb3e1f3cba4a8c0cffa9712cbff016bc/flake8_array_spacing-0.2.0.tar.gz"))
+
+(test-equal "find-project-url, foo/goo"
+  "foo"
+  (find-project-url
+   "foo"
+   "https://files.pythonhosted.org/packages/f0/f00/goo-0.0.0.tar.gz"))
+
+(test-assert "pypi->guix-package, no wheel"
+  (let ((tarball (pypi-tarball
+                  "foo-1.0.0"
+                  `(("src/bizarre.egg-info/requires.txt"
+                     ,test-requires.txt))))
+        (twice (lambda (lst) (append lst lst))))
+    (with-pypi (twice `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                        ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
+                        ("/foo/json" 200 ,(lambda (port)
+                                            (display (foo-json) port)))))
+      (match (pypi->guix-package "foo")
+        (`(package
+            (name "python-foo")
+            (version "1.0.0")
+            (source (origin
+                      (method url-fetch)
+                      (uri (pypi-uri "foo" version))
+                      (sha256
+                       (base32 ,(? string? hash)))))
+            (build-system pyproject-build-system)
+            (propagated-inputs (list python-bar python-foo))
+            (native-inputs (list python-pytest
+                                 python-setuptools
+                                 python-wheel))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary.")
+            (license license:lgpl2.0))
+         (and (string=? default-sha256/base32 hash)
+              (equal? (pypi->guix-package "foo" #:version "1.0.0")
+                      (pypi->guix-package "foo"))
+              (guard (c ((error? c) #t))
+                (pypi->guix-package "foo" #:version "42"))))
+        (x
+         (pk 'fail x #f))))))
+
+(test-assert "pypi->guix-package, no wheel, no requires.txt, but pyproject.toml"
+  (let ((tarball (pypi-tarball
+                  "foo-1.0.0"
+                  `(("pyproject.toml" ,test-pyproject.toml))))
+        (twice (lambda (lst) (append lst lst))))
+    (with-pypi (twice `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                        ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
+                        ("/foo/json" 200 ,(lambda (port)
+                                            (display (foo-json) port)))))
+      ;; Not clearing the memoization cache here would mean returning the value
+      ;; computed in the previous test.
+      (invalidate-memoization! pypi->guix-package)
+      (match (pypi->guix-package "foo")
+        (`(package
+            (name "python-foo")
+            (version "1.0.0")
+            (source (origin
+                      (method url-fetch)
+                      (uri (pypi-uri "foo" version))
+                      (sha256
+                       (base32 ,(? string? hash)))))
+            (build-system pyproject-build-system)
+            (propagated-inputs (list python-dummy-dep-a python-dummy-dep-b))
+            (native-inputs (list python-dummy-build-dep-a python-dummy-build-dep-b
+                                 python-dummy-test-dep-a python-dummy-test-dep-b))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary.")
+            (license license:lgpl2.0))
+         (and (string=? default-sha256/base32 hash)
+              (equal? (pypi->guix-package "foo" #:version "1.0.0")
+                      (pypi->guix-package "foo"))
+              (guard (c ((error? c) #t))
+                (pypi->guix-package "foo" #:version "42"))))
+        (x
+         (pk 'fail x #f))))))
+
+(test-assert "pypi->guix-package, no wheel, but requires.txt and pyproject.toml"
+  (let ((tarball (pypi-tarball
+                  "foo-1.0.0"
+                  `(("foo-1.0.0/pyproject.toml" ,test-pyproject.toml)
+                    ("foo-1.0.0/bizarre.egg-info/requires.txt"
+                     ,test-requires.txt))))
+        (twice (lambda (lst) (append lst lst))))
+    (with-pypi (twice `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                        ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
+                        ("/foo/json" 200 ,(lambda (port)
+                                            (display (foo-json) port)))))
+      ;; Not clearing the memoization cache here would mean returning the value
+      ;; computed in the previous test.
+      (invalidate-memoization! pypi->guix-package)
+      (match (pypi->guix-package "foo")
+        (`(package
+            (name "python-foo")
+            (version "1.0.0")
+            (source (origin
+                      (method url-fetch)
+                      (uri (pypi-uri "foo" version))
+                      (sha256
+                       (base32 ,(? string? hash)))))
+            (build-system pyproject-build-system)
+            ;; Information from requires.txt and pyproject.toml is combined.
+            (propagated-inputs (list python-bar python-dummy-dep-a python-dummy-dep-b
+                                     python-foo))
+            (native-inputs (list python-dummy-build-dep-a python-dummy-build-dep-b
+                                 python-dummy-test-dep-a python-dummy-test-dep-b
+                                 python-pytest))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary.")
+            (license license:lgpl2.0))
+         (and (string=? default-sha256/base32 hash)
+              (equal? (pypi->guix-package "foo" #:version "1.0.0")
+                      (pypi->guix-package "foo"))
+              (guard (c ((error? c) #t))
+                (pypi->guix-package "foo" #:version "42"))))
+        (x
+         (pk 'fail x #f))))))
+
+(test-skip (if (which "zip") 0 1))
+(test-assert "pypi->guix-package, no requires.txt, but wheel."
+  (let ((tarball (pypi-tarball
+                  "foo-1.0.0"
+                  '(("foo-1.0.0/foo.egg-info/.empty" ""))))
+        (wheel (wheel-file "foo-1.0.0"
+                           `(("METADATA" ,test-metadata)))))
+    (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                 ("/foo-1.0.0-py2.py3-none-any.whl"
+                  200 ,(file-dump wheel))
+                 ("/foo/json" 200 ,(lambda (port)
+                                     (display (foo-json) port))))
+      ;; Not clearing the memoization cache here would mean returning the value
+      ;; computed in the previous test.
+      (invalidate-memoization! pypi->guix-package)
+      (match (pypi->guix-package "foo")
+        (`(package
+            (name "python-foo")
+            (version "1.0.0")
+            (source (origin
+                      (method url-fetch)
+                      (uri (pypi-uri "foo" version))
+                      (sha256
+                       (base32 ,(? string? hash)))))
+            (build-system pyproject-build-system)
+            (propagated-inputs (list python-bar python-baz))
+            (native-inputs (list python-pytest
+                                 python-setuptools
+                                 python-wheel))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary.")
+            (license license:lgpl2.0))
+         (string=? default-sha256/base32 hash))
+        (x
+         (pk 'fail x #f))))))
+
+(test-assert "pypi->guix-package, no usable requirement file, no wheel."
+  (let ((tarball (pypi-tarball "foo-1.0.0"
+                               '(("foo.egg-info/.empty" "")))))
+    (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
+                 ("/foo/json" 200 ,(lambda (port)
+                                     (display (foo-json) port))))
+      ;; Not clearing the memoization cache here would mean returning the
+      ;; value computed in the previous test.
+      (invalidate-memoization! pypi->guix-package)
+      (match (pypi->guix-package "foo")
+        (`(package
+            (name "python-foo")
+            (version "1.0.0")
+            (source (origin
+                       (method url-fetch)
+                       (uri (pypi-uri "foo" version))
+                       (sha256
+                        (base32 ,(? string? hash)))))
+            (build-system pyproject-build-system)
+            (native-inputs (list python-setuptools python-wheel))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary.")
+            (license license:lgpl2.0))
+         (string=? default-sha256/base32 hash))
+        (x
+         (pk 'fail x #f))))))
+
+(test-assert "pypi->guix-package, package name contains \"-\" followed by digits"
+  (let ((tarball (pypi-tarball "foo-99-1.0.0"
+                               `(("src/bizarre.egg-info/requires.txt"
+                                  ,test-requires.txt)))))
+    (with-pypi `(("/foo-99-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                 ("/foo-99-1.0.0-py2.py3-none-any.whl" 404 "")
+                 ("/foo-99/json" 200 ,(lambda (port)
+                                        (display (foo-json #:name "foo-99")
+                                                 port))))
+      (match (pypi->guix-package "foo-99")
+        (`(package
+            (name "python-foo-99")
+            (version "1.0.0")
+            (source (origin
+                      (method url-fetch)
+                      (uri (pypi-uri "foo-99" version))
+                      (sha256
+                       (base32 ,(? string? hash)))))
+            (properties (quote (("upstream-name" . "foo-99"))))
+            (build-system pyproject-build-system)
+            (propagated-inputs (list python-bar python-foo))
+            (native-inputs (list python-pytest
+                                 python-setuptools
+                                 python-wheel))
+            (home-page "http://example.com")
+            (synopsis "summary")
+            (description "summary.")
+            (license license:lgpl2.0))
+         (string=? default-sha256/base32 hash))
+        (x
+         (pk 'fail x #f))))))
+
+(test-equal "package-latest-release"
+  (list '("foo-1.0.0.tar.gz")
+        '("foo-1.0.0.tar.gz.asc")
+        (list (upstream-input
+               (name "bar")
+               (downstream-name "python-bar")
+               (type 'propagated))
+              (upstream-input
+               (name "foo")
+               (downstream-name "python-foo")
+               (type 'propagated))
+              (upstream-input
+               (name "pytest")
+               (downstream-name "python-pytest")
+               (type 'native))
+              (upstream-input
+               (name "setuptools")
+               (downstream-name "python-setuptools")
+               (type 'native))
+              (upstream-input
+               (name "wheel")
+               (downstream-name "python-wheel")
+               (type 'native))))
+  (let ((tarball (pypi-tarball
+                  "foo-1.0.0"
+                  `(("src/bizarre.egg-info/requires.txt"
+                     ,test-requires.txt)))))
+    (with-pypi `(("/foo-1.0.0.tar.gz" 200 ,(file-dump tarball))
+                 ("/foo-1.0.0-py2.py3-none-any.whl" 404 "")
+                 ("/foo/json" 200 ,(lambda (port)
+                                     (display (foo-json) port))))
+      (define source
+        (package-latest-release
+         (dummy-package "python-foo"
+                        (version "0.1.2")
+                        (source (dummy-origin
+                                 (method url-fetch)
+                                 (uri (pypi-uri "foo" version))))
+                        (build-system python-build-system))
+         (list %pypi-updater)))
+
+      (list (map basename (upstream-source-urls source))
+            (map basename (upstream-source-signature-urls source))
+            (upstream-source-inputs source)))))
+
+(test-end "pypi")
+(delete-file-recursively sample-directory)
+
+;; Local Variables:
+;; eval: (put 'with-pypi 'scheme-indent-function 1)
+;; End: