From f6b0e1f8ff6a6459d7d39238ced165f4caa988fe Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Thu, 4 Apr 2019 17:36:49 +0100 Subject: services: Add getmail. Getmail is a mail retriever written in Python, this commit adds a service-type to run getmail. I'm looking at this, as it's a convinient way of getting mailing list messages in to Patchwork. I initially tried putting this in the (gnu services mail) module, but due to also trying to use the define-configuration pattern, it conflicted with the dovecot service. * gnu/services/getmail.scm: New file. * gnu/local.mk: Add it. * gnu/tests/mail.scm (%getmail-os, %test-getmail): New variables. (run-getmail-test): New procedure. --- gnu/tests/mail.scm | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 177 insertions(+), 1 deletion(-) (limited to 'gnu/tests') diff --git a/gnu/tests/mail.scm b/gnu/tests/mail.scm index 33aa4d3437a..10e5be71d89 100644 --- a/gnu/tests/mail.scm +++ b/gnu/tests/mail.scm @@ -4,6 +4,7 @@ ;;; Copyright © 2017 Ludovic Courtès ;;; Copyright © 2018 Oleg Pykhalov ;;; Copyright © 2018 Clément Lassieur +;;; Copyright © 2019 Christopher Baines ;;; ;;; This file is part of GNU Guix. ;;; @@ -25,6 +26,7 @@ #:use-module (gnu system) #:use-module (gnu system vm) #:use-module (gnu services) + #:use-module (gnu services getmail) #:use-module (gnu services mail) #:use-module (gnu services networking) #:use-module (guix gexp) @@ -32,7 +34,8 @@ #:use-module (ice-9 ftw) #:export (%test-opensmtpd %test-exim - %test-dovecot)) + %test-dovecot + %test-getmail)) (define %opensmtpd-os (simple-operating-system @@ -394,3 +397,176 @@ Subject: Hello Nice to meet you!") (name "dovecot") (description "Connect to a running Dovecot server.") (value (run-dovecot-test)))) + +(define %getmail-os + (simple-operating-system + (service dhcp-client-service-type) + (service dovecot-service-type + (dovecot-configuration + (disable-plaintext-auth? #f) + (ssl? "no") + (auth-mechanisms '("anonymous" "plain")) + (auth-anonymous-username "alice") + (mail-location + (string-append "maildir:~/Maildir" + ":INBOX=~/Maildir/INBOX" + ":LAYOUT=fs")))) + (service getmail-service-type + (list + (getmail-configuration + (name 'test) + (user "alice") + (directory "/var/lib/getmail/alice") + (idle '("TESTBOX")) + (rcfile + (getmail-configuration-file + (retriever + (getmail-retriever-configuration + (type "SimpleIMAPRetriever") + (server "localhost") + (username "alice") + (port 143) + (extra-parameters + '((password . "testpass") + (mailboxes . ("TESTBOX")))))) + (destination + (getmail-destination-configuration + (type "Maildir") + (path "/home/alice/TestMaildir/"))) + (options + (getmail-options-configuration + (read-all #f)))))))))) + +(define (run-getmail-test) + "Return a test of an OS running Getmail service." + (define vm + (virtual-machine + (operating-system (marionette-operating-system + %getmail-os + #:imported-modules '((gnu services herd)))) + (port-forwardings '((8143 . 143))))) + + (define test + (with-imported-modules '((gnu build marionette)) + #~(begin + (use-modules (gnu build marionette) + (ice-9 iconv) + (ice-9 rdelim) + (rnrs base) + (rnrs bytevectors) + (srfi srfi-64)) + + (define marionette + (make-marionette '(#$vm))) + + (define* (message-length message #:key (encoding "iso-8859-1")) + (bytevector-length (string->bytevector message encoding))) + + (define message "From: test@example.com\n\ +Subject: Hello Nice to meet you!") + + (mkdir #$output) + (chdir #$output) + + (test-begin "getmail") + + ;; Wait for dovecot to be up and running. + (test-assert "dovecot running" + (marionette-eval + '(begin + (use-modules (gnu services herd)) + (start-service 'dovecot)) + marionette)) + + (test-assert "set password for alice" + (marionette-eval + '(system "echo -e \"testpass\ntestpass\" | passwd alice") + marionette)) + + ;; Wait for getmail to be up and running. + (test-assert "getmail-test running" + (marionette-eval + '(let* ((pw (getpw "alice")) + (uid (passwd:uid pw)) + (gid (passwd:gid pw))) + (use-modules (gnu services herd)) + + (for-each + (lambda (dir) + (mkdir dir) + (chown dir uid gid)) + '("/home/alice/TestMaildir" + "/home/alice/TestMaildir/cur" + "/home/alice/TestMaildir/new" + "/home/alice/TestMaildir/tmp" + "/home/alice/TestMaildir/TESTBOX" + "/home/alice/TestMaildir/TESTBOX/cur" + "/home/alice/TestMaildir/TESTBOX/new" + "/home/alice/TestMaildir/TESTBOX/tmp")) + + (start-service 'getmail-test)) + marionette)) + + ;; Check Dovecot service's PID. + (test-assert "service process id" + (let ((pid + (number->string (wait-for-file "/var/run/dovecot/master.pid" + marionette)))) + (marionette-eval `(file-exists? (string-append "/proc/" ,pid)) + marionette))) + + (test-assert "accept an email" + (let ((imap (socket AF_INET SOCK_STREAM 0)) + (addr (make-socket-address AF_INET INADDR_LOOPBACK 8143))) + (connect imap addr) + ;; Be greeted. + (read-line imap) ;OK + ;; Authenticate + (write-line "a AUTHENTICATE ANONYMOUS" imap) + (read-line imap) ;+ + (write-line "c2lyaGM=" imap) + (read-line imap) ;OK + ;; Create a TESTBOX mailbox + (write-line "a CREATE TESTBOX" imap) + (read-line imap) ;OK + ;; Append a message to a TESTBOX mailbox + (write-line (format #f "a APPEND TESTBOX {~a}" + (number->string (message-length message))) + imap) + (read-line imap) ;+ + (write-line message imap) + (read-line imap) ;OK + ;; Logout + (write-line "a LOGOUT" imap) + (close imap) + #t)) + + (sleep 1) + + (test-assert "mail arrived" + (string-contains + (marionette-eval + '(begin + (use-modules (ice-9 ftw) + (ice-9 match)) + (let ((TESTBOX/new "/home/alice/TestMaildir/new/")) + (match (scandir TESTBOX/new) + (("." ".." message-file) + (call-with-input-file + (string-append TESTBOX/new message-file) + get-string-all))))) + marionette) + message)) + + (test-end) + (exit (= (test-runner-fail-count (test-runner-current)) 0))))) + + (gexp->derivation "getmail-test" test)) + +(define %test-getmail + (system-test + (name "getmail") + (description "Connect to a running Getmail server.") + (value (run-getmail-test)))) + +%getmail-os -- cgit v1.3 From 2177d9222f8c228fe5cd4e9c98d96f97e9601b86 Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Fri, 3 May 2019 19:55:35 +0100 Subject: services: Add patchwork. * gnu/service/web.scm ( , ): New record types. (patchwork-virtualhost): New procedure. (patchwork-service-type): New variable. * gnu/tests/web.scm (%test-patchwork): New variable. * doc/guix.text (Web Services): Document it. --- doc/guix.texi | 174 ++++++++++++++++++++++++ gnu/services/web.scm | 368 ++++++++++++++++++++++++++++++++++++++++++++++++++- gnu/tests/web.scm | 164 ++++++++++++++++++++++- 3 files changed, 702 insertions(+), 4 deletions(-) (limited to 'gnu/tests') diff --git a/doc/guix.texi b/doc/guix.texi index d94b1f2b16d..786788bad7a 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -19389,6 +19389,180 @@ Additional arguments to pass to the @command{varnishd} process. @end table @end deftp +@subsubheading Patchwork +@cindex Patchwork +Patchwork is a patch tracking system. It can collect patches sent to a +mailing list, and display them in a web interface. + +@defvr {Scheme Variable} patchwork-service-type +Service type for Patchwork. +@end defvr + +The following example is an example of a minimal service for Patchwork, for +the @code{patchwork.example.com} domain. + +@example +(service patchwork-service-type + (patchwork-configuration + (domain "patchwork.example.com") + (settings-module + (patchwork-settings-module + (allowed-hosts (list domain)) + (default-from-email "patchwork@@patchwork.example.com"))) + (getmail-retriever-config + (getmail-retriever-configuration + (type "SimpleIMAPSSLRetriever") + (server "imap.example.com") + (port 993) + (username "patchwork") + (password-command + (list (file-append coreutils "/bin/cat") + "/etc/getmail-patchwork-imap-password")) + (extra-parameters + '((mailboxes . ("Patches")))))))) + +@end example + +There are three records for configuring the Patchwork service. The +@code{} relates to the configuration for Patchwork +within the HTTPD service. + +The @code{settings-module} field within the @code{} +record can be populated with the @code{} record, +which describes a settings module that is generated within the Guix store. + +For the @code{database-configuration} field within the +@code{}, the +@code{} must be used. + +@deftp {Data Type} patchwork-configuration +Data type representing the Patchwork service configuration. This type has the +following parameters: + +@table @asis +@item @code{patchwork} (default: @code{patchwork}) +The Patchwork package to use. + +@item @code{domain} +The domain to use for Patchwork, this is used in the HTTPD service virtual +host. + +@item @code{settings-module} +The settings module to use for Patchwork. As a Django application, Patchwork +is configured with a Python module containing the settings. This can either be +an instance of the @code{} record, any other record +that represents the settings in the store, or a directory outside of the +store. + +@item @code{static-path} (default: @code{"/static/"}) +The path under which the HTTPD service should serve the static files. + +@item @code{getmail-retriever-config} +The getmail-retriever-configuration record value to use with +Patchwork. Getmail will be configured with this value, the messages will be +delivered to Patchwork. + +@end table +@end deftp + +@deftp {Data Type} patchwork-settings-module +Data type representing a settings module for Patchwork. Some of these +settings relate directly to Patchwork, but others relate to Django, the web +framework used by Patchwork, or the Django Rest Framework library. This type +has the following parameters: + +@table @asis +@item @code{database-configuration} (default: @code{(patchwork-database-configuration)}) +The database connection settings used for Patchwork. See the +@code{} record type for more information. + +@item @code{secret-key-file} (default: @code{"/etc/patchwork/django-secret-key"}) +Patchwork, as a Django web application uses a secret key for cryptographically +signing values. This file should contain a unique unpredictable value. + +If this file does not exist, it will be created and populated with a random +value by the patchwork-setup shepherd service. + +This setting relates to Django. + +@item @code{allowed-hosts} +A list of valid hosts for this Patchwork service. This should at least include +the domain specified in the @code{} record. + +This is a Django setting. + +@item @code{default-from-email} +The email address from which Patchwork should send email by default. + +This is a Patchwork setting. + +@item @code{static-url} (default: @code{#f}) +The URL to use when serving static assets. It can be part of a URL, or a full +URL, but must end in a @code{/}. + +If the default value is used, the @code{static-path} value from the +@code{} record will be used. + +This is a Django setting. + +@item @code{admins} (default: @code{'()}) +Email addresses to send the details of errors that occur. Each value should +be a list containing two elements, the name and then the email address. + +This is a Django setting. + +@item @code{debug?} (default: @code{#f}) +Whether to run Patchwork in debug mode. If set to @code{#t}, detailed error +messages will be shown. + +This is a Django setting. + +@item @code{enable-rest-api?} (default: @code{#t}) +Whether to enable the Patchwork REST API. + +This is a Patchwork setting. + +@item @code{enable-xmlrpc?} (default: @code{#t}) +Whether to enable the XML RPC API. + +This is a Patchwork setting. + +@item @code{force-https-links?} (default: @code{#t}) +Whether to use HTTPS links on Patchwork pages. + +This is a Patchwork setting. + +@item @code{extra-settings} (default: @code{""}) +Extra code to place at the end of the Patchwork settings module. + +@end table +@end deftp + +@deftp {Data Type} patchwork-database-configuration +Data type representing the database configuration for Patchwork. + +@table @asis +@item @code{engine} (default: @code{"django.db.backends.postgresql_psycopg2"}) +The database engine to use. + +@item @code{name} (default: @code{"patchwork"}) +The name of the database to use. + +@item @code{user} (default: @code{"httpd"}) +The user to connect to the database as. + +@item @code{password} (default: @code{""}) +The password to use when connecting to the database. + +@item @code{host} (default: @code{""}) +The host to make the database connection to. + +@item @code{port} (default: @code{""}) +The port on which to connect to the database. + +@end table +@end deftp + @subsubheading FastCGI @cindex fastcgi @cindex fcgiwrap diff --git a/gnu/services/web.scm b/gnu/services/web.scm index 84294db53bc..35efddb0ae9 100644 --- a/gnu/services/web.scm +++ b/gnu/services/web.scm @@ -7,7 +7,7 @@ ;;; Copyright © 2017 nee ;;; Copyright © 2017, 2018 Clément Lassieur ;;; Copyright © 2018 Pierre-Antoine Rouby -;;; Copyright © 2017 Christopher Baines +;;; Copyright © 2017, 2018, 2019 Christopher Baines ;;; Copyright © 2018 Marius Bakke ;;; ;;; This file is part of GNU Guix. @@ -29,14 +29,23 @@ #:use-module (gnu services) #:use-module (gnu services shepherd) #:use-module (gnu services admin) + #:use-module (gnu services getmail) + #:use-module (gnu services mail) #:use-module (gnu system pam) #:use-module (gnu system shadow) #:use-module (gnu packages admin) + #:use-module (gnu packages databases) #:use-module (gnu packages web) + #:use-module (gnu packages patchutils) #:use-module (gnu packages php) + #:use-module (gnu packages python) + #:use-module (gnu packages gnupg) + #:use-module (gnu packages guile) #:use-module (gnu packages logging) + #:use-module (guix packages) #:use-module (guix records) #:use-module (guix modules) + #:use-module (guix utils) #:use-module (guix gexp) #:use-module ((guix store) #:select (text-file)) #:use-module ((guix utils) #:select (version-major)) @@ -210,7 +219,42 @@ varnish-configuration-parameters varnish-configuration-extra-options - varnish-service-type)) + varnish-service-type + + + patchwork-database-configuration + patchwork-database-configuration? + patchwork-database-configuration-engine + patchwork-database-configuration-name + patchwork-database-configuration-user + patchwork-database-configuration-password + patchwork-database-configuration-host + patchwork-database-configuration-port + + + patchwork-settings-module + patchwork-settings-module? + patchwork-settings-module-database-configuration + patchwork-settings-module-secret-key + patchwork-settings-module-allowed-hosts + patchwork-settings-module-default-from-email + patchwork-settings-module-static-url + patchwork-settings-module-admins + patchwork-settings-module-debug? + patchwork-settings-module-enable-rest-api? + patchwork-settings-module-enable-xmlrpc? + patchwork-settings-module-force-https-links? + patchwork-settings-module-extra-settings + + + patchwork-configuration + patchwork-configuration? + patchwork-configuration-patchwork + patchwork-configuration-settings-module + patchwork-configuration-domain + + patchwork-virtualhost + patchwork-service-type)) ;;; Commentary: ;;; @@ -1268,3 +1312,323 @@ files.") varnish-shepherd-service))) (default-value (varnish-configuration)))) + + +;;; +;;; Patchwork +;;; + +(define-record-type* + patchwork-database-configuration make-patchwork-database-configuration + patchwork-database-configuration? + (engine patchwork-database-configuration-engine + (default "django.db.backends.postgresql_psycopg2")) + (name patchwork-database-configuration-name + (default "patchwork")) + (user patchwork-database-configuration-user + (default "httpd")) + (password patchwork-database-configuration-password + (default "")) + (host patchwork-database-configuration-host + (default "")) + (port patchwork-database-configuration-port + (default ""))) + +(define-record-type* + patchwork-settings-module make-patchwork-settings-module + patchwork-settings-module? + (database-configuration patchwork-settings-module-database-configuration + (default (patchwork-database-configuration))) + (secret-key-file patchwork-settings-module-secret-key-file + (default "/etc/patchwork/django-secret-key")) + (allowed-hosts patchwork-settings-module-allowed-hosts) + (default-from-email patchwork-settings-module-default-from-email) + (static-url patchwork-settings-module-static-url + (default "/static/")) + (admins patchwork-settings-module-admins + (default '())) + (debug? patchwork-settings-module-debug? + (default #f)) + (enable-rest-api? patchwork-settings-module-enable-rest-api? + (default #t)) + (enable-xmlrpc? patchwork-settings-module-enable-xmlrpc? + (default #t)) + (force-https-links? patchwork-settings-module-force-https-links? + (default #t)) + (extra-settings patchwork-settings-module-extra-settings + (default ""))) + +(define-record-type* + patchwork-configuration make-patchwork-configuration + patchwork-configuration? + (patchwork patchwork-configuration-patchwork + (default patchwork)) + (domain patchwork-configuration-domain) + (settings-module patchwork-configuration-settings-module) + (static-path patchwork-configuration-static-url + (default "/static/")) + (getmail-retriever-config getmail-retriever-config)) + +;; Django uses a Python module for configuration, so this compiler generates a +;; Python module from the configuration record. +(define-gexp-compiler (patchwork-settings-module-compiler + (file ) system target) + (match file + (($ database-configuration secret-key-file + allowed-hosts default-from-email + static-url admins debug? enable-rest-api? + enable-xmlrpc? force-https-links? + extra-configuration) + (gexp->derivation + "patchwork-settings" + (with-imported-modules '((guix build utils)) + #~(let ((output #$output)) + (define (create-__init__.py filename) + (call-with-output-file filename + (lambda (port) (display "" port)))) + + (use-modules (guix build utils) + (srfi srfi-1)) + + (mkdir-p (string-append output "/guix/patchwork")) + (create-__init__.py + (string-append output "/guix/__init__.py")) + (create-__init__.py + (string-append output "/guix/patchwork/__init__.py")) + + (call-with-output-file + (string-append output "/guix/patchwork/settings.py") + (lambda (port) + (display + (string-append "from patchwork.settings.base import * + +# Configuration from Guix +with open('" #$secret-key-file "') as f: + SECRET_KEY = f.read().strip() + +ALLOWED_HOSTS = [ +" #$(string-concatenate + (map (lambda (allowed-host) + (string-append " '" allowed-host "'\n")) + allowed-hosts)) +"] + +ADMINS = [ +" #$(string-concatenate + (map (match-lambda + ((name email-address) + (string-append + "('" name "','" email-address "'),"))) + admins)) +"] + +DEBUG = " #$(if debug? "True" "False") " + +ENABLE_REST_API = " #$(if enable-xmlrpc? "True" "False") " +ENABLE_XMLRPC = " #$(if enable-xmlrpc? "True" "False") " + +FORCE_HTTPS_LINKS = " #$(if force-https-links? "True" "False") " + +DATABASES = { + 'default': { +" #$(match database-configuration + (($ + engine name user password host port) + (string-append + " 'ENGINE': '" engine "',\n" + " 'NAME': '" name "',\n" + " 'USER': '" user "',\n" + " 'PASSWORD': '" password "',\n" + " 'HOST': '" host "',\n" + " 'PORT': '" port "',\n"))) " + }, +} + +" #$(if debug? + #~(string-append "STATIC_ROOT = '" + #$(file-append patchwork "/share/patchwork/htdocs") + "'") + #~(string-append "STATIC_URL = '" #$static-url "'")) " + +STATICFILES_STORAGE = ( + 'django.contrib.staticfiles.storage.StaticFilesStorage' +) + +# Guix Extra Configuration +" #$extra-configuration " +") port))) + #t)) + #:local-build? #t)))) + +(define patchwork-virtualhost + (match-lambda + (($ patchwork domain + settings-module static-path + getmail-retriever-config) + (define wsgi.py + (file-append patchwork + (string-append + "/lib/python" + (version-major+minor + (package-version python)) + "/site-packages/patchwork/wsgi.py"))) + + (httpd-virtualhost + "*:8080" + `("ServerAdmin admin@example.com` +ServerName " ,domain " + +LogFormat \"%v %h %l %u %t \\\"%r\\\" %>s %b \\\"%{Referer}i\\\" \\\"%{User-Agent}i\\\"\" customformat +LogLevel info +CustomLog \"/var/log/httpd/" ,domain "-access_log\" customformat + +ErrorLog /var/log/httpd/error.log + +WSGIScriptAlias / " ,wsgi.py " +WSGIDaemonProcess " ,(package-name patchwork) " user=httpd group=httpd processes=1 threads=2 display-name=%{GROUP} lang='en_US.UTF-8' locale='en_US.UTF-8' python-path=" ,settings-module " +WSGIProcessGroup " ,(package-name patchwork) " +WSGIPassAuthorization On + + + Require all granted + + +" ,@(if static-path + `("Alias " ,static-path " " ,patchwork "/share/patchwork/htdocs/") + '()) +" + + AllowOverride None + Options MultiViews Indexes SymlinksIfOwnerMatch IncludesNoExec + Require method GET POST OPTIONS +"))))) + +(define (patchwork-httpd-configuration patchwork-configuration) + (list "WSGISocketPrefix /var/run/mod_wsgi" + (list "LoadModule wsgi_module " + (file-append mod-wsgi "/modules/mod_wsgi.so")) + (patchwork-virtualhost patchwork-configuration))) + +(define (patchwork-django-admin-gexp patchwork settings-module) + #~(lambda command + (let ((pid (primitive-fork)) + (user (getpwnam "httpd"))) + (if (eq? pid 0) + (dynamic-wind + (const #t) + (lambda () + (setgid (passwd:gid user)) + (setuid (passwd:uid user)) + + (setenv "DJANGO_SETTINGS_MODULE" "guix.patchwork.settings") + (setenv "PYTHONPATH" #$settings-module) + (primitive-exit + (if (zero? + (apply system* + #$(file-append patchwork "/bin/patchwork-admin") + command)) + 0 + 1))) + (lambda () + (primitive-exit 1))) + (zero? (cdr (waitpid pid))))))) + +(define (patchwork-django-admin-action patchwork settings-module) + (shepherd-action + (name 'django-admin) + (documentation + "Run a django admin command for patchwork") + (procedure (patchwork-django-admin-gexp patchwork settings-module)))) + +(define patchwork-shepherd-services + (match-lambda + (($ patchwork domain + settings-module static-path + getmail-retriever-config) + (define secret-key-file-creation-gexp + (if (patchwork-settings-module? settings-module) + (with-extensions (list guile-gcrypt) + #~(let ((secret-key-file + #$(patchwork-settings-module-secret-key-file + settings-module))) + (use-modules (guix build utils) + (gcrypt random)) + + (unless (file-exists? secret-key-file) + (mkdir-p (dirname secret-key-file)) + (call-with-output-file secret-key-file + (lambda (port) + (display (random-token 30 'very-strong) port))) + (let* ((pw (getpwnam "httpd")) + (uid (passwd:uid pw)) + (gid (passwd:gid pw))) + (chown secret-key-file uid gid) + (chmod secret-key-file #o400))))) + #~())) + + (list (shepherd-service + (requirement '(postgres)) + (provision (list (string->symbol + (string-append (package-name patchwork) + "-setup")))) + (start + #~(lambda () + (define run-django-admin-command + #$(patchwork-django-admin-gexp patchwork + settings-module)) + + #$secret-key-file-creation-gexp + + (run-django-admin-command "migrate"))) + (stop #~(const #f)) + (actions + (list (patchwork-django-admin-action patchwork + settings-module))) + (respawn? #f) + (documentation "Setup Patchwork.")))))) + +(define patchwork-getmail-configs + (match-lambda + (($ patchwork domain + settings-module static-path + getmail-retriever-config) + (list + (getmail-configuration + (name (string->symbol (package-name patchwork))) + (user "httpd") + (directory (string-append + "/var/lib/getmail/" (package-name patchwork))) + (rcfile + (getmail-configuration-file + (retriever getmail-retriever-config) + (destination + (getmail-destination-configuration + (type "MDA_external") + (path (file-append patchwork "/bin/patchwork-admin")) + (extra-parameters + '((arguments . ("parsemail")))))) + (options + (getmail-options-configuration + (read-all #f) + (delivered-to #f) + (received #f))))) + (idle (assq-ref + (getmail-retriever-configuration-extra-parameters + getmail-retriever-config) + 'mailboxes)) + (environment-variables + (list "DJANGO_SETTINGS_MODULE=guix.patchwork.settings" + #~(string-append "PYTHONPATH=" #$settings-module)))))))) + +(define patchwork-service-type + (service-type + (name 'patchwork-setup) + (extensions + (list (service-extension httpd-service-type + patchwork-httpd-configuration) + (service-extension shepherd-root-service-type + patchwork-shepherd-services) + (service-extension getmail-service-type + patchwork-getmail-configs))) + (description + "Patchwork patch tracking system."))) diff --git a/gnu/tests/web.scm b/gnu/tests/web.scm index 319655396a8..7c1c0aa511e 100644 --- a/gnu/tests/web.scm +++ b/gnu/tests/web.scm @@ -1,6 +1,6 @@ ;;; GNU Guix --- Functional package management for GNU ;;; Copyright © 2017 Ludovic Courtès -;;; Copyright © 2017 Christopher Baines +;;; Copyright © 2017, 2019 Christopher Baines ;;; Copyright © 2017, 2018 Clément Lassieur ;;; Copyright © 2018 Pierre-Antoine Rouby ;;; Copyright © 2018 Marius Bakke @@ -28,15 +28,29 @@ #:use-module (gnu system vm) #:use-module (gnu services) #:use-module (gnu services web) + #:use-module (gnu services databases) + #:use-module (gnu services getmail) #:use-module (gnu services networking) + #:use-module (gnu services shepherd) + #:use-module (gnu services mail) + #:use-module (gnu packages databases) + #:use-module (gnu packages patchutils) + #:use-module (gnu packages python) + #:use-module (gnu packages web) + #:use-module (guix packages) + #:use-module (guix modules) + #:use-module (guix records) #:use-module (guix gexp) #:use-module (guix store) + #:use-module (guix utils) + #:use-module (ice-9 match) #:export (%test-httpd %test-nginx %test-varnish %test-php-fpm %test-hpcguix-web - %test-tailon)) + %test-tailon + %test-patchwork)) (define %index.html-contents ;; Contents of the /index.html file. @@ -498,3 +512,149 @@ HTTP-PORT." (name "tailon") (description "Connect to a running Tailon server.") (value (run-tailon-test)))) + + +;;; +;;; Patchwork +;;; + +(define patchwork-initial-database-setup-service + (match-lambda + (($ + engine name user password host port) + + (define start-gexp + #~(lambda () + (let ((pid (primitive-fork)) + (postgres (getpwnam "postgres"))) + (if (eq? pid 0) + (dynamic-wind + (const #t) + (lambda () + (setgid (passwd:gid postgres)) + (setuid (passwd:uid postgres)) + (primitive-exit + (if (and + (zero? + (system* #$(file-append postgresql "/bin/createuser") + #$user)) + (zero? + (system* #$(file-append postgresql "/bin/createdb") + "-O" #$user #$name))) + 0 + 1))) + (lambda () + (primitive-exit 1))) + (zero? (cdr (waitpid pid))))))) + + (shepherd-service + (requirement '(postgres)) + (provision '(patchwork-postgresql-user-and-database)) + (start start-gexp) + (stop #~(const #f)) + (respawn? #f) + (documentation "Setup patchwork database."))))) + +(define (patchwork-os patchwork) + (simple-operating-system + (service dhcp-client-service-type) + (service httpd-service-type + (httpd-configuration + (config + (httpd-config-file + (listen '("8080")))))) + (service postgresql-service-type) + (service patchwork-service-type + (patchwork-configuration + (patchwork patchwork) + (domain "localhost") + (settings-module + (patchwork-settings-module + (allowed-hosts (list domain)) + (default-from-email ""))) + (getmail-retriever-config + (getmail-retriever-configuration + (type "SimpleIMAPSSLRetriever") + (server "imap.example.com") + (port 993) + (username "username") + (password "password") + (extra-parameters + '((mailboxes . ("INBOX")))))))) + (simple-service 'patchwork-database-setup + shepherd-root-service-type + (list + (patchwork-initial-database-setup-service + (patchwork-database-configuration)))))) + +(define (run-patchwork-test patchwork) + "Run tests in %NGINX-OS, which has nginx running and listening on +HTTP-PORT." + (define os + (marionette-operating-system + (patchwork-os patchwork) + #:imported-modules '((gnu services herd) + (guix combinators)))) + + (define forwarded-port 8080) + + (define vm + (virtual-machine + (operating-system os) + (port-forwardings `((8080 . ,forwarded-port))))) + + (define test + (with-imported-modules '((gnu build marionette)) + #~(begin + (use-modules (srfi srfi-11) (srfi srfi-64) + (gnu build marionette) + (web uri) + (web client) + (web response)) + + (define marionette + (make-marionette (list #$vm))) + + (mkdir #$output) + (chdir #$output) + + (test-begin "patchwork") + + (test-assert "patchwork-postgresql-user-and-service started" + (marionette-eval + '(begin + (use-modules (gnu services herd)) + (match (start-service 'patchwork-postgresql-user-and-database) + (#f #f) + (('service response-parts ...) + (match (assq-ref response-parts 'running) + ((#t) #t) + ((pid) (number? pid)))))) + marionette)) + + (test-assert "httpd running" + (marionette-eval + '(begin + (use-modules (gnu services herd)) + (start-service 'httpd)) + marionette)) + + (test-equal "http-get" + 200 + (let-values + (((response text) + (http-get #$(simple-format + #f "http://localhost:~A/" forwarded-port) + #:decode-body? #t))) + (response-code response))) + + (test-end) + (exit (= (test-runner-fail-count (test-runner-current)) 0))))) + + (gexp->derivation "patchwork-test" test)) + +(define %test-patchwork + (system-test + (name "patchwork") + (description "Connect to a running Patchwork service.") + (value (run-patchwork-test patchwork)))) -- cgit v1.3 From 08814aec6ae75adcd059c5235c90ad26e5d5607e Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 4 Jun 2019 22:29:40 +0200 Subject: services: Add Singularity. * gnu/packages/linux.scm (singularity)[source](snippet): Change file name of setuid helpers in libexec/cli/*.exec. [arguments]: Remove "--disable-suid". * gnu/services/docker.scm (%singularity-activation): New variable. (singularity-setuid-programs): New procedure. (singularity-service-type): New variable. * gnu/tests/singularity.scm: New file. * gnu/local.mk (GNU_SYSTEM_MODULES): Add it. * doc/guix.texi (Miscellaneous Services): Document it. --- doc/guix.texi | 13 ++++- gnu/local.mk | 1 + gnu/packages/linux.scm | 10 ++-- gnu/services/docker.scm | 61 +++++++++++++++++++++- gnu/tests/singularity.scm | 128 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 gnu/tests/singularity.scm (limited to 'gnu/tests') diff --git a/doc/guix.texi b/doc/guix.texi index bdfe14c7240..d37d63066fd 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -24090,7 +24090,7 @@ The following is an example @code{dicod-service} configuration. @cindex Docker @subsubheading Docker Service -The @code{(gnu services docker)} module provides the following service. +The @code{(gnu services docker)} module provides the following services. @defvr {Scheme Variable} docker-service-type @@ -24163,6 +24163,17 @@ The audit package to use. @end table @end deftp +@defvr {Scheme Variable} singularity-service-type +This is the type of the service that allows you to run +@url{https://www.sylabs.io/singularity/, Singularity}, a Docker-style tool to +create and run application bundles (aka. ``containers''). The value for this +service is the Singularity package to use. + +The service does not install a daemon; instead, it installs helper programs as +setuid-root (@pxref{Setuid Programs}) such that unprivileged users can invoke +@command{singularity run} and similar commands. +@end defvr + @node Setuid Programs @section Setuid Programs diff --git a/gnu/local.mk b/gnu/local.mk index 203445ef1b1..98f6ee9679c 100644 --- a/gnu/local.mk +++ b/gnu/local.mk @@ -587,6 +587,7 @@ GNU_SYSTEM_MODULES = \ %D%/tests/networking.scm \ %D%/tests/rsync.scm \ %D%/tests/security-token.scm \ + %D%/tests/singularity.scm \ %D%/tests/ssh.scm \ %D%/tests/version-control.scm \ %D%/tests/virtualization.scm \ diff --git a/gnu/packages/linux.scm b/gnu/packages/linux.scm index b2f43bb1f74..cf3b838ea8a 100644 --- a/gnu/packages/linux.scm +++ b/gnu/packages/linux.scm @@ -2899,12 +2899,16 @@ thanks to the use of namespaces.") (substitute* "bin/singularity.in" (("^PATH=.*" all) (string-append "#" all "\n"))) + + (substitute* (find-files "libexec/cli" "\\.exec$") + (("\\$SINGULARITY_libexecdir/singularity/bin/([a-z]+)-suid" + _ program) + (string-append "/run/setuid-programs/singularity-" + program "-helper"))) #t)))) (build-system gnu-build-system) (arguments - `(#:configure-flags - (list "--disable-suid" - "--localstatedir=/var") + `(#:configure-flags '("--localstatedir=/var") #:phases (modify-phases %standard-phases (add-after 'unpack 'patch-reference-to-squashfs-tools diff --git a/gnu/services/docker.scm b/gnu/services/docker.scm index 94a04c8996f..04f91273464 100644 --- a/gnu/services/docker.scm +++ b/gnu/services/docker.scm @@ -24,12 +24,14 @@ #:use-module (gnu services shepherd) #:use-module (gnu system shadow) #:use-module (gnu packages docker) + #:use-module (gnu packages linux) ;singularity #:use-module (guix records) #:use-module (guix gexp) #:use-module (guix packages) #:export (docker-configuration - docker-service-type)) + docker-service-type + singularity-service-type)) ;;; We're not using serialize-configuration, but we must define this because ;;; the define-configuration macro validates it exists. @@ -120,3 +122,60 @@ bundles in Docker containers.") (service-extension account-service-type (const %docker-accounts)))) (default-value (docker-configuration)))) + + +;;; +;;; Singularity. +;;; + +(define %singularity-activation + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (define %mount-directory + "/var/singularity/mnt/") + + ;; Create the directories that Singularity 2.6 expects to find. Make + ;; them #o755 like the 'install-data-hook' rule in 'Makefile.am' of + ;; Singularity 2.6.1. + (for-each (lambda (directory) + (let ((directory (string-append %mount-directory + directory))) + (mkdir-p directory) + (chmod directory #o755))) + '("container" "final" "overlay" "session")) + (chmod %mount-directory #o755)))) + +(define (singularity-setuid-programs singularity) + "Return the setuid-root programs that SINGULARITY needs." + (define helpers + ;; The helpers, under a meaningful name. + (computed-file "singularity-setuid-helpers" + #~(begin + (mkdir #$output) + (for-each (lambda (program) + (symlink (string-append #$singularity + "/libexec/singularity" + "/bin/" + program "-suid") + (string-append #$output + "/singularity-" + program + "-helper"))) + '("action" "mount" "start"))))) + + (list (file-append helpers "/singularity-action-helper") + (file-append helpers "/singularity-mount-helper") + (file-append helpers "/singularity-start-helper"))) + +(define singularity-service-type + (service-type (name 'singularity) + (description + "Install the Singularity application bundle tool.") + (extensions + (list (service-extension setuid-program-service-type + singularity-setuid-programs) + (service-extension activation-service-type + (const %singularity-activation)))) + (default-value singularity))) diff --git a/gnu/tests/singularity.scm b/gnu/tests/singularity.scm new file mode 100644 index 00000000000..55324ef9ea0 --- /dev/null +++ b/gnu/tests/singularity.scm @@ -0,0 +1,128 @@ +;;; GNU Guix --- Functional package management for GNU +;;; Copyright © 2019 Ludovic Courtès +;;; +;;; This file is part of GNU Guix. +;;; +;;; GNU Guix is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; GNU Guix is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with GNU Guix. If not, see . + +(define-module (gnu tests singularity) + #:use-module (gnu tests) + #:use-module (gnu system) + #:use-module (gnu system vm) + #:use-module (gnu system shadow) + #:use-module (gnu services) + #:use-module (gnu services docker) + #:use-module (gnu packages bash) + #:use-module (gnu packages guile) + #:use-module (gnu packages linux) ;singularity + #:use-module (guix gexp) + #:use-module (guix store) + #:use-module (guix grafts) + #:use-module (guix monads) + #:use-module (guix packages) + #:use-module (guix profiles) + #:use-module (guix scripts pack) + #:export (%test-singularity)) + +(define %singularity-os + (simple-operating-system + (service singularity-service-type) + (simple-service 'guest-account + account-service-type + (list (user-account (name "guest") (uid 1000) (group "guest")) + (user-group (name "guest") (id 1000)))))) + +(define (run-singularity-test image) + "Load IMAGE, a Squashfs image, as a Singularity image and run it inside +%SINGULARITY-OS." + (define os + (marionette-operating-system %singularity-os)) + + (define singularity-exec + #~(begin + (use-modules (ice-9 popen) (rnrs io ports)) + + (let* ((pipe (open-pipe* OPEN_READ + #$(file-append singularity + "/bin/singularity") + "exec" #$image "/bin/guile" + "-c" "(display \"hello, world\")")) + (str (get-string-all pipe)) + (status (close-pipe pipe))) + (and (zero? status) + (string=? str "hello, world"))))) + + (define test + (with-imported-modules '((gnu build marionette)) + #~(begin + (use-modules (srfi srfi-11) (srfi srfi-64) + (gnu build marionette)) + + (define marionette + (make-marionette (list #$(virtual-machine os)))) + + (mkdir #$output) + (chdir #$output) + + (test-begin "singularity") + + (test-assert "singularity exec /bin/guile (as root)" + (marionette-eval '#$singularity-exec + marionette)) + + (test-equal "singularity exec /bin/guile (unprivileged)" + 0 + (marionette-eval + `(begin + (use-modules (ice-9 match)) + + (match (primitive-fork) + (0 + (dynamic-wind + (const #f) + (lambda () + (setgid 1000) + (setuid 1000) + (execl #$(program-file "singularity-exec-test" + #~(exit #$singularity-exec)) + "test")) + (lambda () + (primitive-exit 127)))) + (pid + (cdr (waitpid pid))))) + marionette)) + + (test-end) + (exit (= (test-runner-fail-count (test-runner-current)) 0))))) + + (gexp->derivation "singularity-test" test)) + +(define (build-tarball&run-singularity-test) + (mlet* %store-monad + ((_ (set-grafting #f)) + (guile (set-guile-for-build (default-guile))) + ;; 'singularity exec' insists on having /bin/sh in the image. + (profile (profile-derivation (packages->manifest + (list bash-minimal guile-2.2)) + #:hooks '() + #:locales? #f)) + (tarball (squashfs-image "singularity-pack" profile + #:symlinks '(("/bin" -> "bin"))))) + (run-singularity-test tarball))) + +(define %test-singularity + (system-test + (name "singularity") + (description "Test Singularity container of Guix.") + (value (build-tarball&run-singularity-test)))) -- cgit v1.3 From a0f352b30f4869a7af7017b8a5011ac7602dd115 Mon Sep 17 00:00:00 2001 From: Ludovic Courtès Date: Tue, 4 Jun 2019 18:43:23 +0200 Subject: pack: Add '--entry-point'. * guix/scripts/pack.scm (self-contained-tarball): Add #:entry-point and warn when it's true. (squashfs-image): Add #:entry-point and honor it. (docker-image): Add #:entry-point and honor it. (%options, show-help): Add '--entry-point'. (guix-pack): Honor '--entry-point' and pass #:entry-point to BUILD-IMAGE. * gnu/tests/docker.scm (run-docker-test): Test 'docker run' with the default entry point. (build-tarball&run-docker-test): Pass #:entry-point to 'docker-image'. * doc/guix.texi (Invoking guix pack): Document it. * gnu/tests/singularity.scm (run-singularity-test)["singularity run"]: New test. (build-tarball&run-singularity-test): Pass #:entry-point to 'squashfs-image'. --- doc/guix.texi | 23 +++++++++++++++++++++++ gnu/tests/docker.scm | 19 ++++++++++++------- gnu/tests/singularity.scm | 9 +++++++++ guix/scripts/pack.scm | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 7 deletions(-) (limited to 'gnu/tests') diff --git a/doc/guix.texi b/doc/guix.texi index d37d63066fd..bd0f3e8fd5a 100644 --- a/doc/guix.texi +++ b/doc/guix.texi @@ -4866,6 +4866,29 @@ advantage to work without requiring special kernel support, but it incurs run-time overhead every time a system call is made. @end quotation +@cindex entry point, for Docker images +@item --entry-point=@var{command} +Use @var{command} as the @dfn{entry point} of the resulting pack, if the pack +format supports it---currently @code{docker} and @code{squashfs} (Singularity) +support it. @var{command} must be relative to the profile contained in the +pack. + +The entry point specifies the command that tools like @code{docker run} or +@code{singularity run} automatically start by default. For example, you can +do: + +@example +guix pack -f docker --entry-point=bin/guile guile +@end example + +The resulting pack can easily be loaded and @code{docker run} with no extra +arguments will spawn @code{bin/guile}: + +@example +docker load -i pack.tar.gz +docker run @var{image-id} +@end example + @item --expression=@var{expr} @itemx -e @var{expr} Consider the package @var{expr} evaluates to. diff --git a/gnu/tests/docker.scm b/gnu/tests/docker.scm index 3cd3a278840..f2674cdbe83 100644 --- a/gnu/tests/docker.scm +++ b/gnu/tests/docker.scm @@ -101,7 +101,7 @@ inside %DOCKER-OS." marionette)) (test-equal "Load docker image and run it" - "hello world" + '("hello world" "hi!") (marionette-eval `(begin (define slurp @@ -117,12 +117,16 @@ inside %DOCKER-OS." (repository&tag (string-drop raw-line (string-length "Loaded image: "))) - (response (slurp - ,(string-append #$docker-cli "/bin/docker") - "run" "--entrypoint" "bin/Guile" - repository&tag - "/aa.scm"))) - response)) + (response1 (slurp + ,(string-append #$docker-cli "/bin/docker") + "run" "--entrypoint" "bin/Guile" + repository&tag + "/aa.scm")) + (response2 (slurp ;default entry point + ,(string-append #$docker-cli "/bin/docker") + "run" repository&tag + "-c" "(display \"hi!\")"))) + (list response1 response2))) marionette)) (test-end) @@ -161,6 +165,7 @@ standard output device and then enters a new line.") (tarball (docker-image "docker-pack" profile #:symlinks '(("/bin/Guile" -> "bin/guile") ("aa.scm" -> "a.scm")) + #:entry-point "bin/guile" #:localstatedir? #t))) (run-docker-test tarball))) diff --git a/gnu/tests/singularity.scm b/gnu/tests/singularity.scm index 55324ef9ea0..668043a0bc0 100644 --- a/gnu/tests/singularity.scm +++ b/gnu/tests/singularity.scm @@ -103,6 +103,14 @@ (cdr (waitpid pid))))) marionette)) + (test-equal "singularity run" ;test the entry point + 42 + (marionette-eval + `(status:exit-val + (system* #$(file-append singularity "/bin/singularity") + "run" #$image "-c" "(exit 42)")) + marionette)) + (test-end) (exit (= (test-runner-fail-count (test-runner-current)) 0))))) @@ -118,6 +126,7 @@ #:hooks '() #:locales? #f)) (tarball (squashfs-image "singularity-pack" profile + #:entry-point "bin/guile" #:symlinks '(("/bin" -> "bin"))))) (run-singularity-test tarball))) diff --git a/guix/scripts/pack.scm b/guix/scripts/pack.scm index c17b3743301..5da23e038b1 100644 --- a/guix/scripts/pack.scm +++ b/guix/scripts/pack.scm @@ -152,6 +152,7 @@ dependencies are registered." #:key target (profile-name "guix-profile") deduplicate? + entry-point (compressor (first %compressors)) localstatedir? (symlinks '()) @@ -275,6 +276,10 @@ added to the pack." (_ #f)) directives))))))))) + (when entry-point + (warning (G_ "entry point not supported in the '~a' format~%") + 'tarball)) + (gexp->derivation (string-append name ".tar" (compressor-extension compressor)) build @@ -284,6 +289,7 @@ added to the pack." #:key target (profile-name "guix-profile") (compressor (first %compressors)) + entry-point localstatedir? (symlinks '()) (archiver squashfs-tools-next)) @@ -315,6 +321,7 @@ added to the pack." (ice-9 match)) (define database #+database) + (define entry-point #$entry-point) (setenv "PATH" (string-append #$archiver "/bin")) @@ -371,6 +378,28 @@ added to the pack." target))))))) '#$symlinks) + ;; Create /.singularity.d/actions, and optionally the 'run' + ;; script, used by 'singularity run'. + "-p" "/.singularity.d d 555 0 0" + "-p" "/.singularity.d/actions d 555 0 0" + ,@(if entry-point + `(;; This one if for Singularity 2.x. + "-p" + ,(string-append + "/.singularity.d/actions/run s 777 0 0 " + (relative-file-name "/.singularity.d/actions" + (string-append #$profile "/" + entry-point))) + + ;; This one is for Singularity 3.x. + "-p" + ,(string-append + "/.singularity.d/runscript s 777 0 0 " + (relative-file-name "/.singularity.d" + (string-append #$profile "/" + entry-point)))) + '()) + ;; Create empty mount points. "-p" "/proc d 555 0 0" "-p" "/sys d 555 0 0" @@ -392,6 +421,7 @@ added to the pack." #:key target (profile-name "guix-profile") (compressor (first %compressors)) + entry-point localstatedir? (symlinks '()) (archiver tar)) @@ -425,6 +455,8 @@ the image." #$profile #:database #+database #:system (or #$target (utsname:machine (uname))) + #:entry-point (string-append #$profile "/" + #$entry-point) #:symlinks '#$symlinks #:compressor '#$(compressor-command compressor) #:creation-time (make-time time-utc 0 1)))))) @@ -689,6 +721,9 @@ please email '~a'~%") (lambda (opt name arg result) (alist-cons 'system arg (alist-delete 'system result eq?)))) + (option '("entry-point") #t #f + (lambda (opt name arg result) + (alist-cons 'entry-point arg result))) (option '("target") #t #f (lambda (opt name arg result) (alist-cons 'target arg @@ -765,6 +800,9 @@ Create a bundle of PACKAGE.\n")) -S, --symlink=SPEC create symlinks to the profile according to SPEC")) (display (G_ " -m, --manifest=FILE create a pack with the manifest from FILE")) + (display (G_ " + --entry-point=PROGRAM + use PROGRAM as the entry point of the pack")) (display (G_ " --save-provenance save provenance information")) (display (G_ " @@ -889,6 +927,7 @@ Create a bundle of PACKAGE.\n")) (leave (G_ "~a: unknown pack format~%") pack-format)))) (localstatedir? (assoc-ref opts 'localstatedir?)) + (entry-point (assoc-ref opts 'entry-point)) (profile-name (assoc-ref opts 'profile-name)) (gc-root (assoc-ref opts 'gc-root))) (when (null? (manifest-entries manifest)) @@ -919,6 +958,8 @@ Create a bundle of PACKAGE.\n")) symlinks #:localstatedir? localstatedir? + #:entry-point + entry-point #:profile-name profile-name #:archiver -- cgit v1.3