Commit 3edceba2 authored by Ludovic Courtès's avatar Ludovic Courtès
Browse files

kernels: Define <connection> record type and use it.

* jupyter/kernels.scm (<kernel>): Use 'define-immutable-record-type'.
Add 'set-kernel-name' and 'set-kernel-pid'.
(find-kernel-by-name): Rename to...
(find-kernel-specs-file): ... this.
(find-kernel-specs): New procedure.
(<connection>): New record type.
(new-connection-file, kernel-arguments, exec-kernel): Remove.
(allocate-connection, spawn-kernel): New procedure.
(run-kernel): Rewrite to take a <kernel-spec> and to use
'allocate-connection'.
* tests/kernels.scm (%python3-specs): New variable.
("run-kernel python3"): Adjust accordingly.
* guix-jupyter-container.scm (reply-execute-request): Likewise.
parent 825da79e
......@@ -237,7 +237,8 @@ KEY for signing."
(socket->kernel kernel notebook-key)
(string->utf8 name) message)
(values kernels (+ count 1)))
(let* ((new-kernel (run-kernel context name
(let* ((new-kernel (run-kernel context
(find-kernel-specs name)
notebook-key
#:identity
(string->utf8 name)))
......
......@@ -26,8 +26,11 @@
#:use-module (rnrs bytevectors)
#:use-module (srfi srfi-1)
#:use-module (srfi srfi-9)
#:use-module (srfi srfi-9 gnu)
#:use-module (srfi srfi-26)
#:export (find-kernel-by-name
#:use-module (srfi srfi-71)
#:export (find-kernel-specs
find-kernel-specs-file
run-kernel
kernel-specs?
......@@ -49,6 +52,18 @@
kernel-iopub ;alias
kernel-sockets
connection?
connection-transport
connection-ip
connection-signature-scheme
connection-key
connection-control-port
connection-shell-port
connection-stdin-port
connection-heartbeat-port
connection-iopub-port
json->connection
read-message
send-message
relay-message
......@@ -71,11 +86,11 @@
;;
;; Type of a Jupyter kernel.
(define-record-type <kernel>
(define-immutable-record-type <kernel>
(%kernel name pid key control shell stdin heartbeat iosub)
kernel?
(name kernel-name) ;string
(pid kernel-pid) ;integer
(name kernel-name set-kernel-name) ;string
(pid kernel-pid set-kernel-pid) ;integer
(key kernel-key) ;bytevector
(control kernel-control) ;zmq socket
(shell kernel-shell) ;zmq socket
......@@ -125,8 +140,27 @@
(lambda (profile)
(list (string-append profile "/share/jupyter"))))))
(define* (find-kernel-by-name kernel
#:optional (path (jupyter-kernel-path)))
(define* (find-kernel-specs kernel
#:optional (path (jupyter-kernel-path)))
"Return the kernel specs for KERNEL, or #f if KERNEL could not be found in
PATH."
(match kernel
("ipython"
;; IPython does not provide a 'kernel.json' file like other
;; kernels do; instead, we need to execute 'ipython' from $PATH.
(let ((ipython (search-path (search-path->list (getenv "PATH"))
"ipython")))
(and ipython
(kernel-specs
(list ipython "kernel" "--quiet" "-f" "{connection_file}")
"IPython"
"Python"))))
(_
(let ((file (find-kernel-specs-file kernel)))
(and file (call-with-input-file file json->kernel-specs))))))
(define* (find-kernel-specs-file kernel
#:optional (path (jupyter-kernel-path)))
"Return the absolute file name of the 'kernel.json' file for Jupyter kernel
KERNEL, searched for in PATH, a list of directories. Return #f if KERNEL
could not be found."
......@@ -136,116 +170,73 @@ could not be found."
(and (file-exists? json) json)))
path))
;;
;; Running procedure.
;; Kernel "connection files".
;;
(define* (new-connection-file type ip key
#:key control shell heartbeat stdin iosub)
"Return the file name of a new JSON \"connection file\" for the given IP
and TYPE (e.g., \"tcp\"), with the given notebook KEY, and the specified
sockets."
(let* ((scm `(("transport" . ,type)
("ip" . ,ip)
("signature_scheme" . "hmac-sha256")
("key" . ,key)
("control_port" . ,control)
("shell_port" . ,shell)
("stdin_port" . ,stdin)
("hb_port" . ,heartbeat)
("iopub_port" . ,iosub)))
(json (scm->json-string scm))
(file (string-append "/tmp/guix-kernel/conn/"
(number->string (string-hash json))
".json")))
(mkdir-p (dirname file))
(call-with-output-file file
(lambda (port)
(scm->json scm port #:pretty #t)
file))))
(define (test-bind socket port)
(catch 'zmq-error
(lambda ()
(zmq-bind-socket socket (string-append "tcp://127.0.0.1:"
(number->string port)))
(zmq-unbind-socket socket (string-append "tcp://127.0.0.1:"
(number->string port)))
port)
(lambda stuff
(let ((errno (cadr stuff)))
(cond
((= errno EADDRINUSE) ;The requested address is
;already in use.
(test-bind socket (+ port 1)))
(else #f))))))
(define (kernel-arguments kernel-file connection-file)
"Return the argument list read from KERNEL-FILE, substituting references to
'connection-file' with CONNECTION-FILE."
(let ((specs (call-with-input-file kernel-file json->kernel-specs)))
(map (match-lambda
("{connection_file}" connection-file)
(x x))
(kernel-specs-arguments specs))))
(define search-path->list
(let ((not-colon (char-set-complement (char-set #\:))))
(match-lambda
(#f '())
((? string? str)
(string-tokenize str not-colon)))))
(define (exec-kernel kernel connection-file)
"Execute KERNEL as a separate process, passing it CONNECTION-FILE, and
return its PID."
(define arguments
(cond ((string=? kernel "ipython")
;; IPython does not provide a 'kernel.json' file like other
;; kernels do; instead, we need to execute 'ipython' from $PATH.
(let ((ipython (search-path (search-path->list (getenv "PATH"))
"ipython")))
(list ipython "kernel" "--quiet" "-f" connection-file)))
(else
(kernel-arguments (find-kernel-by-name kernel)
connection-file))))
(let ((pid (primitive-fork)))
(if (zero? pid)
(match arguments
((command . _)
(apply execl command arguments)))
pid)))
(define* (run-kernel context name key #:key identity)
;; A "connection" represents a rendezvous between the client and a kernel:
;; <https://jupyter-client.readthedocs.io/en/stable/kernels.html#connection-files>.
;; Usually the client creates it, writes it to a "connection file", which it
;; passes to the kernel.
(define-json-mapping <connection> %connection connection?
json->connection <=> connection->json
(transport connection-transport) ;string
(ip connection-ip) ;string
(signature-scheme connection-signature-scheme) ;string
(key connection-key) ;string
(control-port connection-control-port ;integer
"control_port")
(shell-port connection-shell-port ;integer
"shell_port")
(stdin-port connection-stdin-port ;integer
"stdin_port")
(heartbeat-port connection-heartbeat-port ;integer
"hb_port")
(iopub-port connection-iopub-port ;integer
"iopub_port"))
(define* (connection transport ip
#:key
(signature-scheme "hmac-sha256")
key control-port shell-port stdin-port heartbeat-port
iopub-port)
(%connection transport ip signature-scheme key
control-port shell-port stdin-port heartbeat-port
iopub-port))
(define* (allocate-connection context transport ip key
#:key
(first-port 1024) identity)
"Allocate ports for TRANSPORT and IP, starting at FIRST-PORT, and bind zmq
sockets for them under CONTEXT. Return a <connection> and a <kernel>: the
connection can be passed to a kernel as its \"connection file\", and the
kernel can be used by the client to talk to it."
(let* ((socket-control (zmq-create-socket context ZMQ_DEALER))
(socket-shell (zmq-create-socket context ZMQ_DEALER))
(socket-stdin (zmq-create-socket context ZMQ_DEALER))
(socket-heartbeat (zmq-create-socket context ZMQ_REQ))
(socket-iosub (zmq-create-socket context ZMQ_SUB))
(++ (lambda (port) (+ port 1)))
(port-control (test-bind socket-control 1024))
(port-shell (test-bind socket-shell (++ port-control)))
(port-stdin (test-bind socket-stdin (++ port-shell)))
(port-heartbeat (test-bind socket-heartbeat (++ port-stdin)))
(port-iosub (test-bind socket-iosub (++ port-heartbeat)))
(connection-file (new-connection-file "tcp" "127.0.0.1" key
#:control port-control
#:shell port-shell
#:heartbeat port-heartbeat
#:stdin port-stdin
#:iosub port-iosub))
(pid (exec-kernel name connection-file))
(addr "tcp://127.0.0.1:")
(addr-p (lambda (port)
(string-append addr
(number->string port))))
(kernel (kernel name pid
(port-control (test-bind socket-control first-port))
(port-shell (test-bind socket-shell (+ 1 port-control)))
(port-stdin (test-bind socket-stdin (+ 1 port-shell)))
(port-heartbeat (test-bind socket-heartbeat (+ 1 port-stdin)))
(port-iosub (test-bind socket-iosub (+ 1 port-heartbeat)))
(connection (connection "tcp" "127.0.0.1"
#:key key
#:control-port port-control
#:shell-port port-shell
#:heartbeat-port port-heartbeat
#:stdin-port port-stdin
#:iopub-port port-iosub))
(port->uri (lambda (port)
(string-append transport "://" ip ":"
(number->string port))))
(kernel (kernel #f #f ;no name and PID yet
#:key key
#:control socket-control
#:shell socket-shell
......@@ -263,13 +254,72 @@ return its PID."
(utf8->string identity)))
;; Sub socket connection.
(zmq-connect socket-control (addr-p port-control))
(zmq-connect socket-shell (addr-p port-shell))
(zmq-connect socket-stdin (addr-p port-stdin))
(zmq-connect socket-heartbeat (addr-p port-heartbeat))
(zmq-connect socket-iosub (addr-p port-iosub))
(zmq-connect socket-control (port->uri port-control))
(zmq-connect socket-shell (port->uri port-shell))
(zmq-connect socket-stdin (port->uri port-stdin))
(zmq-connect socket-heartbeat (port->uri port-heartbeat))
(zmq-connect socket-iosub (port->uri port-iosub))
(values connection kernel)))
(define (spawn-kernel specs connection)
"Spawn the kernel specified by SPECS, a <kernel-specs> record, passing it
CONNECTION serialized to a \"connection file\". Return its PID."
(define connection-file
(string-append "/tmp/guix-kernel/conn/"
(number->string (string-hash
(scm->json-string
(connection->json connection))))
".json"))
(define arguments
(map (match-lambda
("{connection_file}" connection-file)
(x x))
(kernel-specs-arguments specs)))
(mkdir-p (dirname connection-file))
(call-with-output-file connection-file
(lambda (port)
(scm->json (connection->json connection) port
#:pretty #t)))
(let ((pid (primitive-fork)))
(if (zero? pid)
(apply execl (first arguments) arguments)
pid)))
(define (test-bind socket port)
(catch 'zmq-error
(lambda ()
(zmq-bind-socket socket (string-append "tcp://127.0.0.1:"
(number->string port)))
(zmq-unbind-socket socket (string-append "tcp://127.0.0.1:"
(number->string port)))
port)
(lambda stuff
(let ((errno (cadr stuff)))
(cond
((= errno EADDRINUSE) ;The requested address is
;already in use.
(test-bind socket (+ port 1)))
(else #f))))))
(define search-path->list
(let ((not-colon (char-set-complement (char-set #\:))))
(match-lambda
(#f '())
((? string? str)
(string-tokenize str not-colon)))))
kernel))
(define* (run-kernel context specs key #:key identity)
"Spawn the kernel specified by SPECS. Return a <kernel> record usable by
its client."
(let ((connection kernel (allocate-connection context "tcp" "127.0.0.1"
key #:identity identity)))
(set-kernel-pid (set-kernel-name kernel
(kernel-specs-display-name specs))
(spawn-kernel specs connection))))
;;
......
......@@ -28,6 +28,9 @@
(define %context
(zmq-create-context))
(define %python3-specs
(false-if-exception (find-kernel-specs "python3")))
(define %kernel-key "secretkey")
(define %kernel #f)
......@@ -35,10 +38,10 @@
(test-begin "kernels")
;; These tests require the "python3" kernel provided by ipykernel.
(unless (find-kernel-by-name "python3") (test-skip 1))
(unless %python3-specs (test-skip 1))
(test-assert "run-kernel python3"
(let ((kernel (run-kernel %context "python3" %kernel-key)))
(let ((kernel (run-kernel %context %python3-specs %kernel-key)))
(set! %kernel kernel)
(and (kernel? (pk 'kernel kernel))
(begin
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment