MAJ terminée. Nous sommes passés en version 14.6.2 . Pour consulter les "releases notes" associées c'est ici :

https://about.gitlab.com/releases/2022/01/11/security-release-gitlab-14-6-2-released/
https://about.gitlab.com/releases/2022/01/04/gitlab-14-6-1-released/

kernels.scm 18.2 KB
Newer Older
1
;;; Guix-kernel -- Guix kernel for Jupyter
2
;;; Copyright (C) 2018, 2019 Pierre-Antoine Rouby <pierre-antoine.rouby@inria.fr>
3
;;; Copyright (C) 2018, 2019 Ludovic Courtès <ludovic.courtes@inria.fr>
4
5
6
7
8
9
10
11
12
13
14
15
16
17
;;;
;;; This program is free software: you can redistribute it and/or modify
;;; it under the terms of the GNU General Public License as published by
;;; the Free Software Foundation, either version 3 of the License, or
;;; (at your option) any later version.
;;;
;;; This program is distributed in the hope that it will be useful,
;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with this program.  If not, see <https://www.gnu.org/licenses/>.

18
(define-module (jupyter kernels)
19
  #:use-module (jupyter messages)
20
  #:use-module (jupyter json)
21
  #:use-module (guix build utils)
22
  #:use-module (gcrypt mac)
23
  #:use-module (gcrypt base64)
24
  #:use-module (ice-9 rdelim)
25
  #:use-module (ice-9 match)
26
  #:use-module (ice-9 ftw)
27
28
  #:use-module (simple-zmq)
  #:use-module (json)
29
  #:use-module (rnrs bytevectors)
30
  #:use-module (srfi srfi-1)
31
  #:use-module (srfi srfi-9)
32
  #:use-module (srfi srfi-9 gnu)
33
  #:use-module (srfi srfi-26)
34
  #:use-module (srfi srfi-71)
35
36
  #:export (jupyter-kernel-path
            find-kernel-specs
37
            find-kernel-specs-file
38
            available-kernel-specs-files
39
40
            exec-kernel
            spawn-kernel
41
42
            run-kernel

43
            kernel-specs?
44
            kernel-specs
45
46
47
48
            kernel-specs-arguments
            kernel-specs-display-name
            kernel-specs-language
            json->kernel-specs
49
            kernel-specs->json
50

51
52
53
54
            kernel
            kernel?
            kernel-name
            kernel-pid
55
            kernel-key
56
57
58
59
            kernel-control
            kernel-shell
            kernel-standard-input
            kernel-heartbeat
60
61
            kernel-iosub
            kernel-iopub                          ;alias
62
            kernel-sockets
63
            kernel-socket-kind
64
65
            set-kernel-name
            set-kernel-pid
66

67
68
69
70
71
72
73
74
75
76
77
            connection?
            connection-transport
            connection-ip
            connection-signature-scheme
            connection-key
            connection-control-port
            connection-shell-port
            connection-stdin-port
            connection-heartbeat-port
            connection-iopub-port
            json->connection
78
79
            connection->json
            generate-key
80
            allocate-connection
81

82
            zmq-poll*                             ;XXX: temporary hack
83
84
85
86
87
88
89
            read-message
            send-message
            pub-busy
            pub-idle

            reply-html

90
            close-kernel))
91

92
93
94
95
96
97
98
;;; Commentary:
;;;
;;; This module implements Jupyter "kernels" as defined at
;;; <https://jupyter-client.readthedocs.io/en/latest/kernels.html>.
;;;
;;; Code:

99
100
101
102
;;
;; Kernel execution.
;;

103
;; Type of a Jupyter kernel.
104
(define-immutable-record-type <kernel>
105
  (%kernel name pid key control shell stdin heartbeat iosub)
106
  kernel?
107
108
  (name        kernel-name set-kernel-name)       ;string
  (pid         kernel-pid set-kernel-pid)         ;integer
109
  (key         kernel-key)                        ;bytevector
110
111
112
113
114
115
  (control     kernel-control)                    ;zmq socket
  (shell       kernel-shell)                      ;zmq socket
  (stdin       kernel-standard-input)             ;zmq socket
  (heartbeat   kernel-heartbeat)                  ;zmq socket
  (iosub       kernel-iosub))                     ;zmq socket

116
117
(define-syntax kernel-iopub (identifier-syntax kernel-iosub))

118
119
120
;; Kernel metadata taken from a 'kernel.json' file.
;; <https://jupyter-client.readthedocs.io/en/latest/kernels.html#kernel-specs>
(define-json-mapping <kernel-specs>
121
  kernel-specs make-kernel-specs kernel-specs?
122
  json->kernel-specs <=> kernel-specs->json
123
  (arguments      kernel-specs-arguments
124
                  (json "argv" vector->list list->vector))
125
126
  (display-name   kernel-specs-display-name
                  (json "display_name"))
127
128
  (language       kernel-specs-language))

129
(define* (kernel name pid #:key key control shell
130
                 standard-input heartbeat iosub)
131
132
133
  (%kernel name pid key
           control shell standard-input heartbeat iosub))

134
135
136
137
138
139
140
141
142
143
(define (kernel-sockets kernel)
  "Return all the ZeroMQ sockets associated with KERNEL."
  (map (lambda (socket)
         (socket kernel))
       (list kernel-shell
             kernel-control
             kernel-standard-input
             kernel-heartbeat
             kernel-iopub)))

144
145
146
147
148
149
150
151
152
153
154
155
156
157
(define (kernel-socket-kind kernel socket)
  "Return the procedure (e.g., 'kernel-shell', 'kernel-control') that, when
applied on KERNEL, returns SOCKET.  This allows the caller to determine the
purpose of SOCKET for KERNEL.

Return #f if SOCKET is not one of KERNEL's sockets."
  (find (lambda (kernel-socket)
          (eq? (kernel-socket kernel) socket))
        (list kernel-shell
              kernel-control
              kernel-standard-input
              kernel-heartbeat
              kernel-iopub)))

158
159
(define (close-kernel kernel)
  "Close all the open connections of KERNEL."
160
  (for-each zmq-close-socket (kernel-sockets kernel)))
161

162
163
164
165
166
(define jupyter-kernel-path
  ;; Default search path for Jupyter kernels.
  (make-parameter (or (and=> (getenv "JUPYTER_PATH")
                             (cut string-split <> #\:))
                      '())))
167

168
169
170
171
172
173
174
175
176
177
178
179
(define* (find-kernel-specs kernel
                            #:optional (path (jupyter-kernel-path)))
  "Return the kernel specs for KERNEL, or #f if KERNEL could not be found in
PATH."
  (match kernel
    ("ipython"
     ;; IPython does not provide a 'kernel.json' file like other
     ;; kernels do; instead, we need to execute 'ipython' from $PATH.
     (let ((ipython (search-path (search-path->list (getenv "PATH"))
                                 "ipython")))
       (and ipython
            (kernel-specs
180
181
182
183
             (arguments (list ipython "kernel" "--quiet"
                              "-f" "{connection_file}"))
             (display-name "IPython")
             (language "Python")))))
184
185
186
187
188
189
    (_
     (let ((file (find-kernel-specs-file kernel)))
       (and file (call-with-input-file file json->kernel-specs))))))

(define* (find-kernel-specs-file kernel
                                 #:optional (path (jupyter-kernel-path)))
190
191
192
193
194
195
196
197
  "Return the absolute file name of the 'kernel.json' file for Jupyter kernel
KERNEL, searched for in PATH, a list of directories.  Return #f if KERNEL
could not be found."
  (any (lambda (directory)
         (let ((json (string-append directory "/kernels/"
                                    kernel "/kernel.json")))
           (and (file-exists? json) json)))
       path))
198

199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
(define* (available-kernel-specs-files #:optional
                                       (path (jupyter-kernel-path)))
  "Return the list of available kernel specs files (the 'kernel.json' files)
found in PATH."
  (define (not-dot? file)
    (not (member file '("." ".."))))

  (append-map (lambda (directory)
                (let* ((directory (string-append directory "/kernels"))
                       (entries   (map (cut string-append directory "/" <>)
                                       (or (scandir directory not-dot?) '()))))
                  (filter-map (lambda (entry)
                                (let ((spec (string-append entry
                                                           "/kernel.json")))
                                  (and (file-exists? spec) spec)))
                              entries)))
              path))

217

218
;;
219
;; Kernel "connection files".
220
221
;;

222
223
224
225
;; A "connection" represents a rendezvous between the client and a kernel:
;; <https://jupyter-client.readthedocs.io/en/stable/kernels.html#connection-files>.
;; Usually the client creates it, writes it to a "connection file", which it
;; passes to the kernel.
226
(define-json-mapping <connection> connection make-connection connection?
227
228
229
  json->connection <=> connection->json
  (transport         connection-transport)        ;string
  (ip                connection-ip)               ;string
230
231
  (signature-scheme  connection-signature-scheme
                     (default "hmac-sha256"))     ;string
232
233
  (key               connection-key)              ;string
  (control-port      connection-control-port      ;integer
234
                     (json "control_port"))
235
  (shell-port        connection-shell-port        ;integer
236
                     (json "shell_port"))
237
  (stdin-port        connection-stdin-port        ;integer
238
                     (json "stdin_port"))
239
  (heartbeat-port    connection-heartbeat-port    ;integer
240
                     (json "hb_port"))
241
  (iopub-port        connection-iopub-port        ;integer
242
                     (json "iopub_port")))
243

244
245
246
(define (generate-key)
  "Return a string usable as a shared secret key between a kernel server and
its clients."
247
  (base64-encode (generate-signing-key)))
248

249
250
(define* (allocate-connection context transport ip key
                              #:key
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
                              first-port identity)
  "Allocate ports for TRANSPORT and IP and connect zmq sockets for them under
CONTEXT.  Return a <connection> and a <kernel>: the connection can be passed
to a kernel as its \"connection file\", and the kernel can be used by the
client to talk to it.

When FIRST-PORT is true, use TCP ports from FIRST-PORT on.  When FIRST-PORT
is false, use an OS-allocated port number, which will hopefully not already
be in use (but this is racy)."
  (define (port->uri port)
    (string-append transport "://" ip ":"
                   (number->string port)))

  (define (allocate-port)
    ;; Ask the OS to give us a port number, and return it (jupyter-client
    ;; does the same in 'connect.py'.)  XXX: There's fundamentally a TOCTTOU
    ;; race here because we're on the client side and a server could very
    ;; bind(2) to that port after we've decided to use it, but that's
    ;; inherent to the way the Jupyter protocol works: it's the client that
    ;; chooses ports the kernels should bind to.
    (let ((sock    (socket AF_INET SOCK_STREAM 0))
          (address (make-socket-address AF_INET
                                        (inet-pton AF_INET ip) 0)))
      (setsockopt sock SOL_SOCKET SO_LINGER '(0 . 0))
      (bind sock address)
      (let ((port (sockaddr:port (getsockname sock))))
        (close sock)
        port)))

  (define (try-connect socket port)
    (let ((port (or port (allocate-port))))
      (zmq-connect socket (port->uri port))
      port))

285
286
287
288
289
290
  (let* ((socket-control   (zmq-create-socket context ZMQ_DEALER))
         (socket-shell     (zmq-create-socket context ZMQ_DEALER))
         (socket-stdin     (zmq-create-socket context ZMQ_DEALER))
         (socket-heartbeat (zmq-create-socket context ZMQ_REQ))
         (socket-iosub     (zmq-create-socket context ZMQ_SUB))

291
292
293
294
295
296
297
298
299
         (port-control     (try-connect socket-control   first-port))
         (port-shell       (try-connect socket-shell
                                        (and first-port (+ 1 port-control))))
         (port-stdin       (try-connect socket-stdin
                                        (and first-port (+ 1 port-shell))))
         (port-heartbeat   (try-connect socket-heartbeat
                                        (and first-port (+ 1 port-stdin))))
         (port-iosub       (try-connect socket-iosub
                                        (and first-port (+ 1 port-heartbeat))))
300

301
302
303
304
305
306
307
308
         (connection       (connection
                            (transport transport) (ip ip)
                            (key key)
                            (control-port port-control)
                            (shell-port port-shell)
                            (heartbeat-port port-heartbeat)
                            (stdin-port port-stdin)
                            (iopub-port port-iosub)))
309
310

         (kernel (kernel #f #f                    ;no name and PID yet
311
                         #:key key
312
313
314
315
316
                         #:control socket-control
                         #:shell socket-shell
                         #:standard-input socket-stdin
                         #:heartbeat socket-heartbeat
                         #:iosub socket-iosub)))
317

318
    (zmq-set-socket-option socket-iosub ZMQ_SUBSCRIBE "")
319
320
321
322
323
324
325

    ;; <http://zguide.zeromq.org/page:all#The-Extended-Reply-Envelope> says
    ;; that ZeroMQ "generate[s] a 5 byte identity by default", so usually we
    ;; don't need to provide our own identity.
    (when identity
      (zmq-set-socket-option socket-shell ZMQ_IDENTITY
                             (utf8->string identity)))
326

327
328
    (values connection kernel)))

329
330
331
(define (exec-kernel specs connection)
  "Execute the kernel specified by SPECS, a <kernel-specs> record, passing it
CONNECTION serialized to a \"connection file\"."
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
  (define connection-file
    (string-append "/tmp/guix-kernel/conn/"
                   (number->string (string-hash
                                    (scm->json-string
                                     (connection->json connection))))
                   ".json"))

  (define arguments
    (map (match-lambda
           ("{connection_file}" connection-file)
           (x x))
         (kernel-specs-arguments specs)))

  (mkdir-p (dirname connection-file))
  (call-with-output-file connection-file
    (lambda (port)
      (scm->json (connection->json connection) port
                 #:pretty #t)))

351
352
353
354
355
356
  (apply execlp (first arguments) arguments))

(define (spawn-kernel specs connection)
  "Spawn the kernel designated by SPECS, a <kernel-specs> record, as a new
process, and return its PID.  Pass it CONNECTION serialized to a \"connection
file\"."
357
358
  (let ((pid (primitive-fork)))
    (if (zero? pid)
359
        (exec-kernel specs connection)
360
361
362
363
364
365
366
367
        pid)))

(define search-path->list
  (let ((not-colon (char-set-complement (char-set #\:))))
    (match-lambda
      (#f '())
      ((? string? str)
       (string-tokenize str not-colon)))))
368

369
370
371
372
373
374
375
376
(define* (run-kernel context specs key #:key identity)
  "Spawn the kernel specified by SPECS.  Return a <kernel> record usable by
its client."
  (let ((connection kernel (allocate-connection context "tcp" "127.0.0.1"
                                                key #:identity identity)))
    (set-kernel-pid (set-kernel-name kernel
                                     (kernel-specs-display-name specs))
                    (spawn-kernel specs connection))))
377
378
379
380
381
382


;;
;; Communicating with a kernel.
;;

383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
(define (EINTR-safe proc)
  "Return a variant of PROC that catches EINTR 'zmq-error' exceptions and
retries a call to PROC."
  (define (safe . args)
    (catch 'zmq-error
      (lambda ()
        (apply proc args))
      (lambda (key errno . rest)
        (if (= errno EINTR)
            (apply safe args)
            (apply throw key errno rest)))))

  safe)

(define zmq-poll*
  ;; XXX: As of guile-simple-zmq commit 878f6dc (Nov. 2018), 'zmq-poll'
  ;; doesn't catch EINTR, so do it here.
  (EINTR-safe zmq-poll))

402
(define* (read-message kernel #:optional (timeout -1))
403
404
405
406
407
408
409
  "Read one message from one of the sockets of KERNEL and return it.  If the
message is a \"regular\" JSON message, return it as an alist; if it's a
heartbeat message, return it as a bytevector.

If TIMEOUT is -1, wait indefinitely; otherwise wait that number of
milliseconds.  If TIMEOUT expires before a message has been received, return
#f."
410
411
412
  (define shell (kernel-shell kernel))
  (define iopub (kernel-iopub kernel))
  (define items
413
414
415
416
    (zmq-poll* (map (lambda (socket)
                      (poll-item socket (logior ZMQ_POLLIN ZMQ_POLLERR)))
                    (kernel-sockets kernel))
               timeout))
417
418
419
420
421
422

  (let loop ((items items))
    (match items
      (()
       #f)
      ((item rest ...)
423
424
       (let* ((socket (poll-item-socket item))
              (parts  (zmq-get-msg-parts-bytevector socket)))
425
426
427
         ;; Heartbeat messages are raw "bytestrings" that should be echoed
         ;; back right away.
         (if (eq? socket (kernel-heartbeat kernel))
428
429
             parts
             (parts->message parts)))))))
430
431
432
433
434
435
436
437
438
439
440
441
442
443

(define* (send-message kernel message
                       #:key
                       (kernel-socket kernel-shell)
                       (recipient (and=> (message-parent-header message)
                                         header-sender)))
  "Send message over the shell socket of KERNEL to RECIPIENT, a ZeroMQ
identity (bytevector)."
  (zmq-send-msg-parts-bytevector (kernel-socket kernel)
                                 (message-parts message
                                                (kernel-key kernel)
                                                #:recipient recipient)))

(define (pub kernel message status)
444
  (let ((content (kernel-status->json status)))
445
446
447
448
449
    (send-message kernel (reply message "status"
                                  (scm->json-string content))
                  #:kernel-socket kernel-iopub)))

(define (pub-busy kernel message)
450
451
  (pub kernel message
       (kernel-status (execution-state 'busy))))
452
453

(define (pub-idle kernel message)
454
455
  (pub kernel message
       (kernel-status (execution-state 'idle))))
456
457
458
459
460
461
462

;;
;; Reply.
;;

(define (reply-html kernel message html count)
  "Reply to MESSAGE with HTML."
Ludovic Courtès's avatar
Ludovic Courtès committed
463
464
465
  (let ((code (assoc-ref (json-string->scm (message-content message))
                         "code"))
	(empty-object '())
466
467
468
469
470
471
	(counter      (+ count 1))                ;execution counter

        (send (lambda (socket type content)
                (send-message kernel
                              (reply message type
                                     (scm->json-string content))
472
                              #:kernel-socket socket
473
474
                              #:recipient (and (eq? socket kernel-shell)
                                               (message-sender message))))))
475
476
477
478
479
480
481
482
483
484
485
486
487
488

    (send kernel-iopub
          "execute_input" `(("code"            . ,code)
                            ("execution_count" . ,counter)))
    (send kernel-iopub
          "execute_result" `(("data"     . (("text/html" . ,html)))
                             ("metadata" . ,empty-object)
                             ("execution_count" . ,counter)))
    (send kernel-shell
          "execute_reply" `(("status"           . "ok")
                            ("execution_count"  . ,counter)
                            ("payload"          . [])
                            ("user_expressions" . ,empty-object)))
    counter))