Debian Codegraph @

lib/control/plumbing.scm

raw
1 ;;;
2 ;;; control.plumbing - Ports as communication channel
3 ;;;
4 ;;; Copyright (c) 2023-2024 Shiro Kawai <[email protected]>
5 ;;;
6 ;;; Redistribution and use in source and binary forms, with or without
7 ;;; modification, are permitted provided that the following conditions
8 ;;; are met:
9 ;;;
10 ;;; 1. Redistributions of source code must retain the above copyright
11 ;;; notice, this list of conditions and the following disclaimer.
12 ;;;
13 ;;; 2. Redistributions in binary form must reproduce the above copyright
14 ;;; notice, this list of conditions and the following disclaimer in the
15 ;;; documentation and/or other materials provided with the distribution.
16 ;;;
17 ;;; 3. Neither the name of the authors nor the names of its contributors
18 ;;; may be used to endorse or promote products derived from this
19 ;;; software without specific prior written permission.
20 ;;;
21 ;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 ;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 ;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 ;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 ;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 ;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 ;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 ;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 ;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 ;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 ;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 ;;;
33
34 ;; This module provides utilities that uses ports for communication
35 ;; between threads and/or processes.
36
37 ;; TODO: Flow control. Hold inlets if internal buffer reaches a certain limit.
38
39 (define-module control.plumbing
40 (use data.queue)
41 (use gauche.record)
42 (use gauche.threads)
43 (use gauche.uvector)
44 (use gauche.vport)
45 (use scheme.list) ;list-tabulate
46 (use util.match)
47
48 (export make-plumbing plumbing? plumbing-inlet-ports plumbing-outlet-ports
49 plumbing-get-port port-plumbing
50 open-inlet-output-port add-inlet-input-port!
51 open-outlet-input-port add-outlet-output-port!
52
53 open-broadcast-output-port open-tapping-input-port
54 make-pipe make-pump
55 plumbing
56 )
57 )
58 (select-module control.plumbing)
59
60 ;; A plumbing is a device that takes data from one or more ports (inlets)
61 ;; and feed them to one or more ports (outlets). Producers and consumers
62 ;; can be different threads, and synchronization is handled within the
63 ;; device.
64
65 (define-class <plumbing> ()
66 ;; All internals are private
67 ((impl :init-keyword :impl) ;(atom <plumbing-impl>)
68 ;; These two slots are for information only. They are used for external
69 ;; representation. We don't want to lock the object just to display it,
70 ;; so they're outside of atom.
71 (num-inlets :init-value 0)
72 (num-outlets :init-value 0)))
73
74 (define-method write-object ((p <plumbing>) port)
75 (format port "#<plumbing ~a inlet~:p, ~a outlet~:p>"
76 (~ p'num-inlets)
77 (~ p'num-outlets)))
78
79 ;; Private classes - they're not exposed to outside.
80 (define-class <plumbing-impl> ()
81 ((outlets :init-value '())
82 (inlets :init-value '())))
83
84 (define-class <plumbing-outlet> ()
85 ((name :init-keyword :name)
86 (port :init-keyword :port)
87 (put :init-keyword :put) ; (^ impl u8vector) -> void
88 (close :init-keyword :close) ; (^ impl) -> void
89 (thread :init-value #f)))
90
91 (define-class <plumbing-inlet> ()
92 ((name :init-keyword :name)
93 (port :init-keyword :port)
94 (thread :init-keyword :thread)))
95
96 ;; Key for port attributes. Ports created by this module
97 ;; holds the originating <plumbing> object in this attribute.
98 (define *plumbing-key* '#:plumbing)
99
100 ;; The direction of the port (input/output) and whether it is used
101 ;; as inlet or outlet for a plumbing is independent. We have the
102 ;; following combinations:
103 ;;
104 ;; inlet oport - A producer pushes the data to a plumbing.
105 ;;
106 ;; inlet iport - A producer make data available to read. The plumbing
107 ;; uses a dedicated thread to pull the data from it.
108 ;;
109 ;; outlet oport - Data flows through the plubming is pushed to the oport.
110 ;;
111 ;; outlet iport - Data flows through the plumbing is buffered and
112 ;; made available to the consumer.
113 ;;
114 ;; Inlet oports and outlet iports are created by the plumbing. Inlet iports
115 ;; and outlet oports should be provided by the user.
116
117 ;; API
118 (define (make-plumbing)
119 (make <plumbing> :impl (atom (make <plumbing-impl>))))
120
121 ;; API
122 (define (plumbing? obj) (is-a? obj <plumbing>))
123
124 (define-syntax %with-locked-plumbing
125 (syntax-rules ()
126 [(_ plumbing proc)
127 (atomic (~ plumbing'impl) proc)]))
128
129 ;; API
130 (define (plumbing-inlet-ports plumbing)
131 (assume-type plumbing <plumbing>)
132 ($ %with-locked-plumbing plumbing
133 (^[impl] (map (cut ~ <> 'port) (~ impl'inlets)))))
134
135 ;; API
136 (define (plumbing-outlet-ports plumbing)
137 (assume-type plumbing <plumbing>)
138 ($ %with-locked-plumbing plumbing
139 (^[impl] (map (cut ~ <> 'port) (~ impl'outlets)))))
140
141 ;; API
142 (define (plumbing-get-port plumbing name)
143 (assume-type plumbing <plumbing>)
144 (assume-type name <symbol>)
145 ($ %with-locked-plumbing plumbing
146 (^[impl]
147 (define (port-with-name x) (and (eq? (~ x'name) name) (~ x'port)))
148 (or (any port-with-name (~ impl'outlets))
149 (any port-with-name (~ impl'inlets))))))
150
151 ;; API
152 (define (port-plumbing port)
153 (assume-type port <port>)
154 (port-attribute-ref port *plumbing-key* #f))
155
156 ;; impl is assumed to be locked in the following procedures
157 (define (%add-inlet! plumbing impl inlet)
158 (push! (~ impl'inlets) inlet)
159 (set! (~ plumbing'num-inlets) (length (~ impl'inlets))))
160
161 (define (%delete-inlet! plumbing impl inlet)
162 (update! (~ impl'inlets) (cut delete inlet <>))
163 (set! (~ plumbing'num-inlets) (length (~ impl'inlets))))
164
165 (define (%add-outlet! plumbing impl outlet)
166 (push! (~ impl'outlets) outlet)
167 (set! (~ plumbing'num-outlets) (length (~ impl'outlets))))
168
169 (define (%delete-outlet! plumbing impl outlet)
170 (update! (~ impl'outlets) (cut delete outlet <>))
171 (set! (~ plumbing'num-outlets) (length (~ impl'outlets))))
172
173 ;;----------------------------------------------------------
174 ;; inlets
175 ;;
176
177 ;; API
178 (define (open-inlet-output-port plumbing :optional (name #f))
179 (define inlet (make <plumbing-inlet> :thread #f :name name))
180 (define (flusher buffer flag)
181 (let1 data (u8vector-copy buffer)
182 ($ %with-locked-plumbing plumbing
183 (^[impl] (dolist [o (~ impl'outlets)]
184 ((~ o'put) impl data))))
185 (u8vector-length data)))
186 (define (closer)
187 ($ %with-locked-plumbing plumbing
188 (^[impl]
189 (%delete-inlet! plumbing impl inlet)
190 (when (null? (~ impl'inlets))
191 (dolist [o (~ impl'outlets)]
192 ((~ o'close) impl))))))
193 (define port
194 (make <buffered-output-port> :flush flusher :close closer))
195 (port-attribute-set! port *plumbing-key* plumbing)
196 (set! (~ inlet'port) port)
197 (%with-locked-plumbing plumbing (cut %add-inlet! plumbing <> inlet))
198 port)
199
200 ;; API
201 (define (add-inlet-input-port! plumbing iport :optional (name #f))
202 (define inlet (make <plumbing-inlet> :port iport :name name))
203 (define (pump)
204 (let1 data (read-uvector <u8vector> 4096 iport)
205 (if (eof-object? data)
206 ($ %with-locked-plumbing plumbing
207 (^[impl]
208 (%delete-inlet! plumbing impl inlet)
209 (when (null? (~ impl'inlets))
210 (dolist [o (~ impl'outlets)]
211 ((~ o'close) impl)))))
212 (begin
213 ($ %with-locked-plumbing plumbing
214 (^[impl]
215 (dolist [o (~ impl'outlets)]
216 ((~ o'put) impl data))))
217 (pump)))))
218 (define thread (make-thread pump))
219 (set! (~ inlet'thread) thread)
220 (%with-locked-plumbing plumbing (cut %add-inlet! plumbing <> inlet))
221 (thread-start! thread)
222 plumbing)
223
224 ;;----------------------------------------------------------
225 ;; outlets
226 ;;
227
228 ;; API
229 (define (add-outlet-output-port! plumbing oport
230 :optional (name #f)
231 :key (close-on-eof #f)
232 (asynchronous #f))
233 (define outlet
234 (if asynchronous
235 (%make-async-output-outlet! plumbing oport name close-on-eof)
236 (%make-simple-output-outlet! plumbing oport name close-on-eof)))
237 (%with-locked-plumbing plumbing (cut %add-outlet! plumbing <> outlet))
238 plumbing)
239
240 (define (%make-simple-output-outlet! plumbing oport name close-on-eof)
241 (define outlet (make <plumbing-outlet>
242 :name name
243 :port oport
244 :put (^[impl data] (write-uvector data oport))
245 :close (^[impl]
246 (when close-on-eof
247 (close-output-port oport))
248 (%delete-outlet! plumbing impl outlet))))
249 outlet)
250
251 (define (%make-async-output-outlet! plumbing oport name close-on-eof)
252 (define mtq (make-mtqueue))
253 (define (feeder)
254 (let1 data (dequeue/wait! mtq)
255 (if (eof-object? data)
256 (begin (when close-on-eof
257 (close-output-port oport))
258 ($ %with-locked-plumbing plumbing
259 (cut %delete-outlet! plumbing <> outlet)))
260 (begin (write-uvector data oport)
261 (feeder)))))
262 (define outlet
263 (make <plumbing-outlet>
264 :name name
265 :port oport
266 :put (^[impl data] (enqueue/wait! mtq data))
267 :close (^[impl] (enqueue/wait! mtq (eof-object)))))
268 (set! (~ outlet'thread) (thread-start! (make-thread feeder)))
269 outlet)
270
271 ;; API
272 (define (open-outlet-input-port plumbing :optional (name #f))
273 (define mtq (make-mtqueue))
274 (define eof-reached? #f)
275 (define outlet
276 (make <plumbing-outlet>
277 :name name
278 :put (^[impl data] (enqueue/wait! mtq data))
279 :close (^[impl] (enqueue/wait! mtq (eof-object)))))
280 (define (filler buf)
281 (if eof-reached?
282 (eof-object)
283 (let ([len (u8vector-length buf)]
284 [data (dequeue/wait! mtq)]) ;this may block
285 (cond [(eof-object? data)
286 (set! eof-reached? #t)
287 ($ %with-locked-plumbing plumbing
288 (cut %delete-outlet! plumbing <> outlet))
289 data]
290 [(<= (u8vector-length data) len)
291 (u8vector-copy! buf 0 data 0)
292 (u8vector-length data)]
293 [else
294 (u8vector-copy! buf 0 data 0 len)
295 (queue-push/wait! mtq (uvector-alias <u8vector> data len))
296 len]))))
297 (define (ready?)
298 (or eof-reached? (not (queue-empty? mtq))))
299 (define port (make <buffered-input-port> :fill filler :ready ready?))
300 (set! (~ outlet'port) port)
301 (port-attribute-set! port *plumbing-key* plumbing)
302 (set! (port-buffering port) :none)
303 (%with-locked-plumbing plumbing (cut %add-outlet! plumbing <> outlet))
304 port)
305
306 ;;;
307 ;;; Convenience utilities
308 ;;;
309
310 ;; CL's make-broadcast-stream
311 (define (open-broadcast-output-port . oports)
312 (define plumbing (make-plumbing))
313 (dolist [oport oports]
314 (add-outlet-output-port! plumbing oport #f))
315 (open-inlet-output-port plumbing))
316
317 ;; Pipe owns one or more inlet oports and one or more outlet iports.
318 ;; Returns a list of inlet oports and a list of outlet iports.
319 (define (make-pipe :key (num-inlets 1) (num-outlets 1))
320 (assume (and (exact-integer? num-inlets)
321 (>= num-inlets 1))
322 "One or more exact integer is expected, but got" num-inlets)
323 (assume (and (exact-integer? num-outlets)
324 (>= num-outlets 1))
325 "One or more exact integer is expected, but got" num-outlets)
326 (let1 plumbing (make-plumbing)
327 (values (list-tabulate num-inlets (^_ (open-inlet-output-port plumbing)))
328 (list-tabulate num-outlets (^_ (open-outlet-input-port plumbing))))))
329
330 ;; Create a 'pump' - a device that reads from inlet-iport(s) and
331 ;; writes out to outlet-oport(s), run in an independent thread.
332 ;; Returns a plumbing.
333 (define (make-pump inlet-iports outlet-oports)
334 (assume (every input-port? inlet-iports))
335 (assume (every output-port? outlet-oports))
336 (rlet1 plumbing (make-plumbing)
337 (dolist [ip inlet-iports]
338 (add-inlet-input-port! plumbing ip))
339 (dolist [op outlet-oports]
340 (add-outlet-output-port! plumbing op #f))))
341
342 ;; Similar to CL's make-echo-stream, but we support only reading from
343 ;; the craeted port. We may make it bidirectional stream later.
344 (define (open-tapping-input-port inlet-iport outlet-oport :key (close-on-eof #f))
345 (assume (input-port? inlet-iport))
346 (assume (output-port? outlet-oport))
347 (let1 plumbing (make-plumbing)
348 (add-inlet-input-port! plumbing inlet-iport)
349 (add-outlet-output-port! plumbing outlet-oport #f :close-on-eof close-on-eof)
350 (open-outlet-input-port plumbing)))
351
352 ;; A convenience routine to create a plumbing and attach inlets&outlets.
353 ;;
354 ;; Each argument can be one of the following forms:
355 ;; (< #<iport> [name]) ; add iport as inlet
356 ;; (< name) ; create inlet oport with the name
357 ;; (> #<oport> [name] [(option ...)]) ; add oport as outlet
358 ;; (> name) ; create outlet iport with the name
359 ;;
360 ;; NAME is a symbol. OPTION can be :coe, :close-on-eof, or :async.
361 ;;
362 ;; Returns a plumbing. Created ports can be accessed by their name.
363
364 (define (plumbing . args)
365 (rlet1 p (make-plumbing)
366 (define (add-oo! port name opts)
367 ($ apply add-outlet-output-port! p port name
368 $ fold-right (^[opt r]
369 (case opt
370 [(:coe :close-on-eof) `(:close-on-eof #t ,@r)]
371 [(:async :asynchronous) `(:asynchronous #t ,@r)]
372 [else (error "Unrecognized outlet option:" opt)]))
373 '() opts))
374 (dolist [arg args]
375 (match arg
376 [('< (? symbol? name)) (open-inlet-output-port p name)]
377 [('< (? input-port? port)) (add-inlet-input-port! p port)]
378 [('< (? input-port? port) (? symbol? name))
379 (add-inlet-input-port! p port name)]
380 [('< . _) (error "Invalid inlet spec:" arg)]
381 [('> (? symbol? name)) (open-outlet-input-port p name)]
382 [('> (? output-port? port)) (add-oo! port #f '())]
383 [('> (? output-port? port) (? symbol? name)) (add-oo! port name '())]
384 [('> (? output-port? port) (opts ...)) (add-oo! port #f opts)]
385 [('> (? output-port? port) (? symbol? name) (opts ...))
386 (add-oo! port name opts)]
387 [('> . _) (error "Invalid outlet spec:" arg)]
388 [_ (error "Invalid plumbing spec:" arg)]))))