Debian Codegraph @

lib/rfc/http.scm

raw
1 ;;;
2 ;;; http.scm - HTTP 1.1
3 ;;;
4 ;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 ;;;
6 ;;; Redistribution and use in source and binary forms, with or without
7 ;;; modification, are permitted provided that the following conditions
8 ;;; are met:
9 ;;;
10 ;;; 1. Redistributions of source code must retain the above copyright
11 ;;; notice, this list of conditions and the following disclaimer.
12 ;;;
13 ;;; 2. Redistributions in binary form must reproduce the above copyright
14 ;;; notice, this list of conditions and the following disclaimer in the
15 ;;; documentation and/or other materials provided with the distribution.
16 ;;;
17 ;;; 3. Neither the name of the authors nor the names of its contributors
18 ;;; may be used to endorse or promote products derived from this
19 ;;; software without specific prior written permission.
20 ;;;
21 ;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 ;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 ;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 ;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 ;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 ;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 ;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 ;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 ;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 ;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 ;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 ;;;
33
34 ;; HTTP handling routines.
35
36 ;; RFC2616 Hypertext Transfer Protocol -- HTTP/1.1
37 ;; http://www.ietf.org/rfc/rfc2616.txt
38 ;; RFC2617 HTTP Authentication: Basic and Digest Access Authentication
39 ;; http://www.ietf.org/rfc/rfc2617.txt
40 ;; RFC2388 Returning Values from Forms: multipart/form-data
41 ;; http://www.ietf.org/rfc/rfc2388.txt
42
43 ;; HTTP 1.1 has lots of features. This module doesn't yet cover all of them.
44 ;; The features required for typical client usage are implemented first.
45
46 (define-module rfc.http
47 (use srfi.13)
48 (use rfc.822)
49 (use rfc.uri)
50 (use rfc.base64)
51 (use rfc.tls)
52 (use gauche.net)
53 (use gauche.charconv)
54 (use gauche.sequence)
55 (use gauche.uvector)
56 (use gauche.connection)
57 (use util.match)
58 (use text.tree)
59 (export <http-error>
60 http-user-agent make-http-connection reset-http-connection
61 http-compose-query http-compose-form-data
62 http-status-code->description
63
64 http-proxy http-request
65 http-null-receiver http-string-receiver http-oport-receiver
66 http-file-receiver http-cond-receiver
67 http-null-sender http-string-sender http-blob-sender
68 http-file-sender http-multipart-sender
69
70 http-get http-head http-post http-put http-delete
71 http-default-auth-handler
72 http-default-redirect-handler
73
74 http-secure-connection-available?
75 )
76 )
77 (select-module rfc.http)
78
79 ;;
80 (autoload rfc.mime
81 mime-compose-message
82 mime-compose-message-string
83 mime-compose-parameters
84 mime-parse-content-type)
85 (autoload rfc.http.tunnel probe-stunnel make-stunnel-connection)
86 (autoload file.util file-size find-file-in-paths null-device)
87
88 ;==============================================================
89 ;; Conditions
90 ;;
91
92 (define-condition-type <http-error> <error> #f)
93
94 ;;==============================================================
95 ;; Global parameters
96 ;;
97
98 ;; default string to be used for user-agent.
99 (define http-user-agent
100 (make-parameter #"gauche.http/~(gauche-version)"))
101
102 ;; global proxy settings. can be overridden by :proxy keyword
103 ;; argument.
104 (define http-proxy (make-parameter #f))
105
106 ;; The default redirect handler
107 (define http-default-redirect-handler
108 (make-parameter
109 (^[method code headers body]
110 (and-let* ([loc (rfc822-header-ref headers "location")])
111 (case (x->integer code)
112 [(300 301 305 307)
113 (case method [(GET HEAD) `(,method . ,loc)] [else #f])]
114 [(302 303) ;found / see other
115 ;; See rfc2616 notes - the agent isn't supposed to automatically
116 ;; redirect 302 response of POST, but redirect with GET for it is
117 ;; the de-facto behavior.
118 (case method
119 [(GET HEAD) `(,method . ,loc)]
120 [else `(GET . ,loc)])]
121 [else #f])))))
122
123 ;;==============================================================
124 ;; Higher-level API
125 ;;
126
127 ;; Higher-level API is for conventional call-return API.
128 ;;
129 ;; The METHOD argument specifies the http request method by a symbol.
130 ;; GET, HEAD, POST, PUT and DELETE is currently supported.
131 ;;
132 ;; The SERVER argument maybe a string naming the server (and
133 ;; optionally a port number by the format of "server:port"), or
134 ;; an <http-connection> object. Using a server name is suitable
135 ;; for easy one-shot http access; the connection and related states
136 ;; are discarded once the procedure returns.
137 ;; On the other hand, a connection object can keep the states such
138 ;; as persistent connection and authentication tokens, suitable for
139 ;; a series of communications to a server.
140 ;;
141 ;; The REQUEST-URI argument can be a string as specified in RFC2616,
142 ;; or a list in the form of (<path> (<name> <value>) ...). In the
143 ;; latter form, (<name> <value>) assoc list is converted into a
144 ;; url query form as defined in HTML4 (application/x-www-form-urlencoded)
145 ;; and appended to <path>.
146 ;;
147 ;; The keyword arguments are handled by various low-level routines,
148 ;; and here's the summary:
149 ;;
150 ;; request-body - gives the body.
151 ;; receiver - A procedure that handles the response.
152 ;; It takes four arguments: response-code, response headers,
153 ;; total response size if known (#f otherwise), and
154 ;; a retriever thunk. The retriever thunk is, when called,
155 ;; returns two values: an input port and an integer size.
156 ;; The receiver must loop (1) to read as many bytes as
157 ;; specified in the size from the input port, and (2)
158 ;; call the retriver again, while it returns positive integer.
159 ;; The retriever procedure returns zero when body is exhausted.
160 ;; In that case the value(s) the receiver returns will be
161 ;; the return value of http-request.
162 ;; The retriever thunk can return #f as size, if we don't know the
163 ;; chunk of response size. In that case, the receiver may read
164 ;; up to EOF, or as much data as it wants, then call the
165 ;; retriever. The second time the retriever returns size=0,
166 ;; so you can do cleanup work.
167 ;; The retriever procedure returns -1 if an error occurs
168 ;; during comminucation. In that case, the receiver can do
169 ;; whatever cleanup work. After the receiver returns, an
170 ;; appropriate error is thrown from http-request.
171 ;;
172 ;; sender - A procedure that handles sending the request body.
173 ;; It takes three arguments: tentative list of headers,
174 ;; the value given to :request-encoding argument,
175 ;; and a procedure HEADER-SINK.
176 ;;
177 ;; HEADER-SINK takes one argument, a list of http headers
178 ;; in the form of (("field-name" "field-value") ...).
179 ;; In general, besides the given tentative headers, the sender
180 ;; needs to add either Content-Length header if it sends entire
181 ;; contents at once, or ("transfer-encoding" "chunked") if
182 ;; it uses chunked transfer. If neither headers exist in the
183 ;; given header list, an error is raised and request is aborted.
184 ;; Sender can add other headers, e.g. content-type or mime-type.
185 ;; Sender can also remove or modify given tentative headers.
186 ;;
187 ;; HEADER-SINK returns a procedure BODY-SINK on success.
188 ;; BODY-SINK takes an integer SIZE argument and returns an
189 ;; output PORT. What the sender needs to do is to call
190 ;; BODY-SINK with the size of data chunk, then to write out
191 ;; the data into the returned PORT, and repeat it until
192 ;; all the data is sent. To indicate the end of data,
193 ;; the sender needs to call BODY-SINK with argument 0.
194 ;;
195 ;; host - the host name passed to the 'host' header field.
196 ;; secure - if true, using secure connection (via gauche.tls).
197 ;; auth-user, auth-password, auth-handler - authentication parameters.
198 ;; request-encoding - when http-* is to construct request-uri and/or
199 ;; request body, this argument specifies the character encoding
200 ;; to be used as the external encoding.
201 ;; redirect-handler - Called if the server responds with 3xx status.
202 ;; The argument is the request method, a status code, list of
203 ;; headers and response body (can be #f). It may return a
204 ;; (METHOD . URL) or #f. For the first case,
205 ;; http-request re-attempts to fetch the URL with the given method.
206 ;; (unless we're not looping). If it returns #f, the original
207 ;; code, headers and body are returned from http-request.
208 ;; If given #t (default), the procedure bound to the parameter
209 ;; http-default-redirect-handler is called.
210 ;; no-redirect - If true, we ignore the value of redirect-handler and
211 ;; returns without attempting retrying.
212 ;; This is provided for the backward compatibility; newer code
213 ;; should use :redirect-handler #f
214 ;;
215 ;; Other unrecognized options are passed as request headers.
216
217 (define (http-request method server request-uri
218 :key (host #f)
219 (redirect-handler #t)
220 (no-redirect #f)
221 auth-handler
222 auth-user
223 auth-password
224 (proxy (http-proxy))
225 (user-agent (http-user-agent))
226 (secure #f)
227 (receiver (http-string-receiver))
228 (sender #f)
229 ((:request-encoding enc) (gauche-character-encoding))
230 :allow-other-keys opts)
231
232 (define extra-headers
233 ($ concatenate $ reverse
234 $ rlet1 z '()
235 (do-plist [(k v) opts]
236 (when v (push! z (list k v))))))
237
238 (define conn (ensure-connection server auth-handler auth-user auth-password
239 proxy secure extra-headers))
240 (define redirector (if no-redirect
241 #f
242 (case redirect-handler
243 [(#t) (http-default-redirect-handler)]
244 [(#f) #f]
245 [else => identity])))
246 (define no-body-replies '("204" "304"))
247
248 (define (get-body iport method code headers receiver)
249 (and (not (eq? method 'HEAD))
250 (not (member code no-body-replies))
251 (receive-body iport code headers receiver)))
252
253 ;; final touch of request headers
254 (define (req-headers host)
255 (cond-list [(~ conn'persistent) @ (if (~ conn'proxy)
256 '(:proxy-connection keep-alive)
257 '(:connection keep-alive))]
258 [#t @ `(:host ,host :user-agent ,user-agent
259 ,@(http-auth-headers conn) ,@extra-headers)]))
260
261 ;; If we decide to give up redirection, we read from already-retrieved
262 ;; body of 3xx reply. This modifies reply headers if necessary.
263 (define (redirect-headers body rep-headers)
264 (if body
265 `(("content-length" ,(string-size body))
266 ,@(remove (^p (member (car p) '("content-length"
267 "content-transfer-encoding")))
268 rep-headers))
269 rep-headers))
270
271 ;; returns either one of:
272 ;; (reply <code> <headers> <body>)
273 ;; (redirect-to <method> <location>)
274 (define (request-response in out method uri host sender)
275 (send-request out method uri sender (req-headers host) enc)
276 (receive (code rep-headers) (receive-header in)
277 (if-let1 consider-redirect (and (string-prefix? "3" code) redirector)
278 ;; we retrieve body as string, not using caller-provided receiver
279 (let* ([body (get-body in method code rep-headers
280 (http-string-receiver))]
281 [verdict (consider-redirect method code rep-headers body)])
282 ;; consider-redirect returns either #f (don't redirect) or
283 ;; (METHOD . LOCATION).
284 (if verdict
285 `(redirect-to ,(car verdict) ,(cdr verdict))
286 (let1 hdrs (redirect-headers body rep-headers) ;giving up
287 `(reply ,code ,hdrs
288 ,(and body
289 (receive-body (open-input-string body) code
290 hdrs receiver))))))
291 ;; no redirection
292 `(reply ,code ,rep-headers
293 ,(get-body in method code rep-headers receiver)))))
294
295 ;; main loop
296 (let loop ([history '()]
297 [host host]
298 [method method]
299 [request-uri (ensure-request-uri request-uri enc)])
300 (receive (host uri)
301 (consider-proxy conn (or host (~ conn'server)) request-uri)
302 (let1 result
303 (with-connection
304 conn
305 (^[i o] (request-response i o method uri host sender)))
306 (match result
307 [('reply code rep-headers body) (values code rep-headers body)]
308 [('redirect-to method location)
309 (receive (uri proto new-server path*)
310 (canonical-uri conn location (ref conn'server))
311 (when (or (member uri history)
312 (> (length history) 20))
313 (errorf <http-error> "redirection is looping via ~a" uri))
314 (loop (cons uri history)
315 (~ (redirect-connection! conn proto new-server)'server)
316 method path*))])))))
317
318 ;;
319 ;; Pre-defined receivers
320 ;;
321 (define (http-string-receiver)
322 (^[code hdrs total retr]
323 ;; TODO: check headers for encoding
324 (let loop ([sink (open-output-string)])
325 (receive (remote size) (retr)
326 (cond [(eqv? size 0) (get-output-string sink)]
327 [(or (not size) (> size 0))
328 (copy-port remote sink :size size) (loop sink)])))))
329
330 (define (http-null-receiver)
331 (^[code hdrs total retr]
332 (let loop ([sink (open-output-file (null-device))])
333 (receive (remote size) (retr)
334 (cond [(and size (<= size 0)) (close-output-port sink)]
335 [else (copy-port remote sink :size size) (loop sink)])))))
336
337 (define (http-oport-receiver sink flusher)
338 (^[code hdrs total retr]
339 (let loop ()
340 (receive (remote size) (retr)
341 (cond [(and size (<= size 0)) (flusher sink hdrs)]
342 [else (copy-port remote sink :size size) (loop)])))))
343
344 (define (http-file-receiver filename :key (temporary #f))
345 (^[code hdrs total retr]
346 (receive (port tmpname) (sys-mkstemp filename)
347 (let loop ()
348 (receive (remote size) (retr)
349 (cond [(or (not size) (> size 0))
350 (copy-port remote port :size size) (loop)]
351 [(= size 0)
352 (close-output-port port)
353 (if temporary
354 tmpname
355 (begin (sys-rename tmpname filename) filename))]
356 [else (close-output-port port) (sys-unlink tmpname)]))))))
357
358 (define-syntax http-cond-receiver
359 (syntax-rules (else =>)
360 [(_) (http-null-receiver)]
361 [(_ [else . exprs]) (begin . exprs)]
362 [(_ [cc => proc] . rest)
363 (^[code hdrs total retr]
364 ((if (match-status-code? cc code)
365 proc
366 (http-cond-receiver . rest))
367 code hdrs total retr))]
368 [(_ [cc . exprs] . rest)
369 (^[code hdrs total retr]
370 ((if (match-status-code? cc code '(cc . exprs))
371 (begin . exprs)
372 (http-cond-receiver . rest))
373 code hdrs total retr))]
374 [(_ other . rest)
375 (syntax-error "invalid clause in http-cond-receiver" other)]))
376
377 (define (match-status-code? pattern code clause)
378 (cond [(string? pattern) (equal? pattern code)]
379 [(regexp? pattern) (rxmatch pattern code)]
380 [else (error "Invalid pattern in a clause of http-cond-receiver:"
381 clause)]))
382
383 ;;
384 ;; Senders
385 ;;
386
387 (define (http-null-sender)
388 (^[hdrs encoding header-sink]
389 (let1 body-sink (header-sink `(("content-length" "0") ,@hdrs))
390 (body-sink 0))))
391
392 (define (http-string-sender string) ;honors encoding
393 (^[hdrs encoding header-sink]
394 (let* ([body (if (ces-equivalent? encoding (gauche-character-encoding))
395 string
396 (ces-convert string (gauche-character-encoding) encoding))]
397 [size (string-size body)]
398 [body-sink (header-sink `(("content-length" ,(x->string size))
399 ,@hdrs))]
400 [oport (body-sink size)])
401 (display body oport)
402 (body-sink 0))))
403
404 (define (http-blob-sender blob) ;blob may be a string or uvector
405 (^[hdrs encoding header-sink]
406 (let* ([size (if (string? blob) (string-size blob) (uvector-size blob))]
407 [body-sink (header-sink `(("content-length" ,(x->string size))
408 ,@hdrs))]
409 [port (body-sink size)])
410 (if (string? blob)
411 (display blob port)
412 (write-block blob port))
413 (body-sink 0))))
414
415 ;; Send contents directly from the file. Encoding is ignored.
416 ;; TODO: The file size may be changed while sending out. If it gets
417 ;; bigger, we can just ignore the rest, but what if it gets shorter?
418 (define (http-file-sender filename)
419 (^[hdrs encoding header-sink]
420 (let* ([size (file-size filename)]
421 [body-sink (header-sink `(("content-length" ,(x->string size))
422 ,@hdrs))]
423 [port (body-sink size)])
424 (call-with-input-file filename (cut copy-port <> port :size size))
425 (body-sink 0))))
426
427 ;; See http-compose-form-data definition for params spec.
428 ;; TODO: support chunked sending, instead of building entire body at once.
429 (define (http-multipart-sender params)
430 (^[hdrs encoding header-sink]
431 (receive (body boundary) (http-compose-form-data params #f encoding)
432 (let* ([size (string-size body)]
433 [hdrs `(("content-length" ,(x->string size))
434 ("mime-version" "1.0")
435 ("content-type" ,#"multipart/form-data; boundary=\"~|boundary|\"")
436 ,@(alist-delete "content-type" hdrs equal?))]
437 [body-sink (header-sink hdrs)]
438 [port (body-sink size)])
439 (display body port)
440 (body-sink 0)))))
441
442 ;;
443 ;; Shortcuts for specific requests.
444 ;;
445 (define (http-get server request-uri . options)
446 (apply %http-request-adaptor 'GET server request-uri #f options))
447
448 (define (http-head server request-uri . options)
449 (apply %http-request-adaptor 'HEAD server request-uri #f options))
450
451 (define (http-post server request-uri body . options)
452 (apply %http-request-adaptor 'POST server request-uri body options))
453
454 (define (http-put server request-uri body . options)
455 (apply %http-request-adaptor 'PUT server request-uri body options))
456
457 (define (http-delete server request-uri . options)
458 (apply %http-request-adaptor 'DELETE server request-uri #f options))
459
460 ;; Adaptor to the new API. Converts :sink and :flusher arguments,
461 ;; which are superseded by :receiver arguments.
462 (define (%http-request-adaptor method server request-uri body
463 :key (receiver #f) (sink #f) (flusher #f)
464 :allow-other-keys opts)
465 (define recvr
466 (cond [(and sink flusher)
467 (http-oport-receiver sink flusher)]
468 [(or sink flusher)
469 (errorf "You need to provide :sink and :flusher together to http-~a"
470 (string-downcase (symbol->string method)))]
471 [receiver]
472 ;; fallback
473 [else (http-oport-receiver (open-output-string)
474 (^[s h] (get-output-string s)))]))
475 (apply http-request method server request-uri
476 :sender (cond [(not body) (http-null-sender)]
477 [(list? body) (http-multipart-sender body)]
478 [else (http-blob-sender body)])
479 :receiver recvr opts))
480
481 ;;==============================================================
482 ;; HTTP connection context
483 ;;
484
485 ;; <http-connection> object can be used to have conversations with
486 ;; (usually) a specific server. A typical usage is to emulate a
487 ;; browser to perform a certain transaction spanning to several
488 ;; webpages. (For the simple "one-shot" request-response access,
489 ;; the rfc.http APIs create a temporary connection under the hood.)
490
491 (define-class <http-connection> ()
492 ;; All slots are private.
493 ((server :init-keyword :server) ; server[:port]
494 (socket :init-value #f) ; A <connection> object; either <socket>
495 ; or wrapped by secure agent.
496 ; If it is shutdown by the server,
497 ; the APIs attempt to reconnect.
498 (persistent :init-keyword :persistent) ; true for persistent connection.
499 (auth-handler :init-keyword :auth-handler :init-value #f)
500 (auth-user :init-keyword :auth-user :init-value #f)
501 (auth-password :init-keyword :auth-password :init-value #f)
502 (proxy :init-keyword :proxy)
503 (extra-headers :init-keyword :extra-headers)
504 (secure :init-keyword :secure) ; either #f, tls or stunnel
505 ))
506
507 (define (make-http-connection server :key
508 (persistent #t)
509 (auth-handler #f)
510 (auth-user #f)
511 (auth-password #f)
512 (proxy #f)
513 (extra-headers '()))
514 (make <http-connection>
515 :persistent persistent
516 :server server
517 :auth-handler (or auth-handler (http-default-auth-handler))
518 :auth-user auth-user
519 :auth-password auth-password
520 :proxy proxy
521 :extra-headers extra-headers))
522
523 ;; This modifies CONN.
524 (define (redirect-connection! conn proto new-server)
525 (let1 orig-server (~ conn'server)
526 ;; Unless we're connecting the same server with same secure level,
527 ;; we reset the connection.
528 (unless (and (string=? orig-server new-server)
529 (or (and (equal? proto "https") (~ conn'secure))
530 (and (not (equal? proto "https")) (not (~ conn'secure)))))
531 (and-let1 c (~ conn'socket)
532 (connection-shutdown (~ conn'socket) 'both)
533 (set! (~ conn'socket) #f))
534 (set! (~ conn'server) new-server)
535 ;; NB: If the server redirects http connection to https, we don't have
536 ;; a chance for the caller to select alternative TLS flavor---for the
537 ;; time being we blindly assume tls.
538 (set! (~ conn'secure) (and (equal? proto "https") 'tls))))
539 conn)
540
541 ;; API
542 (define (reset-http-connection conn)
543 (when (~ conn'socket)
544 (guard (e [(<system-error> e) #f])
545 (connection-shutdown (~ conn'socket) 'both)
546 (connection-close (~ conn'socket)))
547 (set! (~ conn'socket) #f)))
548
549
550 ;; API
551 (define (http-secure-connection-available? :optional (type #t))
552 (case type
553 [(#t tls)
554 (cond-expand
555 [gauche.net.tls #t]
556 [else #f])]
557 [(stunnel)
558 (boolean (probe-stunnel))]
559 [else
560 (error "Unknown secure connection type (must be tls, stunnel or #t):"
561 type)]))
562
563 ;;==============================================================
564 ;; query and request body composition
565 ;;
566
567 ;; Query string composition.
568 ;; NB: Query string syntax (aka application/x-www-form-urlencoded) is
569 ;; not defined in RFC2616. In fact, it is not clearly defined at all
570 ;; in RFC. The most definitive source might be the HTML4 specification,
571 ;; section 17.13.4 "Form content types",
572 ;; <http://www.w3.org/TR/html4/interact/forms.html#h-17.13.4.1>.
573 ;; In reality it is used very frequently with http, so we put it here.
574 (define (http-compose-query path params
575 :optional (encoding (gauche-character-encoding)))
576 (cond [(not path) (uri-compose-query params)]
577 [(null? params) path]
578 [else #"~|path|?~(uri-compose-query params)"]))
579
580 ;; multipart/form-data composition [RFC2388]
581 ;; <params> : (<param> ...)
582 ;; <param> : (<name> <value>) ; same as http-compose-query
583 ;; | (<name> <key> <value> <key2> <value2> ...)
584 ;; <key> : :value | :file | :content-type | :content-transfer-encoding
585 ;; | other keywords (used as a header name)
586 ;; composed message is put to the current output port.
587 ;; returns the boundary string.
588 (define (http-compose-form-data params port
589 :optional (encoding (gauche-character-encoding)))
590 (define (translate-param param)
591 (match param
592 [(name value) (translate-param `(,name :value ,value))]
593 [(name . kvs)
594 (unless (even? (length kvs))
595 (error "Invalid parameter format to create multipart/form-data:" param))
596 (let-keywords kvs ([value ""]
597 [file #f]
598 [content-type #f]
599 [content-transfer-encoding #f] . other-keys)
600 `(,(canonical-content-type (mime-parse-content-type content-type)
601 value file)
602 (("content-transfer-encoding" ,(or content-transfer-encoding "binary"))
603 ("content-disposition" ,(make-content-disposition name file))
604 ,@(map (cut map x->string <>) (slices other-keys 2)))
605 ,(if file `(file ,file) (x->string value))))]))
606 (define (canonical-content-type ct value file)
607 (match ct
608 [#f (if (or file (string-incomplete? value))
609 '("application" "octet-stream")
610 `("text" "plain" ("charset" . ,(x->string encoding))))]
611 [(type subtype . options)
612 (if (assoc "charset" options)
613 ct
614 `(,type ,subtype ("charset" . ,(x->string encoding)) ,@options))]))
615 (define (make-content-disposition name file)
616 (with-output-to-string
617 (^[]
618 (display "form-data")
619 (mime-compose-parameters
620 `(("name" . ,name)
621 ,@(cond-list [file `("filename" . ,file)]))))))
622 (if (not port)
623 (mime-compose-message-string (map translate-param params))
624 (mime-compose-message (map translate-param params) port)))
625
626 ;;==============================================================
627 ;; status codes
628 ;;
629
630 (define *status-code-map*
631 (hash-table 'eqv?
632 '(100 . "Continue")
633 '(101 . "Switching Protocols")
634 '(200 . "OK")
635 '(201 . "Created")
636 '(202 . "Accepted")
637 '(203 . "Non-Authoritative Information")
638 '(204 . "No Content")
639 '(205 . "Reset Content")
640 '(206 . "Partial Content")
641 '(300 . "Multiple Choices")
642 '(301 . "Moved Permanently")
643 '(302 . "Found")
644 '(303 . "See Other")
645 '(304 . "Not Modified")
646 '(305 . "Use Proxy")
647 '(306 . "(Unused)")
648 '(307 . "Temporary Redirect")
649 '(400 . "Bad Request")
650 '(401 . "Unauthorized")
651 '(402 . "Payment Required")
652 '(403 . "Forbidden")
653 '(404 . "Not Found")
654 '(405 . "Method Not Allowed")
655 '(406 . "Not Acceptable")
656 '(407 . "Proxy Authentication Required")
657 '(408 . "Request Timeout")
658 '(409 . "Conflict")
659 '(410 . "Gone")
660 '(411 . "Length Required")
661 '(412 . "Precondition Failed")
662 '(413 . "Request Entity Too Large")
663 '(414 . "Request-URI Too Long")
664 '(415 . "Unsupported Media Type")
665 '(416 . "Requested Range Not Satisfiable")
666 '(417 . "Expectation Failed")
667 '(500 . "Internal Server Error")
668 '(501 . "Not Implemented")
669 '(502 . "Bad Gateway")
670 '(503 . "Service Unavailable")
671 '(504 . "Gateway Timeout")
672 '(505 . "HTTP Version Not Supported")
673 ))
674
675 ;; API
676 ;; code can be an integer or a string, e.g. "200"
677 ;; returns #f for unknown code
678 (define (http-status-code->description code)
679 (hash-table-get *status-code-map* (x->integer code) #f))
680
681 ;;==============================================================
682 ;; internal utilities
683 ;;
684
685 (define (ensure-request-uri request-uri enc)
686 (match request-uri
687 [(? string?) request-uri]
688 [(path n&v ...) (http-compose-query path n&v enc)]
689 [_ (error "Invalid request-uri form for http request API:" request-uri)]))
690
691 ;; Always returns a connection object.
692 (define (ensure-connection server auth-handler auth-user auth-password
693 proxy secure extra-headers)
694 (rlet1 conn (cond
695 [(is-a? server <http-connection>) server]
696 [(string? server) (make-http-connection server :persistent #f)]
697 [else (error "bad type of argument for server: must be an <http-connection> object or a string of the server's name, but got:" server)])
698 ;; TODO: Might need to reset connections if parameters are changed
699 (let-syntax ([check-override
700 (syntax-rules ()
701 [(_ id)
702 (unless (undefined? id) (set! (ref conn'id) id))])])
703 (check-override auth-handler)
704 (check-override auth-user)
705 (check-override auth-password)
706 (check-override proxy)
707 (check-override extra-headers)
708 (set! (~ conn'secure)
709 (and (not (undefined? secure))
710 (case secure
711 [(#f) #f]
712 [(#t tls) 'tls]
713 [(stunnel) 'stunnel]
714 [else (error "`secure' argument for http connection must be \
715 either boolean, 'tls or 'stunnel, but got:"
716 secure)]))))))
717
718 (define (start-connection conn)
719 ;; If address is given ipv6 format such as "[::1]:port", we have to
720 ;; use "::1" part as the hostname, excluding [].
721 (define (parse-address addr)
722 (rxmatch-case addr
723 [#/^unix:(\/.*)$/ (_ path) (values #f #f path)] ;unix domain
724 [#/^\[([a-fA-F\d:]+)\](?::(\d+))?$/ (_ host port) (values host port #f)]
725 [#/^([^:]+)(?::(\d+))?$/ (_ host port) (values host port #f)]
726 [else (error "Unrecognized http server address:" addr)]))
727 (define (connect-socket)
728 (receive (host port path)
729 (parse-address (or (~ conn'proxy) (~ conn'server)))
730 (cond [path (make-client-socket 'unix path)]
731 [(eq? (~ conn'secure) 'tls)
732 (cond-expand
733 [gauche.net.tls
734 (tls-connect (make-tls :server-name host
735 :options (if (tls-ca-bundle-path)
736 0
737 SSL_SERVER_VERIFY_LATER))
738 host (or port "443") 'tcp)]
739 [else
740 (error "TLS support is not available on this Gauche (you need to recompile Gauche with TLS support on.)")])]
741 [(eq? (~ conn'secure) 'stunnel)
742 (let1 host:port (cond [port #"~|host|:~|port|"]
743 [(string-index host #\:) host]
744 [else #"~|host|:https"])
745 (make-stunnel-connection host:port))]
746 [else (make-client-socket host (if port (x->integer port) 80))])))
747
748 (set! (~ conn'socket) (connect-socket)))
749
750 (define (with-connection conn proc)
751 (unwind-protect
752 (begin
753 (unless (~ conn'socket) (start-connection conn))
754 (proc (connection-input-port (~ conn'socket))
755 (connection-output-port (~ conn'socket))))
756 (unless (~ conn'persistent)
757 (reset-http-connection conn))))
758
759 ;; canonicalize uri for the sake of redirection.
760 ;; URI is a request-uri given to the API, or the redirect location specified
761 ;; in 3xx response. It can be a full URI or just a path w.r.t. the current
762 ;; accessing server, so we pass the current server name as HOST in order to
763 ;; fill the URI if necessary.
764 ;; Returns three values; the full URI to access (it is used to detect a loop
765 ;; in redirections), the server name, and the new request uri.
766 (define (canonical-uri conn uri host)
767 (let*-values ([(scheme specific) (uri-scheme&specific uri)]
768 [(h p q f) (uri-decompose-hierarchical specific)])
769 (let ([scheme (or scheme (if (~ conn'secure) "https" "http"))]
770 [host (or h host)])
771 (values (uri-compose :scheme scheme :host host
772 :path p :query q :fragment f)
773 scheme
774 (or h host)
775 ;; drop "//"
776 (string-drop (uri-compose :path p :query q :fragment f) 2)))))
777
778 ;; canonicalize host and uri w.r.t. proxy
779 (define (consider-proxy conn host uri)
780 (if (ref conn'proxy)
781 (values host (uri-compose :scheme "http" :host (ref conn'server) :path* uri))
782 (values host uri)))
783
784 ;; send
785 (define (send-request out method uri sender headers enc)
786 (define request-line #"~method ~uri HTTP/1.1\r\n")
787 (define request-headers
788 ($ map (cut map (^s (if (keyword? s) (keyword->string s) (x->string s))) <>)
789 $ slices headers 2))
790 (case method
791 [(POST PUT)
792 (sender request-headers enc
793 (^[hdrs]
794 (send-headers request-line hdrs out)
795 (let ([chunked?
796 (equal? (rfc822-header-ref hdrs "transfer-encoding")
797 "chunked")]
798 [first-time #t])
799 (^[size]
800 (when chunked?
801 (unless first-time (display "\r\n" out))
802 (format out "~x\r\n" size))
803 (flush out)
804 out))))]
805 [else (send-headers request-line request-headers out)]))
806
807 ;; NB: We try to send the request line and headers in one packet if possible,
808 ;; since some http servers assumes important headers can be read in single
809 ;; read() call.
810 (define (send-headers request-line hdrs out)
811 (display (tree->string
812 `(,request-line
813 ,@(map (^h `(,(car h)": ",(cadr h)"\r\n")) hdrs)
814 "\r\n"))
815 out)
816 (flush out))
817
818 ;; receive
819 (define (receive-header remote)
820 (receive (code reason) (parse-status-line (read-line remote))
821 (values code (rfc822-header->list remote))))
822
823 (define (parse-status-line line)
824 (cond [(eof-object? line)
825 (error <http-error> "http reply contains no data")]
826 [(#/\w+\s+(\d\d\d)\s+(.*)/ line) => (^m (values (m 1) (m 2)))]
827 [else (error <http-error> "bad reply from server" line)]))
828
829 (define (receive-body remote code headers receiver)
830 (let1 total (and-let* ([p (assoc "content-length" headers)])
831 (x->integer (cadr p)))
832 (if-let1 enc (assoc "transfer-encoding" headers)
833 (if (equal? (cadr enc) "chunked")
834 (receive-body-chunked remote code headers total receiver)
835 (error <http-error> "unsupported transfer-encoding:" (cadr enc)))
836 (receive-body-once remote code headers total receiver))))
837
838 (define (receive-body-once remote code headers total receiver)
839 ;; Callback will be called twice (unless total is 0). The first
840 ;; time we return # of total bytes, the second time zero.
841 (let1 rest total
842 (define (callback)
843 (if (equal? rest 0)
844 (values remote 0)
845 (begin (set! rest 0) (values remote total))))
846 (receiver code headers total callback)))
847
848 ;; NB: chunk extension and trailer are ignored for now.
849 (define (receive-body-chunked remote code headers total receiver)
850 (define chunk-size #f)
851 (define condition #f)
852 (define (callback)
853 (if (equal? chunk-size 0)
854 (values remote 0) ;; finalize
855 ;; If we get an error during receiving from the server, we need
856 ;; to return -1 to give the chance to the receiver to clean up
857 ;; things. After the receiver returns we reraise the condition.
858 (guard (e [else (set! condition e) (values remote -1)])
859 ;; If we've already handled some chunks, we need to skip
860 ;; the trailing CRLF of the previous chunk.
861 (when chunk-size
862 (read-line remote))
863 (let1 line (read-line remote)
864 (when (eof-object? line)
865 (error <http-error> "chunked body ended prematurely"))
866 (rxmatch-if (#/^([[:xdigit:]]+)/ line) (#f digits)
867 (begin
868 (set! chunk-size (string->number digits 16))
869 (if (zero? chunk-size)
870 ;; finish reading trailer
871 (do ([line (read-line remote) (read-line remote)])
872 [(or (eof-object? line) (string-null? line))
873 (values remote 0)])
874 (values remote chunk-size)))
875 ;; something's wrong
876 (error <http-error> "bad line in chunked data:" line))))))
877 (begin0 (receiver code headers total callback)
878 (when condition (raise condition))))
879
880 ;;==============================================================
881 ;; authentication handling
882 ;;
883
884 (define (http-auth-headers conn)
885 (or (and-let* ([auth-handler (~ conn'auth-handler)])
886 (auth-handler conn))
887 '()))
888
889 (define (http-basic-auth-handler conn)
890 (and-let* ([user (~ conn 'auth-user)]
891 [pass (or (~ conn 'auth-password) "")])
892 `(:authorization ,($ format "Basic ~a"
893 $ base64-encode-string #"~|user|:~|pass|"))))
894
895 (define http-default-auth-handler
896 (make-parameter http-basic-auth-handler))