| 1 |
;;; |
| 2 |
;;; http.scm - HTTP 1.1 |
| 3 |
;;; |
| 4 |
;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]> |
| 5 |
;;; |
| 6 |
;;; Redistribution and use in source and binary forms, with or without |
| 7 |
;;; modification, are permitted provided that the following conditions |
| 8 |
;;; are met: |
| 9 |
;;; |
| 10 |
;;; 1. Redistributions of source code must retain the above copyright |
| 11 |
;;; notice, this list of conditions and the following disclaimer. |
| 12 |
;;; |
| 13 |
;;; 2. Redistributions in binary form must reproduce the above copyright |
| 14 |
;;; notice, this list of conditions and the following disclaimer in the |
| 15 |
;;; documentation and/or other materials provided with the distribution. |
| 16 |
;;; |
| 17 |
;;; 3. Neither the name of the authors nor the names of its contributors |
| 18 |
;;; may be used to endorse or promote products derived from this |
| 19 |
;;; software without specific prior written permission. |
| 20 |
;;; |
| 21 |
;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 22 |
;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 23 |
;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 24 |
;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 25 |
;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 26 |
;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED |
| 27 |
;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| 28 |
;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| 29 |
;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| 30 |
;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| 31 |
;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 32 |
;;; |
| 33 |
|
| 34 |
;; HTTP handling routines. |
| 35 |
|
| 36 |
;; RFC2616 Hypertext Transfer Protocol -- HTTP/1.1 |
| 37 |
;; http://www.ietf.org/rfc/rfc2616.txt |
| 38 |
;; RFC2617 HTTP Authentication: Basic and Digest Access Authentication |
| 39 |
;; http://www.ietf.org/rfc/rfc2617.txt |
| 40 |
;; RFC2388 Returning Values from Forms: multipart/form-data |
| 41 |
;; http://www.ietf.org/rfc/rfc2388.txt |
| 42 |
|
| 43 |
;; HTTP 1.1 has lots of features. This module doesn't yet cover all of them. |
| 44 |
;; The features required for typical client usage are implemented first. |
| 45 |
|
| 46 |
(define-module rfc.http |
| 47 |
(use srfi.13) |
| 48 |
(use rfc.822) |
| 49 |
(use rfc.uri) |
| 50 |
(use rfc.base64) |
| 51 |
(use rfc.tls) |
| 52 |
(use gauche.net) |
| 53 |
(use gauche.charconv) |
| 54 |
(use gauche.sequence) |
| 55 |
(use gauche.uvector) |
| 56 |
(use gauche.connection) |
| 57 |
(use util.match) |
| 58 |
(use text.tree) |
| 59 |
(export <http-error> |
| 60 |
http-user-agent make-http-connection reset-http-connection |
| 61 |
http-compose-query http-compose-form-data |
| 62 |
http-status-code->description |
| 63 |
|
| 64 |
http-proxy http-request |
| 65 |
http-null-receiver http-string-receiver http-oport-receiver |
| 66 |
http-file-receiver http-cond-receiver |
| 67 |
http-null-sender http-string-sender http-blob-sender |
| 68 |
http-file-sender http-multipart-sender |
| 69 |
|
| 70 |
http-get http-head http-post http-put http-delete |
| 71 |
http-default-auth-handler |
| 72 |
http-default-redirect-handler |
| 73 |
|
| 74 |
http-secure-connection-available? |
| 75 |
) |
| 76 |
) |
| 77 |
(select-module rfc.http) |
| 78 |
|
| 79 |
;; |
| 80 |
(autoload rfc.mime |
| 81 |
mime-compose-message |
| 82 |
mime-compose-message-string |
| 83 |
mime-compose-parameters |
| 84 |
mime-parse-content-type) |
| 85 |
(autoload rfc.http.tunnel probe-stunnel make-stunnel-connection) |
| 86 |
(autoload file.util file-size find-file-in-paths null-device) |
| 87 |
|
| 88 |
;============================================================== |
| 89 |
;; Conditions |
| 90 |
;; |
| 91 |
|
| 92 |
(define-condition-type <http-error> <error> #f) |
| 93 |
|
| 94 |
;;============================================================== |
| 95 |
;; Global parameters |
| 96 |
;; |
| 97 |
|
| 98 |
;; default string to be used for user-agent. |
| 99 |
(define http-user-agent |
| 100 |
(make-parameter #"gauche.http/~(gauche-version)")) |
| 101 |
|
| 102 |
;; global proxy settings. can be overridden by :proxy keyword |
| 103 |
;; argument. |
| 104 |
(define http-proxy (make-parameter #f)) |
| 105 |
|
| 106 |
;; The default redirect handler |
| 107 |
(define http-default-redirect-handler |
| 108 |
(make-parameter |
| 109 |
(^[method code headers body] |
| 110 |
(and-let* ([loc (rfc822-header-ref headers "location")]) |
| 111 |
(case (x->integer code) |
| 112 |
[(300 301 305 307) |
| 113 |
(case method [(GET HEAD) `(,method . ,loc)] [else #f])] |
| 114 |
[(302 303) ;found / see other |
| 115 |
;; See rfc2616 notes - the agent isn't supposed to automatically |
| 116 |
;; redirect 302 response of POST, but redirect with GET for it is |
| 117 |
;; the de-facto behavior. |
| 118 |
(case method |
| 119 |
[(GET HEAD) `(,method . ,loc)] |
| 120 |
[else `(GET . ,loc)])] |
| 121 |
[else #f]))))) |
| 122 |
|
| 123 |
;;============================================================== |
| 124 |
;; Higher-level API |
| 125 |
;; |
| 126 |
|
| 127 |
;; Higher-level API is for conventional call-return API. |
| 128 |
;; |
| 129 |
;; The METHOD argument specifies the http request method by a symbol. |
| 130 |
;; GET, HEAD, POST, PUT and DELETE is currently supported. |
| 131 |
;; |
| 132 |
;; The SERVER argument maybe a string naming the server (and |
| 133 |
;; optionally a port number by the format of "server:port"), or |
| 134 |
;; an <http-connection> object. Using a server name is suitable |
| 135 |
;; for easy one-shot http access; the connection and related states |
| 136 |
;; are discarded once the procedure returns. |
| 137 |
;; On the other hand, a connection object can keep the states such |
| 138 |
;; as persistent connection and authentication tokens, suitable for |
| 139 |
;; a series of communications to a server. |
| 140 |
;; |
| 141 |
;; The REQUEST-URI argument can be a string as specified in RFC2616, |
| 142 |
;; or a list in the form of (<path> (<name> <value>) ...). In the |
| 143 |
;; latter form, (<name> <value>) assoc list is converted into a |
| 144 |
;; url query form as defined in HTML4 (application/x-www-form-urlencoded) |
| 145 |
;; and appended to <path>. |
| 146 |
;; |
| 147 |
;; The keyword arguments are handled by various low-level routines, |
| 148 |
;; and here's the summary: |
| 149 |
;; |
| 150 |
;; request-body - gives the body. |
| 151 |
;; receiver - A procedure that handles the response. |
| 152 |
;; It takes four arguments: response-code, response headers, |
| 153 |
;; total response size if known (#f otherwise), and |
| 154 |
;; a retriever thunk. The retriever thunk is, when called, |
| 155 |
;; returns two values: an input port and an integer size. |
| 156 |
;; The receiver must loop (1) to read as many bytes as |
| 157 |
;; specified in the size from the input port, and (2) |
| 158 |
;; call the retriver again, while it returns positive integer. |
| 159 |
;; The retriever procedure returns zero when body is exhausted. |
| 160 |
;; In that case the value(s) the receiver returns will be |
| 161 |
;; the return value of http-request. |
| 162 |
;; The retriever thunk can return #f as size, if we don't know the |
| 163 |
;; chunk of response size. In that case, the receiver may read |
| 164 |
;; up to EOF, or as much data as it wants, then call the |
| 165 |
;; retriever. The second time the retriever returns size=0, |
| 166 |
;; so you can do cleanup work. |
| 167 |
;; The retriever procedure returns -1 if an error occurs |
| 168 |
;; during comminucation. In that case, the receiver can do |
| 169 |
;; whatever cleanup work. After the receiver returns, an |
| 170 |
;; appropriate error is thrown from http-request. |
| 171 |
;; |
| 172 |
;; sender - A procedure that handles sending the request body. |
| 173 |
;; It takes three arguments: tentative list of headers, |
| 174 |
;; the value given to :request-encoding argument, |
| 175 |
;; and a procedure HEADER-SINK. |
| 176 |
;; |
| 177 |
;; HEADER-SINK takes one argument, a list of http headers |
| 178 |
;; in the form of (("field-name" "field-value") ...). |
| 179 |
;; In general, besides the given tentative headers, the sender |
| 180 |
;; needs to add either Content-Length header if it sends entire |
| 181 |
;; contents at once, or ("transfer-encoding" "chunked") if |
| 182 |
;; it uses chunked transfer. If neither headers exist in the |
| 183 |
;; given header list, an error is raised and request is aborted. |
| 184 |
;; Sender can add other headers, e.g. content-type or mime-type. |
| 185 |
;; Sender can also remove or modify given tentative headers. |
| 186 |
;; |
| 187 |
;; HEADER-SINK returns a procedure BODY-SINK on success. |
| 188 |
;; BODY-SINK takes an integer SIZE argument and returns an |
| 189 |
;; output PORT. What the sender needs to do is to call |
| 190 |
;; BODY-SINK with the size of data chunk, then to write out |
| 191 |
;; the data into the returned PORT, and repeat it until |
| 192 |
;; all the data is sent. To indicate the end of data, |
| 193 |
;; the sender needs to call BODY-SINK with argument 0. |
| 194 |
;; |
| 195 |
;; host - the host name passed to the 'host' header field. |
| 196 |
;; secure - if true, using secure connection (via gauche.tls). |
| 197 |
;; auth-user, auth-password, auth-handler - authentication parameters. |
| 198 |
;; request-encoding - when http-* is to construct request-uri and/or |
| 199 |
;; request body, this argument specifies the character encoding |
| 200 |
;; to be used as the external encoding. |
| 201 |
;; redirect-handler - Called if the server responds with 3xx status. |
| 202 |
;; The argument is the request method, a status code, list of |
| 203 |
;; headers and response body (can be #f). It may return a |
| 204 |
;; (METHOD . URL) or #f. For the first case, |
| 205 |
;; http-request re-attempts to fetch the URL with the given method. |
| 206 |
;; (unless we're not looping). If it returns #f, the original |
| 207 |
;; code, headers and body are returned from http-request. |
| 208 |
;; If given #t (default), the procedure bound to the parameter |
| 209 |
;; http-default-redirect-handler is called. |
| 210 |
;; no-redirect - If true, we ignore the value of redirect-handler and |
| 211 |
;; returns without attempting retrying. |
| 212 |
;; This is provided for the backward compatibility; newer code |
| 213 |
;; should use :redirect-handler #f |
| 214 |
;; |
| 215 |
;; Other unrecognized options are passed as request headers. |
| 216 |
|
| 217 |
(define (http-request method server request-uri |
| 218 |
:key (host #f) |
| 219 |
(redirect-handler #t) |
| 220 |
(no-redirect #f) |
| 221 |
auth-handler |
| 222 |
auth-user |
| 223 |
auth-password |
| 224 |
(proxy (http-proxy)) |
| 225 |
(user-agent (http-user-agent)) |
| 226 |
(secure #f) |
| 227 |
(receiver (http-string-receiver)) |
| 228 |
(sender #f) |
| 229 |
((:request-encoding enc) (gauche-character-encoding)) |
| 230 |
:allow-other-keys opts) |
| 231 |
|
| 232 |
(define extra-headers |
| 233 |
($ concatenate $ reverse |
| 234 |
$ rlet1 z '() |
| 235 |
(do-plist [(k v) opts] |
| 236 |
(when v (push! z (list k v)))))) |
| 237 |
|
| 238 |
(define conn (ensure-connection server auth-handler auth-user auth-password |
| 239 |
proxy secure extra-headers)) |
| 240 |
(define redirector (if no-redirect |
| 241 |
#f |
| 242 |
(case redirect-handler |
| 243 |
[(#t) (http-default-redirect-handler)] |
| 244 |
[(#f) #f] |
| 245 |
[else => identity]))) |
| 246 |
(define no-body-replies '("204" "304")) |
| 247 |
|
| 248 |
(define (get-body iport method code headers receiver) |
| 249 |
(and (not (eq? method 'HEAD)) |
| 250 |
(not (member code no-body-replies)) |
| 251 |
(receive-body iport code headers receiver))) |
| 252 |
|
| 253 |
;; final touch of request headers |
| 254 |
(define (req-headers host) |
| 255 |
(cond-list [(~ conn'persistent) @ (if (~ conn'proxy) |
| 256 |
'(:proxy-connection keep-alive) |
| 257 |
'(:connection keep-alive))] |
| 258 |
[#t @ `(:host ,host :user-agent ,user-agent |
| 259 |
,@(http-auth-headers conn) ,@extra-headers)])) |
| 260 |
|
| 261 |
;; If we decide to give up redirection, we read from already-retrieved |
| 262 |
;; body of 3xx reply. This modifies reply headers if necessary. |
| 263 |
(define (redirect-headers body rep-headers) |
| 264 |
(if body |
| 265 |
`(("content-length" ,(string-size body)) |
| 266 |
,@(remove (^p (member (car p) '("content-length" |
| 267 |
"content-transfer-encoding"))) |
| 268 |
rep-headers)) |
| 269 |
rep-headers)) |
| 270 |
|
| 271 |
;; returns either one of: |
| 272 |
;; (reply <code> <headers> <body>) |
| 273 |
;; (redirect-to <method> <location>) |
| 274 |
(define (request-response in out method uri host sender) |
| 275 |
(send-request out method uri sender (req-headers host) enc) |
| 276 |
(receive (code rep-headers) (receive-header in) |
| 277 |
(if-let1 consider-redirect (and (string-prefix? "3" code) redirector) |
| 278 |
;; we retrieve body as string, not using caller-provided receiver |
| 279 |
(let* ([body (get-body in method code rep-headers |
| 280 |
(http-string-receiver))] |
| 281 |
[verdict (consider-redirect method code rep-headers body)]) |
| 282 |
;; consider-redirect returns either #f (don't redirect) or |
| 283 |
;; (METHOD . LOCATION). |
| 284 |
(if verdict |
| 285 |
`(redirect-to ,(car verdict) ,(cdr verdict)) |
| 286 |
(let1 hdrs (redirect-headers body rep-headers) ;giving up |
| 287 |
`(reply ,code ,hdrs |
| 288 |
,(and body |
| 289 |
(receive-body (open-input-string body) code |
| 290 |
hdrs receiver)))))) |
| 291 |
;; no redirection |
| 292 |
`(reply ,code ,rep-headers |
| 293 |
,(get-body in method code rep-headers receiver))))) |
| 294 |
|
| 295 |
;; main loop |
| 296 |
(let loop ([history '()] |
| 297 |
[host host] |
| 298 |
[method method] |
| 299 |
[request-uri (ensure-request-uri request-uri enc)]) |
| 300 |
(receive (host uri) |
| 301 |
(consider-proxy conn (or host (~ conn'server)) request-uri) |
| 302 |
(let1 result |
| 303 |
(with-connection |
| 304 |
conn |
| 305 |
(^[i o] (request-response i o method uri host sender))) |
| 306 |
(match result |
| 307 |
[('reply code rep-headers body) (values code rep-headers body)] |
| 308 |
[('redirect-to method location) |
| 309 |
(receive (uri proto new-server path*) |
| 310 |
(canonical-uri conn location (ref conn'server)) |
| 311 |
(when (or (member uri history) |
| 312 |
(> (length history) 20)) |
| 313 |
(errorf <http-error> "redirection is looping via ~a" uri)) |
| 314 |
(loop (cons uri history) |
| 315 |
(~ (redirect-connection! conn proto new-server)'server) |
| 316 |
method path*))]))))) |
| 317 |
|
| 318 |
;; |
| 319 |
;; Pre-defined receivers |
| 320 |
;; |
| 321 |
(define (http-string-receiver) |
| 322 |
(^[code hdrs total retr] |
| 323 |
;; TODO: check headers for encoding |
| 324 |
(let loop ([sink (open-output-string)]) |
| 325 |
(receive (remote size) (retr) |
| 326 |
(cond [(eqv? size 0) (get-output-string sink)] |
| 327 |
[(or (not size) (> size 0)) |
| 328 |
(copy-port remote sink :size size) (loop sink)]))))) |
| 329 |
|
| 330 |
(define (http-null-receiver) |
| 331 |
(^[code hdrs total retr] |
| 332 |
(let loop ([sink (open-output-file (null-device))]) |
| 333 |
(receive (remote size) (retr) |
| 334 |
(cond [(and size (<= size 0)) (close-output-port sink)] |
| 335 |
[else (copy-port remote sink :size size) (loop sink)]))))) |
| 336 |
|
| 337 |
(define (http-oport-receiver sink flusher) |
| 338 |
(^[code hdrs total retr] |
| 339 |
(let loop () |
| 340 |
(receive (remote size) (retr) |
| 341 |
(cond [(and size (<= size 0)) (flusher sink hdrs)] |
| 342 |
[else (copy-port remote sink :size size) (loop)]))))) |
| 343 |
|
| 344 |
(define (http-file-receiver filename :key (temporary #f)) |
| 345 |
(^[code hdrs total retr] |
| 346 |
(receive (port tmpname) (sys-mkstemp filename) |
| 347 |
(let loop () |
| 348 |
(receive (remote size) (retr) |
| 349 |
(cond [(or (not size) (> size 0)) |
| 350 |
(copy-port remote port :size size) (loop)] |
| 351 |
[(= size 0) |
| 352 |
(close-output-port port) |
| 353 |
(if temporary |
| 354 |
tmpname |
| 355 |
(begin (sys-rename tmpname filename) filename))] |
| 356 |
[else (close-output-port port) (sys-unlink tmpname)])))))) |
| 357 |
|
| 358 |
(define-syntax http-cond-receiver |
| 359 |
(syntax-rules (else =>) |
| 360 |
[(_) (http-null-receiver)] |
| 361 |
[(_ [else . exprs]) (begin . exprs)] |
| 362 |
[(_ [cc => proc] . rest) |
| 363 |
(^[code hdrs total retr] |
| 364 |
((if (match-status-code? cc code) |
| 365 |
proc |
| 366 |
(http-cond-receiver . rest)) |
| 367 |
code hdrs total retr))] |
| 368 |
[(_ [cc . exprs] . rest) |
| 369 |
(^[code hdrs total retr] |
| 370 |
((if (match-status-code? cc code '(cc . exprs)) |
| 371 |
(begin . exprs) |
| 372 |
(http-cond-receiver . rest)) |
| 373 |
code hdrs total retr))] |
| 374 |
[(_ other . rest) |
| 375 |
(syntax-error "invalid clause in http-cond-receiver" other)])) |
| 376 |
|
| 377 |
(define (match-status-code? pattern code clause) |
| 378 |
(cond [(string? pattern) (equal? pattern code)] |
| 379 |
[(regexp? pattern) (rxmatch pattern code)] |
| 380 |
[else (error "Invalid pattern in a clause of http-cond-receiver:" |
| 381 |
clause)])) |
| 382 |
|
| 383 |
;; |
| 384 |
;; Senders |
| 385 |
;; |
| 386 |
|
| 387 |
(define (http-null-sender) |
| 388 |
(^[hdrs encoding header-sink] |
| 389 |
(let1 body-sink (header-sink `(("content-length" "0") ,@hdrs)) |
| 390 |
(body-sink 0)))) |
| 391 |
|
| 392 |
(define (http-string-sender string) ;honors encoding |
| 393 |
(^[hdrs encoding header-sink] |
| 394 |
(let* ([body (if (ces-equivalent? encoding (gauche-character-encoding)) |
| 395 |
string |
| 396 |
(ces-convert string (gauche-character-encoding) encoding))] |
| 397 |
[size (string-size body)] |
| 398 |
[body-sink (header-sink `(("content-length" ,(x->string size)) |
| 399 |
,@hdrs))] |
| 400 |
[oport (body-sink size)]) |
| 401 |
(display body oport) |
| 402 |
(body-sink 0)))) |
| 403 |
|
| 404 |
(define (http-blob-sender blob) ;blob may be a string or uvector |
| 405 |
(^[hdrs encoding header-sink] |
| 406 |
(let* ([size (if (string? blob) (string-size blob) (uvector-size blob))] |
| 407 |
[body-sink (header-sink `(("content-length" ,(x->string size)) |
| 408 |
,@hdrs))] |
| 409 |
[port (body-sink size)]) |
| 410 |
(if (string? blob) |
| 411 |
(display blob port) |
| 412 |
(write-block blob port)) |
| 413 |
(body-sink 0)))) |
| 414 |
|
| 415 |
;; Send contents directly from the file. Encoding is ignored. |
| 416 |
;; TODO: The file size may be changed while sending out. If it gets |
| 417 |
;; bigger, we can just ignore the rest, but what if it gets shorter? |
| 418 |
(define (http-file-sender filename) |
| 419 |
(^[hdrs encoding header-sink] |
| 420 |
(let* ([size (file-size filename)] |
| 421 |
[body-sink (header-sink `(("content-length" ,(x->string size)) |
| 422 |
,@hdrs))] |
| 423 |
[port (body-sink size)]) |
| 424 |
(call-with-input-file filename (cut copy-port <> port :size size)) |
| 425 |
(body-sink 0)))) |
| 426 |
|
| 427 |
;; See http-compose-form-data definition for params spec. |
| 428 |
;; TODO: support chunked sending, instead of building entire body at once. |
| 429 |
(define (http-multipart-sender params) |
| 430 |
(^[hdrs encoding header-sink] |
| 431 |
(receive (body boundary) (http-compose-form-data params #f encoding) |
| 432 |
(let* ([size (string-size body)] |
| 433 |
[hdrs `(("content-length" ,(x->string size)) |
| 434 |
("mime-version" "1.0") |
| 435 |
("content-type" ,#"multipart/form-data; boundary=\"~|boundary|\"") |
| 436 |
,@(alist-delete "content-type" hdrs equal?))] |
| 437 |
[body-sink (header-sink hdrs)] |
| 438 |
[port (body-sink size)]) |
| 439 |
(display body port) |
| 440 |
(body-sink 0))))) |
| 441 |
|
| 442 |
;; |
| 443 |
;; Shortcuts for specific requests. |
| 444 |
;; |
| 445 |
(define (http-get server request-uri . options) |
| 446 |
(apply %http-request-adaptor 'GET server request-uri #f options)) |
| 447 |
|
| 448 |
(define (http-head server request-uri . options) |
| 449 |
(apply %http-request-adaptor 'HEAD server request-uri #f options)) |
| 450 |
|
| 451 |
(define (http-post server request-uri body . options) |
| 452 |
(apply %http-request-adaptor 'POST server request-uri body options)) |
| 453 |
|
| 454 |
(define (http-put server request-uri body . options) |
| 455 |
(apply %http-request-adaptor 'PUT server request-uri body options)) |
| 456 |
|
| 457 |
(define (http-delete server request-uri . options) |
| 458 |
(apply %http-request-adaptor 'DELETE server request-uri #f options)) |
| 459 |
|
| 460 |
;; Adaptor to the new API. Converts :sink and :flusher arguments, |
| 461 |
;; which are superseded by :receiver arguments. |
| 462 |
(define (%http-request-adaptor method server request-uri body |
| 463 |
:key (receiver #f) (sink #f) (flusher #f) |
| 464 |
:allow-other-keys opts) |
| 465 |
(define recvr |
| 466 |
(cond [(and sink flusher) |
| 467 |
(http-oport-receiver sink flusher)] |
| 468 |
[(or sink flusher) |
| 469 |
(errorf "You need to provide :sink and :flusher together to http-~a" |
| 470 |
(string-downcase (symbol->string method)))] |
| 471 |
[receiver] |
| 472 |
;; fallback |
| 473 |
[else (http-oport-receiver (open-output-string) |
| 474 |
(^[s h] (get-output-string s)))])) |
| 475 |
(apply http-request method server request-uri |
| 476 |
:sender (cond [(not body) (http-null-sender)] |
| 477 |
[(list? body) (http-multipart-sender body)] |
| 478 |
[else (http-blob-sender body)]) |
| 479 |
:receiver recvr opts)) |
| 480 |
|
| 481 |
;;============================================================== |
| 482 |
;; HTTP connection context |
| 483 |
;; |
| 484 |
|
| 485 |
;; <http-connection> object can be used to have conversations with |
| 486 |
;; (usually) a specific server. A typical usage is to emulate a |
| 487 |
;; browser to perform a certain transaction spanning to several |
| 488 |
;; webpages. (For the simple "one-shot" request-response access, |
| 489 |
;; the rfc.http APIs create a temporary connection under the hood.) |
| 490 |
|
| 491 |
(define-class <http-connection> () |
| 492 |
;; All slots are private. |
| 493 |
((server :init-keyword :server) ; server[:port] |
| 494 |
(socket :init-value #f) ; A <connection> object; either <socket> |
| 495 |
; or wrapped by secure agent. |
| 496 |
; If it is shutdown by the server, |
| 497 |
; the APIs attempt to reconnect. |
| 498 |
(persistent :init-keyword :persistent) ; true for persistent connection. |
| 499 |
(auth-handler :init-keyword :auth-handler :init-value #f) |
| 500 |
(auth-user :init-keyword :auth-user :init-value #f) |
| 501 |
(auth-password :init-keyword :auth-password :init-value #f) |
| 502 |
(proxy :init-keyword :proxy) |
| 503 |
(extra-headers :init-keyword :extra-headers) |
| 504 |
(secure :init-keyword :secure) ; either #f, tls or stunnel |
| 505 |
)) |
| 506 |
|
| 507 |
(define (make-http-connection server :key |
| 508 |
(persistent #t) |
| 509 |
(auth-handler #f) |
| 510 |
(auth-user #f) |
| 511 |
(auth-password #f) |
| 512 |
(proxy #f) |
| 513 |
(extra-headers '())) |
| 514 |
(make <http-connection> |
| 515 |
:persistent persistent |
| 516 |
:server server |
| 517 |
:auth-handler (or auth-handler (http-default-auth-handler)) |
| 518 |
:auth-user auth-user |
| 519 |
:auth-password auth-password |
| 520 |
:proxy proxy |
| 521 |
:extra-headers extra-headers)) |
| 522 |
|
| 523 |
;; This modifies CONN. |
| 524 |
(define (redirect-connection! conn proto new-server) |
| 525 |
(let1 orig-server (~ conn'server) |
| 526 |
;; Unless we're connecting the same server with same secure level, |
| 527 |
;; we reset the connection. |
| 528 |
(unless (and (string=? orig-server new-server) |
| 529 |
(or (and (equal? proto "https") (~ conn'secure)) |
| 530 |
(and (not (equal? proto "https")) (not (~ conn'secure))))) |
| 531 |
(and-let1 c (~ conn'socket) |
| 532 |
(connection-shutdown (~ conn'socket) 'both) |
| 533 |
(set! (~ conn'socket) #f)) |
| 534 |
(set! (~ conn'server) new-server) |
| 535 |
;; NB: If the server redirects http connection to https, we don't have |
| 536 |
;; a chance for the caller to select alternative TLS flavor---for the |
| 537 |
;; time being we blindly assume tls. |
| 538 |
(set! (~ conn'secure) (and (equal? proto "https") 'tls)))) |
| 539 |
conn) |
| 540 |
|
| 541 |
;; API |
| 542 |
(define (reset-http-connection conn) |
| 543 |
(when (~ conn'socket) |
| 544 |
(guard (e [(<system-error> e) #f]) |
| 545 |
(connection-shutdown (~ conn'socket) 'both) |
| 546 |
(connection-close (~ conn'socket))) |
| 547 |
(set! (~ conn'socket) #f))) |
| 548 |
|
| 549 |
|
| 550 |
;; API |
| 551 |
(define (http-secure-connection-available? :optional (type #t)) |
| 552 |
(case type |
| 553 |
[(#t tls) |
| 554 |
(cond-expand |
| 555 |
[gauche.net.tls #t] |
| 556 |
[else #f])] |
| 557 |
[(stunnel) |
| 558 |
(boolean (probe-stunnel))] |
| 559 |
[else |
| 560 |
(error "Unknown secure connection type (must be tls, stunnel or #t):" |
| 561 |
type)])) |
| 562 |
|
| 563 |
;;============================================================== |
| 564 |
;; query and request body composition |
| 565 |
;; |
| 566 |
|
| 567 |
;; Query string composition. |
| 568 |
;; NB: Query string syntax (aka application/x-www-form-urlencoded) is |
| 569 |
;; not defined in RFC2616. In fact, it is not clearly defined at all |
| 570 |
;; in RFC. The most definitive source might be the HTML4 specification, |
| 571 |
;; section 17.13.4 "Form content types", |
| 572 |
;; <http://www.w3.org/TR/html4/interact/forms.html#h-17.13.4.1>. |
| 573 |
;; In reality it is used very frequently with http, so we put it here. |
| 574 |
(define (http-compose-query path params |
| 575 |
:optional (encoding (gauche-character-encoding))) |
| 576 |
(cond [(not path) (uri-compose-query params)] |
| 577 |
[(null? params) path] |
| 578 |
[else #"~|path|?~(uri-compose-query params)"])) |
| 579 |
|
| 580 |
;; multipart/form-data composition [RFC2388] |
| 581 |
;; <params> : (<param> ...) |
| 582 |
;; <param> : (<name> <value>) ; same as http-compose-query |
| 583 |
;; | (<name> <key> <value> <key2> <value2> ...) |
| 584 |
;; <key> : :value | :file | :content-type | :content-transfer-encoding |
| 585 |
;; | other keywords (used as a header name) |
| 586 |
;; composed message is put to the current output port. |
| 587 |
;; returns the boundary string. |
| 588 |
(define (http-compose-form-data params port |
| 589 |
:optional (encoding (gauche-character-encoding))) |
| 590 |
(define (translate-param param) |
| 591 |
(match param |
| 592 |
[(name value) (translate-param `(,name :value ,value))] |
| 593 |
[(name . kvs) |
| 594 |
(unless (even? (length kvs)) |
| 595 |
(error "Invalid parameter format to create multipart/form-data:" param)) |
| 596 |
(let-keywords kvs ([value ""] |
| 597 |
[file #f] |
| 598 |
[content-type #f] |
| 599 |
[content-transfer-encoding #f] . other-keys) |
| 600 |
`(,(canonical-content-type (mime-parse-content-type content-type) |
| 601 |
value file) |
| 602 |
(("content-transfer-encoding" ,(or content-transfer-encoding "binary")) |
| 603 |
("content-disposition" ,(make-content-disposition name file)) |
| 604 |
,@(map (cut map x->string <>) (slices other-keys 2))) |
| 605 |
,(if file `(file ,file) (x->string value))))])) |
| 606 |
(define (canonical-content-type ct value file) |
| 607 |
(match ct |
| 608 |
[#f (if (or file (string-incomplete? value)) |
| 609 |
'("application" "octet-stream") |
| 610 |
`("text" "plain" ("charset" . ,(x->string encoding))))] |
| 611 |
[(type subtype . options) |
| 612 |
(if (assoc "charset" options) |
| 613 |
ct |
| 614 |
`(,type ,subtype ("charset" . ,(x->string encoding)) ,@options))])) |
| 615 |
(define (make-content-disposition name file) |
| 616 |
(with-output-to-string |
| 617 |
(^[] |
| 618 |
(display "form-data") |
| 619 |
(mime-compose-parameters |
| 620 |
`(("name" . ,name) |
| 621 |
,@(cond-list [file `("filename" . ,file)])))))) |
| 622 |
(if (not port) |
| 623 |
(mime-compose-message-string (map translate-param params)) |
| 624 |
(mime-compose-message (map translate-param params) port))) |
| 625 |
|
| 626 |
;;============================================================== |
| 627 |
;; status codes |
| 628 |
;; |
| 629 |
|
| 630 |
(define *status-code-map* |
| 631 |
(hash-table 'eqv? |
| 632 |
'(100 . "Continue") |
| 633 |
'(101 . "Switching Protocols") |
| 634 |
'(200 . "OK") |
| 635 |
'(201 . "Created") |
| 636 |
'(202 . "Accepted") |
| 637 |
'(203 . "Non-Authoritative Information") |
| 638 |
'(204 . "No Content") |
| 639 |
'(205 . "Reset Content") |
| 640 |
'(206 . "Partial Content") |
| 641 |
'(300 . "Multiple Choices") |
| 642 |
'(301 . "Moved Permanently") |
| 643 |
'(302 . "Found") |
| 644 |
'(303 . "See Other") |
| 645 |
'(304 . "Not Modified") |
| 646 |
'(305 . "Use Proxy") |
| 647 |
'(306 . "(Unused)") |
| 648 |
'(307 . "Temporary Redirect") |
| 649 |
'(400 . "Bad Request") |
| 650 |
'(401 . "Unauthorized") |
| 651 |
'(402 . "Payment Required") |
| 652 |
'(403 . "Forbidden") |
| 653 |
'(404 . "Not Found") |
| 654 |
'(405 . "Method Not Allowed") |
| 655 |
'(406 . "Not Acceptable") |
| 656 |
'(407 . "Proxy Authentication Required") |
| 657 |
'(408 . "Request Timeout") |
| 658 |
'(409 . "Conflict") |
| 659 |
'(410 . "Gone") |
| 660 |
'(411 . "Length Required") |
| 661 |
'(412 . "Precondition Failed") |
| 662 |
'(413 . "Request Entity Too Large") |
| 663 |
'(414 . "Request-URI Too Long") |
| 664 |
'(415 . "Unsupported Media Type") |
| 665 |
'(416 . "Requested Range Not Satisfiable") |
| 666 |
'(417 . "Expectation Failed") |
| 667 |
'(500 . "Internal Server Error") |
| 668 |
'(501 . "Not Implemented") |
| 669 |
'(502 . "Bad Gateway") |
| 670 |
'(503 . "Service Unavailable") |
| 671 |
'(504 . "Gateway Timeout") |
| 672 |
'(505 . "HTTP Version Not Supported") |
| 673 |
)) |
| 674 |
|
| 675 |
;; API |
| 676 |
;; code can be an integer or a string, e.g. "200" |
| 677 |
;; returns #f for unknown code |
| 678 |
(define (http-status-code->description code) |
| 679 |
(hash-table-get *status-code-map* (x->integer code) #f)) |
| 680 |
|
| 681 |
;;============================================================== |
| 682 |
;; internal utilities |
| 683 |
;; |
| 684 |
|
| 685 |
(define (ensure-request-uri request-uri enc) |
| 686 |
(match request-uri |
| 687 |
[(? string?) request-uri] |
| 688 |
[(path n&v ...) (http-compose-query path n&v enc)] |
| 689 |
[_ (error "Invalid request-uri form for http request API:" request-uri)])) |
| 690 |
|
| 691 |
;; Always returns a connection object. |
| 692 |
(define (ensure-connection server auth-handler auth-user auth-password |
| 693 |
proxy secure extra-headers) |
| 694 |
(rlet1 conn (cond |
| 695 |
[(is-a? server <http-connection>) server] |
| 696 |
[(string? server) (make-http-connection server :persistent #f)] |
| 697 |
[else (error "bad type of argument for server: must be an <http-connection> object or a string of the server's name, but got:" server)]) |
| 698 |
;; TODO: Might need to reset connections if parameters are changed |
| 699 |
(let-syntax ([check-override |
| 700 |
(syntax-rules () |
| 701 |
[(_ id) |
| 702 |
(unless (undefined? id) (set! (ref conn'id) id))])]) |
| 703 |
(check-override auth-handler) |
| 704 |
(check-override auth-user) |
| 705 |
(check-override auth-password) |
| 706 |
(check-override proxy) |
| 707 |
(check-override extra-headers) |
| 708 |
(set! (~ conn'secure) |
| 709 |
(and (not (undefined? secure)) |
| 710 |
(case secure |
| 711 |
[(#f) #f] |
| 712 |
[(#t tls) 'tls] |
| 713 |
[(stunnel) 'stunnel] |
| 714 |
[else (error "`secure' argument for http connection must be \ |
| 715 |
either boolean, 'tls or 'stunnel, but got:" |
| 716 |
secure)])))))) |
| 717 |
|
| 718 |
(define (start-connection conn) |
| 719 |
;; If address is given ipv6 format such as "[::1]:port", we have to |
| 720 |
;; use "::1" part as the hostname, excluding []. |
| 721 |
(define (parse-address addr) |
| 722 |
(rxmatch-case addr |
| 723 |
[#/^unix:(\/.*)$/ (_ path) (values #f #f path)] ;unix domain |
| 724 |
[#/^\[([a-fA-F\d:]+)\](?::(\d+))?$/ (_ host port) (values host port #f)] |
| 725 |
[#/^([^:]+)(?::(\d+))?$/ (_ host port) (values host port #f)] |
| 726 |
[else (error "Unrecognized http server address:" addr)])) |
| 727 |
(define (connect-socket) |
| 728 |
(receive (host port path) |
| 729 |
(parse-address (or (~ conn'proxy) (~ conn'server))) |
| 730 |
(cond [path (make-client-socket 'unix path)] |
| 731 |
[(eq? (~ conn'secure) 'tls) |
| 732 |
(cond-expand |
| 733 |
[gauche.net.tls |
| 734 |
(tls-connect (make-tls :server-name host |
| 735 |
:options (if (tls-ca-bundle-path) |
| 736 |
0 |
| 737 |
SSL_SERVER_VERIFY_LATER)) |
| 738 |
host (or port "443") 'tcp)] |
| 739 |
[else |
| 740 |
(error "TLS support is not available on this Gauche (you need to recompile Gauche with TLS support on.)")])] |
| 741 |
[(eq? (~ conn'secure) 'stunnel) |
| 742 |
(let1 host:port (cond [port #"~|host|:~|port|"] |
| 743 |
[(string-index host #\:) host] |
| 744 |
[else #"~|host|:https"]) |
| 745 |
(make-stunnel-connection host:port))] |
| 746 |
[else (make-client-socket host (if port (x->integer port) 80))]))) |
| 747 |
|
| 748 |
(set! (~ conn'socket) (connect-socket))) |
| 749 |
|
| 750 |
(define (with-connection conn proc) |
| 751 |
(unwind-protect |
| 752 |
(begin |
| 753 |
(unless (~ conn'socket) (start-connection conn)) |
| 754 |
(proc (connection-input-port (~ conn'socket)) |
| 755 |
(connection-output-port (~ conn'socket)))) |
| 756 |
(unless (~ conn'persistent) |
| 757 |
(reset-http-connection conn)))) |
| 758 |
|
| 759 |
;; canonicalize uri for the sake of redirection. |
| 760 |
;; URI is a request-uri given to the API, or the redirect location specified |
| 761 |
;; in 3xx response. It can be a full URI or just a path w.r.t. the current |
| 762 |
;; accessing server, so we pass the current server name as HOST in order to |
| 763 |
;; fill the URI if necessary. |
| 764 |
;; Returns three values; the full URI to access (it is used to detect a loop |
| 765 |
;; in redirections), the server name, and the new request uri. |
| 766 |
(define (canonical-uri conn uri host) |
| 767 |
(let*-values ([(scheme specific) (uri-scheme&specific uri)] |
| 768 |
[(h p q f) (uri-decompose-hierarchical specific)]) |
| 769 |
(let ([scheme (or scheme (if (~ conn'secure) "https" "http"))] |
| 770 |
[host (or h host)]) |
| 771 |
(values (uri-compose :scheme scheme :host host |
| 772 |
:path p :query q :fragment f) |
| 773 |
scheme |
| 774 |
(or h host) |
| 775 |
;; drop "//" |
| 776 |
(string-drop (uri-compose :path p :query q :fragment f) 2))))) |
| 777 |
|
| 778 |
;; canonicalize host and uri w.r.t. proxy |
| 779 |
(define (consider-proxy conn host uri) |
| 780 |
(if (ref conn'proxy) |
| 781 |
(values host (uri-compose :scheme "http" :host (ref conn'server) :path* uri)) |
| 782 |
(values host uri))) |
| 783 |
|
| 784 |
;; send |
| 785 |
(define (send-request out method uri sender headers enc) |
| 786 |
(define request-line #"~method ~uri HTTP/1.1\r\n") |
| 787 |
(define request-headers |
| 788 |
($ map (cut map (^s (if (keyword? s) (keyword->string s) (x->string s))) <>) |
| 789 |
$ slices headers 2)) |
| 790 |
(case method |
| 791 |
[(POST PUT) |
| 792 |
(sender request-headers enc |
| 793 |
(^[hdrs] |
| 794 |
(send-headers request-line hdrs out) |
| 795 |
(let ([chunked? |
| 796 |
(equal? (rfc822-header-ref hdrs "transfer-encoding") |
| 797 |
"chunked")] |
| 798 |
[first-time #t]) |
| 799 |
(^[size] |
| 800 |
(when chunked? |
| 801 |
(unless first-time (display "\r\n" out)) |
| 802 |
(format out "~x\r\n" size)) |
| 803 |
(flush out) |
| 804 |
out))))] |
| 805 |
[else (send-headers request-line request-headers out)])) |
| 806 |
|
| 807 |
;; NB: We try to send the request line and headers in one packet if possible, |
| 808 |
;; since some http servers assumes important headers can be read in single |
| 809 |
;; read() call. |
| 810 |
(define (send-headers request-line hdrs out) |
| 811 |
(display (tree->string |
| 812 |
`(,request-line |
| 813 |
,@(map (^h `(,(car h)": ",(cadr h)"\r\n")) hdrs) |
| 814 |
"\r\n")) |
| 815 |
out) |
| 816 |
(flush out)) |
| 817 |
|
| 818 |
;; receive |
| 819 |
(define (receive-header remote) |
| 820 |
(receive (code reason) (parse-status-line (read-line remote)) |
| 821 |
(values code (rfc822-header->list remote)))) |
| 822 |
|
| 823 |
(define (parse-status-line line) |
| 824 |
(cond [(eof-object? line) |
| 825 |
(error <http-error> "http reply contains no data")] |
| 826 |
[(#/\w+\s+(\d\d\d)\s+(.*)/ line) => (^m (values (m 1) (m 2)))] |
| 827 |
[else (error <http-error> "bad reply from server" line)])) |
| 828 |
|
| 829 |
(define (receive-body remote code headers receiver) |
| 830 |
(let1 total (and-let* ([p (assoc "content-length" headers)]) |
| 831 |
(x->integer (cadr p))) |
| 832 |
(if-let1 enc (assoc "transfer-encoding" headers) |
| 833 |
(if (equal? (cadr enc) "chunked") |
| 834 |
(receive-body-chunked remote code headers total receiver) |
| 835 |
(error <http-error> "unsupported transfer-encoding:" (cadr enc))) |
| 836 |
(receive-body-once remote code headers total receiver)))) |
| 837 |
|
| 838 |
(define (receive-body-once remote code headers total receiver) |
| 839 |
;; Callback will be called twice (unless total is 0). The first |
| 840 |
;; time we return # of total bytes, the second time zero. |
| 841 |
(let1 rest total |
| 842 |
(define (callback) |
| 843 |
(if (equal? rest 0) |
| 844 |
(values remote 0) |
| 845 |
(begin (set! rest 0) (values remote total)))) |
| 846 |
(receiver code headers total callback))) |
| 847 |
|
| 848 |
;; NB: chunk extension and trailer are ignored for now. |
| 849 |
(define (receive-body-chunked remote code headers total receiver) |
| 850 |
(define chunk-size #f) |
| 851 |
(define condition #f) |
| 852 |
(define (callback) |
| 853 |
(if (equal? chunk-size 0) |
| 854 |
(values remote 0) ;; finalize |
| 855 |
;; If we get an error during receiving from the server, we need |
| 856 |
;; to return -1 to give the chance to the receiver to clean up |
| 857 |
;; things. After the receiver returns we reraise the condition. |
| 858 |
(guard (e [else (set! condition e) (values remote -1)]) |
| 859 |
;; If we've already handled some chunks, we need to skip |
| 860 |
;; the trailing CRLF of the previous chunk. |
| 861 |
(when chunk-size |
| 862 |
(read-line remote)) |
| 863 |
(let1 line (read-line remote) |
| 864 |
(when (eof-object? line) |
| 865 |
(error <http-error> "chunked body ended prematurely")) |
| 866 |
(rxmatch-if (#/^([[:xdigit:]]+)/ line) (#f digits) |
| 867 |
(begin |
| 868 |
(set! chunk-size (string->number digits 16)) |
| 869 |
(if (zero? chunk-size) |
| 870 |
;; finish reading trailer |
| 871 |
(do ([line (read-line remote) (read-line remote)]) |
| 872 |
[(or (eof-object? line) (string-null? line)) |
| 873 |
(values remote 0)]) |
| 874 |
(values remote chunk-size))) |
| 875 |
;; something's wrong |
| 876 |
(error <http-error> "bad line in chunked data:" line)))))) |
| 877 |
(begin0 (receiver code headers total callback) |
| 878 |
(when condition (raise condition)))) |
| 879 |
|
| 880 |
;;============================================================== |
| 881 |
;; authentication handling |
| 882 |
;; |
| 883 |
|
| 884 |
(define (http-auth-headers conn) |
| 885 |
(or (and-let* ([auth-handler (~ conn'auth-handler)]) |
| 886 |
(auth-handler conn)) |
| 887 |
'())) |
| 888 |
|
| 889 |
(define (http-basic-auth-handler conn) |
| 890 |
(and-let* ([user (~ conn 'auth-user)] |
| 891 |
[pass (or (~ conn 'auth-password) "")]) |
| 892 |
`(:authorization ,($ format "Basic ~a" |
| 893 |
$ base64-encode-string #"~|user|:~|pass|")))) |
| 894 |
|
| 895 |
(define http-default-auth-handler |
| 896 |
(make-parameter http-basic-auth-handler)) |