Debian Codegraph @

libsrc/rfc/mime.scm

raw
1 ;;;
2 ;;; mime.scm - parsing MIME (rfc2045) message
3 ;;;
4 ;;; Copyright (c) 2000-2024 Shiro Kawai <[email protected]>
5 ;;;
6 ;;; Redistribution and use in source and binary forms, with or without
7 ;;; modification, are permitted provided that the following conditions
8 ;;; are met:
9 ;;;
10 ;;; 1. Redistributions of source code must retain the above copyright
11 ;;; notice, this list of conditions and the following disclaimer.
12 ;;;
13 ;;; 2. Redistributions in binary form must reproduce the above copyright
14 ;;; notice, this list of conditions and the following disclaimer in the
15 ;;; documentation and/or other materials provided with the distribution.
16 ;;;
17 ;;; 3. Neither the name of the authors nor the names of its contributors
18 ;;; may be used to endorse or promote products derived from this
19 ;;; software without specific prior written permission.
20 ;;;
21 ;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 ;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 ;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 ;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 ;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 ;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 ;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 ;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 ;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 ;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 ;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 ;;;
33
34 ;; RFC2045 Multipurpose Internet Mail Extensions (MIME) Part One:
35 ;; Format of Internet Message Bodies
36 ;; RFC2046 Multipurpose Internet Mail Extensions (MIME) Part Two:
37 ;; Media Types
38 ;; RFC2047 Multipurpose Internet Mail Extensions (MIME) Part Three:
39 ;; Message Header Extensions for Non-ASCII Text
40 ;; RFC2183 Communicating Presentation Information in Internet Messages:
41 ;; The Content-Disposition Header Field
42 ;; RFC2231 MIME Parameter Value and Encoded Word Extensions:
43 ;; Character Sets, Languages, and Continuations
44
45 (define-module rfc.mime
46 (use srfi.13)
47 (use scheme.charset)
48 (use rfc.822)
49 (use util.match)
50 (export mime-parse-version
51 mime-parse-content-type mime-parse-content-disposition
52 mime-parse-parameters mime-compose-parameters
53 mime-encode-word mime-encode-text
54 mime-decode-word mime-decode-text
55 <mime-part>
56 mime-parse-message mime-retrieve-body
57 mime-body->string mime-body->file
58 mime-make-boundary mime-compose-message mime-compose-message-string
59 )
60 )
61 (select-module rfc.mime)
62
63 (autoload rfc.quoted-printable quoted-printable-decode-string
64 quoted-printable-encode quoted-printable-encode-string)
65 (autoload rfc.base64 base64-decode-string base64-decode
66 base64-encode-string base64-encode)
67 (autoload gauche.charconv
68 ces-upper-compatible? ces-conversion-supported? ces-convert)
69 (autoload rfc.mime-port make-mime-port)
70 (autoload srfi.27 random-integer) ;for MIME boundary generation
71
72 ;;===============================================================
73 ;; Basic utility
74 ;;
75
76 ;; returns list of major and minor versions in integers
77 (define (mime-parse-version field)
78 (and-let* ([ field ]
79 [s (string-concatenate
80 (map x->string (rfc822-field->tokens field)))]
81 [m (#/^(\d+)\.(\d+)$/ s)])
82 (map (^d (x->integer (m d))) '(1 2))))
83
84 (define-constant *ct-token-chars*
85 (char-set-difference #[\u0021-\u007e] #[()<>@,\;:\\\"/\[\]?=]))
86
87 ;; RFC2045 Content-Type header field
88 ;; returns (<type> <subtype> (<attribute> . <value>) ...)
89 (define (mime-parse-content-type field)
90 (and field
91 (call-with-input-string field
92 (^[input]
93 (and-let* ([type (rfc822-next-token input `(,*ct-token-chars*))]
94 [ (string? type) ]
95 [ (eqv? #\/ (rfc822-next-token input '())) ]
96 [subtype (rfc822-next-token input `(,*ct-token-chars*))]
97 [ (string? subtype) ])
98 (list* (string-downcase type)
99 (string-downcase subtype)
100 (mime-parse-parameters input)))))))
101
102 ;; RFC2183 Content-Disposition header field
103 ;; returns (<token> (<attribute> . <value>) ...)
104 (define (mime-parse-content-disposition field)
105 (and field
106 (call-with-input-string field
107 (^[input]
108 (and-let* ([token (rfc822-next-token input `(,*ct-token-chars*))]
109 [ (string? token) ])
110 (cons (string-downcase token)
111 (mime-parse-parameters input)))))))
112
113
114 ;; parse a parameter-values type header field
115 ;; ;parameter=value;parameter=value
116 ;; => ((parameter . value) ...)
117 ;; NB: This will support RFC2231, but not yet.
118 (define (mime-parse-parameters :optional (input (current-input-port)))
119 (let loop ((r '()))
120 (cond [(and-let* ([ (eqv? #\; (rfc822-next-token input '())) ]
121 [attr (rfc822-next-token input `(,*ct-token-chars*))]
122 [ (string? attr) ]
123 [ (eqv? #\= (rfc822-next-token input '())) ]
124 [val (rfc822-next-token
125 input
126 `(,*ct-token-chars*
127 (#[\"] . ,rfc822-quoted-string)))]
128 [ (string? val) ])
129 (cons attr val))
130 => (^p (loop (cons p r)))]
131 [else (reverse! r)])))
132
133 ;; Inverse of mime-parse-parameters.
134 ;; ((parameter . value) ...) => ;parameter=value;parameter=value ...
135 ;; NB: This will support RFC2231, but not yet.
136 (define (mime-compose-parameters pvs
137 :optional (port (current-output-port))
138 :key (start-column 0))
139 (define (quote-value v) ; TODO: possibly folding?
140 (if (string-every *ct-token-chars* v)
141 v
142 (string-append "\"" (regexp-replace-all #/[\"\\]/ v "\\\\\\0") "\"")))
143 (define (valid-name p)
144 (rlet1 z (x->string p)
145 (unless (string-every *ct-token-chars* z)
146 (error "invalid parameter value for rfc2822 header:" p))))
147 (define (gen)
148 (fold (^[pv column]
149 (match pv
150 [(p . v)
151 (let* ([z #"~(valid-name p)=~(quote-value (x->string v))"]
152 [len (+ (string-length z) column)])
153 (cond [(> len 78)
154 (display ";\r\n ") (display z) (string-length z)]
155 [else
156 (display "; ") (display z) (+ len column 2)]))]
157 [_ (error "bad parameter-value entry:" pv)]))
158 start-column pvs))
159 (match port
160 [#f (with-output-to-string gen)]
161 [#t (gen)]
162 [(? port?) (with-output-to-port port gen)]))
163
164 ;;===============================================================
165 ;; RFC2047 header field encoding
166 ;;
167
168 ;; Decoding
169
170 ;; the end is not anchored, to be used in mime-decode-text.
171 (define-constant *mime-encoded-header-rx*
172 #/^=\?([-!#-'*+\w\^-~]+)\?([-!#-'*+\w\^-~]+)\?([!->@-~]+)\?=/)
173
174 (define (%mime-decode-word word charset encoding body)
175 (if (ces-conversion-supported? charset #f)
176 (guard (e (else word)) ;; capture illegal encoding
177 (cond [(string-ci=? encoding "q")
178 (ces-convert (quoted-printable-decode-string body) charset #f)]
179 [(string-ci=? encoding "b")
180 (ces-convert (base64-decode-string body) charset #f)]
181 [else word])) ;; unsupported encoding
182 word))
183
184 ;; decode rfc2047-encoded word, i.e. "=?...?="
185 ;; if word isn't legal encoded word, it is returned as is.
186 (define (mime-decode-word word)
187 (cond [(string-incomplete? word) word] ;; safety net
188 [(rxmatch *mime-encoded-header-rx* word)
189 => (^m (if (equal? (m'after) "")
190 (%mime-decode-word word (m 1) (m 2) (m 3))
191 word))]
192 [else word]))
193
194 ;; Decode the entire header field body, possibly a mixture of
195 ;; encoded-words and ordinary words. NOTE: If you apply this to
196 ;; a structured field body, the decoded words may contain special
197 ;; characters meaningful to the structured body and thus confuse
198 ;; the parser. The correct order is to parse first, then apply
199 ;; mime-decode-word for each token.
200 (define (mime-decode-text body)
201 (let loop ([s body])
202 (receive (pre rest) (string-scan s "=?" 'before*)
203 (cond [(not pre) s]
204 [(rxmatch *mime-encoded-header-rx* rest)
205 => (^m (string-append pre
206 (%mime-decode-word (m 0) (m 1) (m 2) (m 3))
207 (loop (m'after))))]
208 [else s]))))
209
210 ;; Encode a single word. This one always encode, even WORD contains
211 ;; US-ASCII characters only.
212 (define (mime-encode-word word :key (charset "utf-8") (transfer-encoding 'base64))
213 (let ([converted
214 (cond [(ces-upper-compatible? charset (gauche-character-encoding)) word]
215 [else (ces-convert word (gauche-character-encoding) charset)])]
216 [enc (%canonical-encoding transfer-encoding)])
217 (format "=?~a?~a?~a?=" charset enc
218 ((if (eq? enc 'B)
219 base64-encode-string
220 quoted-printable-encode-string)
221 converted :line-width #f))))
222
223 ;; Encode entire body if necessary, considering line folding.
224 ;; At this moment, this function just treat the body as unstructured
225 ;; text. It's not safe to pass structured text. In future we may have
226 ;; a keyword option to make body parsed as structured text.
227 (define (mime-encode-text body :key
228 (charset "utf-8")
229 (transfer-encoding 'base64)
230 (line-width 76)
231 (start-column 0)
232 (force #f))
233 (let ([enc (%canonical-encoding transfer-encoding)]
234 [cslen (string-length (x->string charset))]
235 [pass-through? (and (not force)
236 (ces-upper-compatible? charset 'ascii)
237 (string-every #[\u0000-\u007f] body))])
238 ;; estimates the length of an encoded word. it is not trivial since
239 ;; we have to ces-convert each segment separately, and there's no
240 ;; general way to estimate the size of ces-converted string.
241 ;; we throw in some heuristics here.
242 (define (estimate-width s i)
243 (let1 na (string-count s #[\u0000-\u007f] 0 i)
244 (+ 6 cslen
245 (if (eq? enc 'B)
246 (ceiling (* (+ na (* (- i na) 3)) 4/3))
247 (let1 ng (string-count s #[!-<>-~] 0 i)
248 (+ (- na ng) (* 3 (+ ng (* (- i na) 3)))))))))
249 (define (encode-word w)
250 (mime-encode-word w :charset charset :transfer-encoding enc))
251 ;; we don't need to pack optimally, so we use some heuristics.
252 (define (encode str width adj)
253 (or (and-let* ([estim (estimate-width str (string-length str))]
254 [ (< (* adj estim) width) ]
255 [ew (encode-word str)])
256 (if (<= (string-length ew) width)
257 `(,ew)
258 (encode str width (* adj (/. (string-length ew) width)))))
259 (let loop ([k (min (string-length str) (quotient width 2))])
260 (let1 estim (* adj (estimate-width str k))
261 (if (<= estim width)
262 (let1 ew (encode-word (string-take str k))
263 (if (<= (string-length ew) width)
264 (list* ew "\r\n "
265 (encode (string-drop str k) (- line-width 1) adj))
266 (loop (floor->exact
267 (* k (/ width (string-length ew)))))))
268 (loop (floor->exact (* k (/ width estim)))))))
269 ))
270 ;; fill pass-through text. we try our best to look for an appropriate
271 ;; place to insert FWS. we know STR is all ASCII.
272 (define (fill str width)
273 (if (<= (string-length str) width)
274 `(,str)
275 (or (and-let* ([pos (string-index-right str #\space 0 width)])
276 (list* (string-take str pos) "\r\n "
277 (fill (string-drop str (+ pos 1)) (- line-width 1))))
278 ;; if we can't find a whitespace, we break in the middle of
279 ;; word for the last resort.
280 (list* (string-take str width) "\r\n "
281 (fill (string-drop str width) (- line-width 1))))))
282
283 (cond [(or (not line-width) (zero? line-width))
284 (if pass-through? body (encode-word body))]
285 [(< line-width 30)
286 (errorf "line width (~a) is too short to encode header field body: ~s" line-width body)]
287 [(< (- line-width start-column) 30)
288 ;; just in case if header name is very long. we insert line break
289 ;; first.
290 (string-concatenate
291 (cons "\r\n " (if pass-through?
292 (fill body (- line-width 1))
293 (encode body (- line-width 1) 1.0))))]
294 [else
295 (string-concatenate
296 (if pass-through?
297 (fill body (- line-width start-column))
298 (encode body (- line-width start-column) 1.0)))])))
299
300 (define (%canonical-encoding transfer-encoding)
301 (case transfer-encoding
302 [(B b base64) 'B]
303 [(Q q quoted-printable) 'Q]
304 [else (error "unsupported MIME header encoding specifier:"
305 transfer-encoding)]))
306
307
308 ;;===============================================================
309 ;; Basic streaming parser
310 ;;
311
312 ;; message information packet
313 (define-class <mime-part> ()
314 ((type :init-keyword :type :init-value "text")
315 (subtype :init-keyword :subtype :init-value "plain")
316 (parameters :init-keyword :parameters :init-value '())
317 (transfer-encoding :init-keyword :transfer-encoding :init-value #f)
318 (headers :init-keyword :headers :init-value '())
319 (parent :init-keyword :parent :init-value #f)
320 (index :init-keyword :index :init-value 0)
321 (content :init-keyword :content :init-value #f)
322 (source :init-keyword :source :init-value #f) ; only used for composing
323 ))
324
325 (define (mime-parse-message port headers handler)
326 (internal-parse port headers handler #f 0
327 '("text" "plain" ("charset" . "us-ascii"))))
328
329 (define (internal-parse port headers handler parent index default-type)
330 (let* ([ctype (or (mime-parse-content-type
331 (rfc822-header-ref headers "content-type"))
332 default-type)]
333 [enc (rfc822-header-ref headers "content-transfer-encoding" "7bit")]
334 [packet (make <mime-part>
335 :type (car ctype)
336 :subtype (cadr ctype)
337 :parameters (cddr ctype)
338 :transfer-encoding enc
339 :parent parent
340 :index index
341 :headers headers)])
342 (cond
343 [(equal? (car ctype) "multipart")
344 (multipart-parse port packet handler)]
345 [(equal? (car ctype) "message")
346 (message-parse port packet handler)]
347 [else
348 ;; normal body
349 (slot-set! packet 'content (handler packet port))
350 packet]
351 )))
352
353 (define (multipart-parse port packet handler)
354 (let* ([boundary (or (assoc-ref (ref packet 'parameters) "boundary")
355 (error "No boundary given for multipart message"))]
356 [default-type (if (equal? (ref packet 'subtype) "digest")
357 '("message" "rfc822")
358 '("text" "plain" ("charset" . "us-ascii")))]
359 [mime-port (make-mime-port boundary port)])
360 (let loop ([index 0]
361 [contents '()])
362 (let* ([headers (rfc822-header->list mime-port)]
363 [r (internal-parse mime-port headers handler
364 packet index
365 default-type)])
366 (case (ref mime-port 'state)
367 [(boundary)
368 (set! (ref mime-port 'state) 'body)
369 (loop (+ index 1) (cons r contents))]
370 [(eof)
371 (set! (ref packet 'content) (reverse! (cons r contents)))
372 packet]
373 [else ;; parser returned without readling entire part.
374 ;; discard the rest of the part.
375 (let loop ((b (read-byte port)))
376 (unless (eof-object? b)
377 (loop (read-byte port))))
378 packet])))
379 ))
380
381 (define (message-parse port packet handler)
382 (let* ([headers (rfc822-header->list port)]
383 [r (internal-parse port headers handler packet 0
384 '("text" "plain" ("charset" . "us-ascii")))])
385 (set! (ref packet 'content) (list r)))
386 packet)
387
388 ;;===============================================================
389 ;; Body readers
390 ;;
391
392 (define (mime-retrieve-body packet inp outp)
393
394 (define (read-line/nl)
395 (let loop ([c (read-char inp)]
396 [chars '()])
397 (cond [(eof-object? c)
398 (if (null? chars) c (list->string (reverse! chars)))]
399 [(char=? c #\newline) (list->string (reverse! (cons c chars)))]
400 [(char=? c #\return)
401 (let1 c (peek-char inp)
402 (if (char=? c #\newline)
403 (list->string (reverse! (list* (read-char inp)
404 #\return
405 chars)))
406 (list->string (reverse! (cons #\return chars)))))]
407 [else (loop (read-char inp) (cons c chars))])))
408
409 (define (read-text decoder)
410 (let loop ([line (read-line/nl)])
411 (unless (eof-object? line)
412 (display (decoder line) outp)
413 (loop (read-line/nl)))))
414
415 (define (read-base64)
416 (define (base64-output string out)
417 (with-input-from-string string
418 (cut with-port-locking (current-input-port)
419 (cut with-output-to-port out base64-decode))))
420 (let1 buf (open-output-string :private? #t)
421 (let loop ([line (read-line inp)])
422 (unless (eof-object? line)
423 (display line buf)
424 (loop (read-line inp))))
425 (base64-output (get-output-string buf) outp))
426 )
427
428 (with-port-locking inp
429 (^[] (let1 enc (ref packet 'transfer-encoding)
430 (cond
431 [(string-ci=? enc "base64") (read-base64)]
432 [(string-ci=? enc "quoted-printable")
433 (read-text quoted-printable-decode-string)]
434 [(member enc '("7bit" "8bit" "binary"))
435 (let loop ((b (read-byte inp)))
436 (unless (eof-object? b)
437 (write-byte b outp)
438 (loop (read-byte inp))))]
439 ))))
440 )
441
442 (define (mime-body->string packet inp)
443 (let1 s (open-output-string :private? #t)
444 (mime-retrieve-body packet inp s)
445 (get-output-string s)))
446
447 (define (mime-body->file packet inp filename)
448 (call-with-output-file filename
449 (^[outp]
450 (with-port-locking outp
451 (cut mime-retrieve-body packet inp outp))))
452 filename)
453
454 ;;===============================================================
455 ;; MIME composer
456 ;;
457
458 ;; Each PART must be either an instance of <mime-part>, or a list
459 ;; of <mime-part-list>, where
460 ;; <mime-part-list> : (<content-type> (<header> ...) <body>)
461 ;; <content-type> : (<type> <subtype> <header-param> ...)
462 ;; <header-param> : (<key> . <value>) ...
463 ;; <header> : (<header-name> <encoded-header-value>)
464 ;; | (<header-name (<header-value> <header-param> ...))
465 ;; <body> : string
466 ;; | (file <filename>)
467 ;; | (subparts <part> ...)
468 ;;
469 ;; Note: In the first form of <header>, <encoded-header-value> must
470 ;; already be encoded using RFC2047 or RFC2231 if the original value
471 ;; contains non-ascii characters.
472 ;; In the second form, we plan to do RFC2231 encoding on behalf of
473 ;; the caller; but the current version does not implement it. The
474 ;; caller SHOULD NOT pass encoded words in this form, since it may
475 ;; result double-encoding when we implement the auto encoding feature;
476 ;; for the time being, the second form restricts ASCII-only values.
477
478 ;; Emits composed message to PORT, and returns boundary string.
479 ;; The boundary string is automatically generated unless provided
480 ;; by the optional argument.
481 (define (mime-compose-message parts
482 :optional (port (current-output-port))
483 :key (boundary (mime-make-boundary)))
484 (dolist [p parts]
485 (for-each (cut display <> port) `("\r\n--" ,boundary "\r\n"))
486 (mime-generate-one-part (canonical-part p) port))
487 (for-each (cut display <> port) `("\r\n--" ,boundary "--\r\n"))
488 boundary)
489
490 ;; Returns composed message in string, AND the boundary.
491 (define (mime-compose-message-string parts
492 :key (boundary (mime-make-boundary)))
493 (values (call-with-output-string
494 (cut mime-compose-message parts <> :boundary boundary))
495 boundary))
496
497 (define (mime-make-boundary)
498 (format "boundary-~a" (number->string (* (random-integer (expt 2 64))
499 (sys-time) (sys-getpid))
500 36)))
501
502 ;; internal stuff
503 (define (canonical-part p)
504 (match p
505 [(? (cut is-a? <> <mime-part>)) p]
506 [((type subtype . params) (headers ...) body)
507 (let1 hs (filter-map canonical-header headers)
508 (apply make <mime-part>
509 :type type :subtype subtype :parameters params :headers hs
510 :transfer-encoding (rfc822-header-ref hs "content-transfer-encoding")
511 (match body
512 [(? string?) `(:content ,body)]
513 [('file name) `(:source ,name)]
514 [('subparts ps ...) `(:content ,(map canonical-part ps))]
515 [_ (error "Invalid mime part body spec:" body)])))]
516 [_ (error "Invalid mime part spec:" p)]))
517
518 (define (canonical-header header)
519 (match header
520 [(name . x) (cons (x->string name) x)]
521 [_ #f]))
522
523 (define (mime-generate-one-part part port)
524 (when (list? (ref part'content))
525 (unless (member "boundary" (ref part'parameters))
526 (push! (ref part'parameters) (cons "boundary" (mime-make-boundary)))))
527 (let1 cte (mime-generate-part-header part port)
528 (with-output-to-port port
529 (^[]
530 (cond
531 [(ref part'source) =>
532 (cut with-input-from-file <> (cut mime-generate-part-body part cte))]
533 [(list? (ref part'content))
534 (mime-compose-message (ref part'content) port
535 :boundary (assoc-ref (ref part'parameters)
536 "boundary"))]
537 [(string? (ref part'content))
538 (with-input-from-string (ref part'content)
539 (cut mime-generate-part-body part cte))]
540 [else (error "unsupported MIME part content")])))))
541
542 ;; returns content-transfer-encoding
543 (define (mime-generate-part-header part port)
544 (rlet1 cte (ref part'transfer-encoding)
545 (rfc822-write-headers
546 `(("Content-type"
547 ,(format "~a/~a~a" (ref part'type) (ref part'subtype)
548 (mime-compose-parameters (ref part'parameters) #f)))
549 ,@(cond-list [cte => (cut list "Content-transfer-encoding" <>)])
550 ,@(filter-map gen-header-1 (ref part'headers)))
551 :output port :check :ignore)))
552
553 (define (gen-header-1 h)
554 (match h
555 [("content-transfer-encoding" . _) #f]
556 [("content-type" . _) #f]
557 [(name (value pv ...))
558 (let* ([sval (x->string value)]
559 [spvs (mime-compose-parameters pv #f
560 :start-column (+ (string-length name) (string-length sval) 2))])
561 `(,name ,(if (null? pv) sval #"~|sval|~|spvs|")))]
562 [(name value) h]))
563
564 ;; current-input-port -> current-output-port
565 (define (mime-generate-part-body part transfer-enc)
566 (cond
567 [(or (not transfer-enc) (member transfer-enc '("binary" "7bit") string-ci=?))
568 (copy-port (current-input-port) (current-output-port))]
569 [(string-ci=? transfer-enc "base64") (base64-encode)]
570 [(string-ci=? transfer-enc "quoted-printable") (quoted-printable-encode)]
571 [else (error "Unsupported transfer encoding encountered while composing \
572 mime message: " transfer-enc)]))