Debian Codegraph @

lib/binary/pack.scm

raw
1 ;;;; binary.pack -- packing and unpacking binary data
2
3 ;;; Author: Alex Shinn <[email protected]>
4
5 ;; It is really insupportable that every hen lays an egg of a different
6 ;; size! What symmetry can there be on the breakfast table? At least
7 ;; they should sort them in dozens at the shop!
8 ;; -- Hercule Poirot, "The Disappearance of Mr. Davenheim"
9
10 ;;; Disclaimer:
11 ;; The following code is horribly complicated. The primary reason is
12 ;; the fact that I wanted to support the @ pack code while only
13 ;; performing the required byte position counting when actually using
14 ;; an @ following a variable length template, without which all the
15 ;; "vlp" references would be gone.
16
17 (define-module binary.pack
18 (use srfi.13) ;; string library
19 (use scheme.charset)
20 (use text.parse)
21 (use gauche.uvector)
22 (use binary.io)
23 (export pack unpack unpack-skip make-packer))
24 (select-module binary.pack)
25
26 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
27 ;; general io utilities
28
29 (define (read-number)
30 (let loop ((c (peek-char))
31 (ls '()))
32 (if (and (char? c) (char-numeric? c))
33 (begin
34 (read-char)
35 (loop (peek-char) (cons c ls)))
36 (string->number (reverse-list->string ls)))))
37
38 (define (string-byte-for-each proc str)
39 (with-input-from-string str
40 (lambda ()
41 (let loop ((i (read-byte)))
42 (unless (eof-object? i)
43 (proc i)
44 (loop (read-byte)))))))
45
46 (define (while-input-from-string str proc)
47 (with-input-from-string str
48 (lambda ()
49 (until (eof-object? (peek-char))
50 (proc)))))
51
52 (define (fold-char-list char-list)
53 (cond
54 ((char-set? char-list) (values char-list #f))
55 ((list? char-list)
56 (let ((cs (char-set-copy char-set:empty))
57 (eof-allowed? #f))
58 (for-each (lambda (item)
59 (cond ((char? item) (char-set-adjoin! cs item))
60 ((char-set? item) (char-set-union! cs item))
61 ((eq? item '*eof*) (set! eof-allowed? #t))
62 (else (error "CHAR-LIST must be a list of characters, character sets and/or symbol '*eof*, but found" item))))
63 char-list)
64 (values cs eof-allowed?)))
65 (else (error "CHAR-LIST must be a char-set or a list of characters, char-sets and/or symbol '*eof*" char-list))))
66
67 ;; text.parse's next-token with a limit
68 (define (next-token-n prefix-char-list/pred break-char-list/pred limit
69 :optional (comment "unexpected EOF") (port (current-input-port)))
70 (define (bad) (errorf "~a~a" (port-position-prefix port) comment))
71 (let ((c (skip-while prefix-char-list/pred port)))
72 (cond
73 ((procedure? break-char-list/pred)
74 (with-output-to-string
75 (lambda ()
76 (let loop ((c c)
77 (i 0))
78 (cond ((break-char-list/pred c))
79 ((eof-object? c) (bad))
80 ((= i limit))
81 (else
82 (display (read-char port))
83 (loop (peek-char port) (+ i 1)))))))
84 )
85 (else
86 (receive (cs eof-ok?) (fold-char-list break-char-list/pred)
87 (with-output-to-string
88 (lambda ()
89 (let loop ((c c)
90 (i 0))
91 (cond ((eof-object? c) (unless eof-ok? (bad)))
92 ((char-set-contains? cs c))
93 ((= i limit))
94 (else (display (read-char port))
95 (loop (peek-char port) (+ i 1))))))))
96 ))))
97
98 ;; same as above but consumes the break-char
99 (define (next-token-n* prefix-char-list/pred break-char-list/pred limit
100 :optional (comment "unexpected EOF") (port (current-input-port)))
101 (let ((res (next-token-n prefix-char-list/pred
102 break-char-list/pred limit comment port)))
103 (if (< (string-size res) limit)
104 (read-char port))
105 res))
106
107 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
108 ;; numeric utilities
109
110 ;; these two only operate on the low-byte of a number
111
112 (define (bit-for-each proc int)
113 (do ((i 0 (+ i 1)))
114 ((= i 8))
115 (if (logbit? i int) (proc 1) (proc 0))))
116
117 (define (bit-reverse-for-each proc int)
118 (do ((i 7 (- i 1)))
119 ((= i -1))
120 (if (logbit? i int) (proc 1) (proc 0))))
121
122 (define (hex-char->number c)
123 (if (< c 16)
124 c
125 (or (digit->integer (integer->char c) 16) 0)))
126
127 (define (number-writer writer . opt-endian)
128 (if (pair? opt-endian)
129 (cute writer <> #f (car opt-endian))
130 writer))
131
132 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
133 ;; parser utilities
134
135 (define (get-splitter count . opt-pad)
136 (if (not (number? count))
137 (cut values <> '())
138 (if (pair? opt-pad)
139 (cute split-at* <> count #t (car opt-pad))
140 (cut split-at* <> count #t))))
141
142 (define (read-bang)
143 (and (equal? (char->integer #\!) (peek-byte)) (read-byte) #t))
144
145 (define (read-count)
146 (skip-pack-comments)
147 (let ((c (peek-char)))
148 (if (char? c)
149 (cond ((char-numeric? c)
150 (read-number))
151 ((eq? c #\*)
152 (begin (read-char) c))
153 ((eq? c #\[)
154 (read-char)
155 (let* ((packers (read-until-token (make-pack-token
156 (char-complement c))
157 0 #f #f))
158 (p (fold pack-merge-folder #f packers)))
159 (if (p 'variable-length?)
160 (error "can't use variable length pack format in []")
161 (p 'length))))
162 (else
163 1))
164 1)))
165
166 (define (get-count unpacked-val)
167 (x->number (car unpacked-val)))
168
169 (define (get-string-padder count . opt-pad)
170 (let ((pad #f))
171 (if (eq? count #\*)
172 (set! count #f)
173 (if (pair? opt-pad) (set! pad (car opt-pad))))
174 (if pad
175 (if (procedure? count)
176 (let ((packer (count 'packer)))
177 ;; size is dynamic, so we leave it as is and pack the size
178 ;; prior to returning the value
179 (lambda (str) (packer (list (string-size str))) str))
180 (cut string-pad-right <> count pad))
181 (cond ((procedure? count)
182 (let ((packer (count 'packer)))
183 (lambda (str) (packer (list (string-size str))) str)))
184 (count
185 (cut string-pad-right <> count))
186 (else
187 identity)))))
188
189 (define (make-port-byte-iterator proc count)
190 (cond
191 ((eq? count #\*)
192 (lambda ()
193 (list
194 (with-output-to-string
195 (cut generator-for-each proc read-byte)))))
196 ((procedure? count)
197 (let ((unpacker (count 'unpacker)))
198 (lambda ()
199 (let ((actual-count (get-count (unpacker))))
200 (list
201 (with-output-to-string
202 (lambda ()
203 (let loop ((i 1))
204 (let ((c (read-byte)))
205 (unless (eof-object? c)
206 (proc c)
207 (unless (= i actual-count)
208 (loop (+ i 1)))))))))))))
209 (else
210 (lambda ()
211 (list
212 (with-output-to-string
213 (lambda ()
214 (let loop ((i 1))
215 (let ((c (read-byte)))
216 (unless (eof-object? c)
217 (proc c)
218 (unless (= i count)
219 (loop (+ i 1)))))))))))))
220
221 ;; Skip a value of a given length n, where n could be an integer,
222 (define (make-skipper n)
223 (cond
224 ((eq? n #\*)
225 (lambda ()
226 (while (not (eof-object? (read-block 1024))))))
227 ((procedure? n)
228 (let ((unpacker (n 'unpacker)))
229 (lambda ()
230 (let ((count (unpacker)))
231 (or (port-seek (current-input-port) count SEEK_CUR)
232 (read-block count))))))
233 (else
234 (lambda ()
235 (or (port-seek (current-input-port) n SEEK_CUR)
236 (read-block n))))))
237
238 (define (char-complement ch)
239 (case ch
240 ((#\() #\)) ((#\)) #\()
241 ((#\[) #\]) ((#\]) #\[)
242 ((#\{) #\}) ((#\}) #\{)
243 ((#\<) #\>) ((#\>) #\<)
244 (else ch)))
245
246 (define (make-pack-token c)
247 (list '*token* c))
248
249 (define the-eof-object (with-input-from-string "" read-char))
250
251 (define (skip-pack-comments)
252 (let loop ((c (peek-char)))
253 (if (eof-object? c)
254 #f
255 (case c
256 ((#\space #\tab #\newline)
257 (read-char)
258 (loop (peek-char)))
259 ((#\#)
260 (skip-until #\newline)
261 (loop (peek-char)))
262 (else #f)))))
263
264 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
265 ;; higher order pack dispatch operations
266
267 ;; basic dispatch interface
268 (define (make-pack-dispatch fixed-len var-len? vlp packer unpacker . opt-args)
269 (let ((skipper (get-optional opt-args unpacker)))
270 (lambda args
271 (let ((command (car args)))
272 (case command
273 ((pack) (apply packer (cdr args)))
274 ((unpack) (apply unpacker (cdr args)))
275 ((skip) (apply skipper (cdr args)))
276 ((packer) packer)
277 ((unpacker) unpacker)
278 ((skipper) skipper)
279 ((length) fixed-len)
280 ((variable-length?) var-len?)
281 ((var-len-param) vlp)
282 ;; XXXX deprecate these?
283 ((set-packer!) (set! packer (cadr args)))
284 ((set-unpacker!) (set! unpacker (cadr args)))
285 ((set-skipper!) (set! skipper (cadr args)))
286 ((set-length!) (set! fixed-len (cadr args)))
287 ((set-variable-length?!) (set! var-len? (cadr args)))
288 ((set-var-len-param!) (set! vlp (cadr args)))
289 ((set!)
290 (let ((slot (cadr args))
291 (val (caddr args)))
292 (case slot
293 ((packer) (set! packer val))
294 ((unpacker) (set! unpacker val))
295 ((skipper) (set! skipper val))
296 ((length) (set! fixed-len val))
297 ((variable-length?) (set! var-len? val))
298 ((var-len-param) (set! vlp val))
299 (else (error "unknown pack slot: " slot)))))
300 (else (error "unknown pack command: " command)))))))
301
302 (define (copy-pack-dispatch packer)
303 (make-pack-dispatch (packer 'length) (packer 'variable-length?)
304 (packer 'var-len-param) (packer 'packer)
305 (packer 'unpacker) (packer 'skipper)))
306
307 ;; XXXX use this to clean up more code
308 (define (modify-pack-dispatch packer . args)
309 (if (null? args)
310 packer
311 (let-optionals* args ((accessor #f)
312 (mutator identity))
313 (packer 'set! accessor (mutator (packer accessor)))
314 (apply modify-pack-dispatch packer (cddr args)))))
315
316 ;; make a basic dispatcher for string types
317 (define (make-string-pack-dispatcher vlp count . opt-pad)
318 (let* ((padder (if (pair? opt-pad)
319 (get-string-padder count (car opt-pad))
320 (get-string-padder count)))
321 (pad (if (pair? opt-pad) (car opt-pad) #\space)))
322 (make-pack-dispatch
323 (if (number? count) count 0)
324 (not (number? count))
325 vlp
326 (^v (display (padder (pop! v))) v)
327 (cond
328 ((equal? count #\*)
329 (cut list (next-token '() (list pad '*eof*))))
330 ((procedure? count)
331 (let ((count-unpacker (count 'unpacker)))
332 (lambda () (list (next-token-n* '() (list pad '*eof*) (get-count (count-unpacker)))))))
333 (else
334 (cut list (next-token-n* '() (list pad '*eof*) count))))
335 (make-skipper count))))
336
337
338 ;; set endianess of the number reader
339 (define (number-reader reader . opt-endian)
340 (if (pair? opt-endian)
341 (cute reader #f (car opt-endian))
342 reader))
343
344 ;; make a basic dispatcher for fixed size numeric types
345 (define (make-number-pack-dispatcher
346 base-reader base-writer size count vlp . opt-endian)
347 (let* ((splitter (get-splitter count))
348 (writer (if (pair? opt-endian)
349 (number-writer base-writer (car opt-endian))
350 (number-writer base-writer)))
351 (reader (if (pair? opt-endian)
352 (number-reader base-reader (car opt-endian))
353 (number-reader base-reader))))
354 (cond
355 ((eq? count #\*)
356 (make-pack-dispatch
357 0 #t vlp
358 (^v (for-each writer v) '())
359 (lambda ()
360 (let ((res '()))
361 (until (reader) eof-object? => x (push! res x))
362 (reverse res)))
363 (make-skipper #\*)))
364 ((procedure? count)
365 (let ((packer (count 'packer))
366 (unpacker (count 'unpacker)))
367 (make-pack-dispatch
368 0 #t vlp
369 (lambda (v)
370 (packer (length v))
371 (for-each writer v)
372 '())
373 (lambda ()
374 (let ((res '()))
375 (dotimes (i (get-count (unpacker)))
376 (push! res (reader)))
377 (concatenate (reverse res))))
378 (lambda ()
379 (let ((n (* size (get-count (unpacker)))))
380 (or (port-seek (current-input-port) n SEEK_CUR)
381 (read-block n)))))))
382 (else
383 (make-pack-dispatch
384 (* count (quotient size 8)) #f vlp
385 (lambda (v)
386 (let-values (((consume tail) (splitter v)))
387 (for-each writer consume)
388 tail))
389 (lambda ()
390 (let ((res '()))
391 (dotimes (i count)
392 (push! res (reader)))
393 (reverse res)))
394 (make-skipper (* count size)))))))
395
396 ;; not an isolated procedure, as it doesn't handle initializing the
397 ;; param to 0 on a pack/unpack/skip.
398 (define (make-count-update-pack-dispatch pack-dispatch param)
399 (let ((orig-pack (pack-dispatch 'packer))
400 (orig-unpack (pack-dispatch 'unpacker))
401 (orig-skip (pack-dispatch 'skipper))
402 (make-in-filter
403 (lambda (p)
404 (open-input-buffered-port
405 (lambda (size)
406 (let ((str (read-block size p)))
407 (param (+ (param) (string-size str)))
408 str))
409 1024)))
410 (make-out-filter
411 (lambda (p)
412 (open-output-buffered-port
413 (lambda (str)
414 (when str
415 (let ((size (string-size str)))
416 (param (+ (param) size))
417 (display str p)
418 size)))
419 1024)))
420 )
421 ;; make a new pack dispatch that updates the variable-length-count
422 ;; parameter on io
423 (make-pack-dispatch
424 (pack-dispatch 'length)
425 (pack-dispatch 'variable-length?)
426 param
427 (lambda (values)
428 (with-output-to-port (make-out-filter (current-output-port))
429 (lambda () (let ((v (orig-pack values))) (flush) v))))
430 (lambda ()
431 (with-input-from-port (make-in-filter (current-input-port))
432 (lambda () (orig-unpack))))
433 (lambda ()
434 (with-input-from-port (make-in-filter (current-input-port))
435 (lambda () (orig-skip)))) )))
436
437 ;; make a dispatch that repeats a variable number of times according to
438 ;; the slash prefix packer
439 (define (make-slash-repeat-pack-dispatch packer counter . opt-param)
440 (let ((orig-pack (packer 'packer))
441 (orig-unpack (packer 'unpacker))
442 (orig-skip (packer 'skipper))
443 (count-pack (counter 'packer))
444 (count-unpack (counter 'unpacker))
445 (param (get-optional opt-param #f)))
446 (make-pack-dispatch
447 (packer 'length)
448 (packer 'variable-length?)
449 param
450 (lambda (v)
451 (count-pack (list (length v)))
452 (for-each (^x (orig-pack (list x))) v)
453 '())
454 (lambda ()
455 (let ((count (get-count (count-unpack)))
456 (res '()))
457 (dotimes (i count)
458 (push! res (orig-unpack)))
459 (concatenate (reverse res))))
460 (lambda ()
461 (let ((count (get-count (count-unpack)))
462 (res '()))
463 (dotimes (i count) (orig-unpack))))
464 )))
465
466 ;; make a packer to repeat an arbitrary count times
467 (define (make-repeat-pack-dispatch packer count . opt-param)
468 (let ((param (get-optional opt-param #f))
469 (orig-pack (packer 'packer))
470 (orig-unpack (packer 'unpacker))
471 (orig-skip (packer 'skipper)))
472 (cond
473 ((procedure? count) (make-slash-repeat-pack-dispatch packer count param))
474 ((number? count)
475 (if (= count 1)
476 packer
477 (let ((splitter (get-splitter count)))
478 (make-pack-dispatch
479 (* count (packer 'length))
480 (packer 'variable-length?)
481 param
482 (lambda (v)
483 (let-values (((head tail) (splitter v)))
484 (for-each (^x (orig-pack (list x))) head)
485 tail))
486 (lambda ()
487 (let ((res '()))
488 (dotimes (i count)
489 (push! res (orig-unpack)))
490 (concatenate (reverse res))))
491 #f))))
492 ((eq? count #\*)
493 (let ((splitter (get-splitter count)))
494 (make-pack-dispatch
495 0
496 #t
497 param
498 (lambda (v)
499 (let-values (((head tail) (splitter v)))
500 (for-each (^x (orig-pack (list x))) head)
501 tail))
502 (lambda ()
503 (let ((res '()))
504 (let loop ((next (read)))
505 (unless (eof-object? next)
506 (push! res next)
507 (loop (read))))
508 (reverse res)))
509 #f)))
510 (else (error "count not supported:" count)))))
511
512 ;; merges the a and b dispatchers (a followed by b)
513 (define (merge-pack-dispatch a b)
514 (let* ((p1 (a 'packer))
515 (p2 (b 'packer))
516 (u1 (a 'unpacker))
517 (u2 (b 'unpacker))
518 (v1 (a 'variable-length?))
519 (v2 (b 'variable-length?))
520 (fixed? (not (or v1 v2)))
521 (l1 (a 'length))
522 (l2 (b 'length))
523 (l (+ l1 l2))
524 (s1 (a 'skipper))
525 (s2 (b 'skipper)))
526 (let ((p (^v (p2 (p1 v))))
527 (u (cut append (u1) (u2)))
528 (s (if fixed? (make-skipper l) (lambda () (begin (s1) (s2))))))
529 (make-pack-dispatch (+ l1 l2) (or v1 v2) (or (a 'var-len-param)
530 (b 'var-len-param))
531 p u s))))
532
533 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
534 ;; the pack specification
535
536 ;; A lot of redundancy here, this requires refactoring. could also
537 ;; easily make the char's dispatch on a runtime configurable hash table,
538 ;; but that doesn't seem too useful.
539 (define (read-one-packer fixed-len var-len? vlp slash-count)
540
541 (let loop ((c (read-char)))
542 (cond
543 ((eof-object? c)
544 (make-pack-token the-eof-object))
545
546 ;; end-of-list characters
547 ((member c '(#\] #\) #\} #\>))
548 (make-pack-token c))
549
550 (else
551 (let* ((bang (read-bang)) ;; optional !
552 (post-count (read-count))
553 (count (or slash-count post-count))
554 (count-unpack (if slash-count (slash-count 'unpacker) #f)))
555
556 (case c
557
558 ;; ignore whitespace
559 ((#\space #\tab #\newline)
560 (loop (read-char)))
561
562 ;; # indicates a line comment
563 ((#\#)
564 (skip-until #\newline)
565 (loop (read-char)))
566
567 ;; ( just groups until the closing )
568 ((#\()
569 (let* ((res (read-packers-until-token (char-complement c)
570 fixed-len var-len? vlp))
571 )
572 res))
573
574 ;; < produces a single list until >
575 ((#\<)
576 (let* ((res (read-packers-until-token (char-complement c)
577 fixed-len var-len? vlp))
578 (orig-pack (res 'packer))
579 (orig-unpack (res 'unpacker)))
580 (res 'set-packer!
581 (^v (orig-pack (car v)) (cdr v)))
582 (res 'set-unpacker!
583 (lambda () (list (orig-unpack))))
584 (let ((group-count (read-count)))
585 (make-repeat-pack-dispatch res group-count vlp))))
586
587 ;; a A string with arbitrary binary data, will be null padded.
588 ((#\a)
589 (let ((pad (get-string-padder count #\null)))
590 (make-pack-dispatch
591 (if (number? count) count 0)
592 (not (number? count))
593 vlp
594 (^v (display (pad (pop! v))) v)
595 (cond
596 ((eq? count #\*)
597 (cut list (port->string (current-input-port))))
598 ((procedure? count)
599 (lambda ()
600 (let* ((actual-count (get-count (count-unpack)))
601 (bstr (read-block actual-count))
602 (str (or (string-incomplete->complete bstr) bstr))
603 (size (string-size str)))
604 (list
605 (if (< size actual-count)
606 (string-append str (make-string (- actual-count size) #\null))
607 str)))))
608 (else
609 (lambda ()
610 (let* ((bstr (read-block count))
611 (str (or (string-incomplete->complete bstr) bstr))
612 (size (string-size str)))
613 (list
614 (if (< size count)
615 (string-append str (make-string (- count size) #\null))
616 str)))))
617 )
618 (make-skipper count)
619 )))
620
621 ;; A A text (ASCII) string, will be space padded.
622 ((#\A) (make-string-pack-dispatcher vlp count))
623
624 ;; Z A null terminated (ASCIZ) string, will be null padded.
625 ((#\Z)
626 (make-pack-dispatch
627 (if (number? count) count 0)
628 (not (number? count))
629 vlp
630 (cond
631 ((number? count)
632 (let ((pad (get-string-padder (- count 1) #\null)))
633 (^v (display (pad (pop! v))) (display #\null))))
634 (else
635 (let ((pad (get-string-padder count #\null)))
636 (^v (display (pad (pop! v))) (display #\null)))))
637 (cond
638 ((eq? count #\*)
639 (cut list (next-token '() '(#\null *eof*))))
640 ((procedure? count)
641 (let ((unpacker (count 'unpacker)))
642 (cut list (next-token-n* '() '(#\null *eof*) (get-count (unpacker))))))
643 (else
644 (cut list (next-token-n* '() '(#\null *eof*) count))))
645 (make-skipper count)
646 ))
647
648 ;; b A bit string (ascending bit order inside each byte, like vec()).
649 ((#\b)
650 (let ((pad (get-string-padder count #\0)))
651 (make-pack-dispatch
652 (if (number? count) (modulo (+ count 7) 8) 0)
653 (not (number? count))
654 vlp
655 (lambda (v)
656 (while-input-from-string (pad (pop! v))
657 (lambda ()
658 (do ((i 0 (+ i 1)) (acc 0))
659 ((= i 8) (write-byte acc))
660 ;; eof-object? and invalid bytes treated as #\0
661 (set! acc (copy-bit i acc (eq? #\1 (read-char)))) )))
662 v)
663 (make-port-byte-iterator
664 (^i (bit-for-each (cut write <>) i)) count)
665 (make-skipper count))))
666
667 ;; B A bit string (descending bit order inside each byte).
668 ((#\B)
669 (let ((pad (get-string-padder count #\0)))
670 (make-pack-dispatch
671 (if (number? count) (modulo (+ count 7) 8) 0)
672 (not (number? count))
673 vlp
674 (lambda (v)
675 (while-input-from-string (pad (pop! v))
676 (lambda ()
677 (do ((i 7 (- i 1)) (acc 0))
678 ((= i -1) (write-byte acc))
679 (set! acc (copy-bit i acc (eq? #\1 (read-char)))) )))
680 v)
681 (make-port-byte-iterator
682 (^i (bit-reverse-for-each (cut write <>) i)) count)
683 (make-skipper count))))
684
685 ;; h A hex string (low nybble first).
686 ((#\h)
687 (let ((pad (get-string-padder count #\0)))
688 (make-pack-dispatch
689 (if (number? count) count 0)
690 (not (number? count))
691 vlp
692 (lambda (v)
693 (while-input-from-string (pad (pop! v))
694 (lambda ()
695 (let* ((n1 (hex-char->number (read-byte)))
696 (n2 (hex-char->number (read-byte))))
697 (write-byte (copy-bit-field n1 n2 4 8)))))
698 v)
699 (make-port-byte-iterator
700 (^i (format #t "~x~x" (logand i #b1111) (ash i -4))) count)
701 (make-skipper count))))
702
703 ;; H A hex string (high nybble first).
704 ((#\H)
705 (let ((pad (get-string-padder count #\0)))
706 (make-pack-dispatch
707 (if (number? count) count 0)
708 (not (number? count))
709 vlp
710 (lambda (v)
711 (while-input-from-string (pad (pop! v))
712 (lambda ()
713 (let* ((n1 (hex-char->number (read-byte)))
714 (n2 (hex-char->number (read-byte))))
715 (write-byte (copy-bit-field n2 n1 4 8)))))
716 v)
717 (make-port-byte-iterator
718 (^i (format #t "~x~x" (ash i -4) (logand i #b1111))) count)
719 (make-skipper count))))
720
721 ;; c A signed char value.
722 ((#\c)
723 (make-number-pack-dispatcher read-binary-sint8 write-binary-sint8
724 8 count vlp))
725
726 ;; C An unsigned char value. Only does bytes. See U for Unicode.
727 ((#\C)
728 (make-number-pack-dispatcher read-binary-uint8 write-binary-uint8
729 8 count vlp))
730
731 ;; s A signed short value.
732 ((#\s)
733 (make-number-pack-dispatcher
734 (if bang read-binary-short read-binary-sint16)
735 (if bang write-binary-short write-binary-sint16)
736 16 count vlp))
737
738 ;; S An unsigned short value.
739 ((#\S)
740 (make-number-pack-dispatcher
741 (if bang read-binary-ushort read-binary-uint16)
742 (if bang write-binary-ushort write-binary-uint16)
743 16 count vlp))
744
745 ;; i A signed integer value.
746 ;; l A signed long value.
747 ((#\i #\l)
748 (make-number-pack-dispatcher
749 (if bang read-binary-long read-binary-sint32)
750 (if bang write-binary-long write-binary-sint32)
751 32 count vlp))
752
753 ;; I An unsigned integer value.
754 ;; L An unsigned long value.
755 ((#\I #\L)
756 (make-number-pack-dispatcher
757 (if bang read-binary-ulong read-binary-uint32)
758 (if bang write-binary-ulong write-binary-uint32)
759 32 count vlp))
760
761 ;; n An unsigned short in "network" (big-endian) order.
762 ((#\n)
763 (make-number-pack-dispatcher
764 (if bang read-binary-sint16 read-binary-uint16)
765 (if bang write-binary-sint16 write-binary-uint16)
766 16 count vlp 'big-endian))
767
768 ;; N An unsigned long in "network" (big-endian) order.
769 ((#\N)
770 (make-number-pack-dispatcher
771 (if bang read-binary-sint32 read-binary-uint32)
772 (if bang write-binary-sint32 write-binary-uint32)
773 32 count vlp 'big-endian))
774
775 ;; v An unsigned short in "VAX" (little-endian) order.
776 ((#\v)
777 (make-number-pack-dispatcher
778 (if bang read-binary-sint16 read-binary-uint16)
779 (if bang write-binary-sint16 write-binary-uint16)
780 16 count vlp 'little-endian))
781
782 ;; V An unsigned long in "VAX" (little-endian) order.
783 ((#\V)
784 (make-number-pack-dispatcher
785 (if bang read-binary-sint32 read-binary-uint32)
786 (if bang write-binary-sint32 write-binary-uint32)
787 32 count vlp 'little-endian))
788
789 ;; q A signed quad (64-bit) value.
790 ((#\q)
791 (make-number-pack-dispatcher
792 read-binary-sint64 write-binary-sint64 64 count vlp))
793
794 ;; Q An unsigned quad value.
795 ((#\Q)
796 (make-number-pack-dispatcher
797 read-binary-uint64 write-binary-uint64 64 count vlp))
798
799 ;; f A single-precision float in the native format.
800 ((#\f)
801 (make-number-pack-dispatcher
802 read-binary-float write-binary-float 32 count vlp))
803
804 ;; d A double-precision float in the native format.
805 ((#\d)
806 (make-number-pack-dispatcher
807 read-binary-double write-binary-double 64 count vlp))
808
809 ;; w A BER compressed integer.
810 ((#\w)
811 (let ((splitter (get-splitter count 0)))
812 (make-pack-dispatch
813 0 #t vlp
814 (lambda (v)
815 (let-values (((head tail) (splitter v)))
816 (for-each (cut write-ber-integer <>) head)
817 tail))
818 (if (eq? count #\*)
819 (lambda ()
820 (let ((res '()))
821 (let loop ((next (read-ber-integer)))
822 (unless (eof-object? next)
823 (push! res next)
824 (loop (read-ber-integer))))
825 (reverse res)))
826 (lambda ()
827 (let ((res '()))
828 (dotimes (i count)
829 (push! res (read-ber-integer)))
830 (reverse res))))
831 (if (eq? count #\*)
832 ;; anything is a valid sequence of ber integers so we
833 ;; can safely skip to the end at this point
834 (make-skipper count)
835 (lambda ()
836 (dotimes (i count)
837 (let loop ((c (read-byte)))
838 (unless (> c 128) (loop (read-byte))))))))))
839
840 ;; o An sexp.
841 ((#\o)
842 (let ((splitter (get-splitter count 0)))
843 (make-pack-dispatch
844 0 #t vlp
845 (lambda (v)
846 (let-values (((head tail) (splitter v)))
847 (for-each (cut write <>) head)
848 tail))
849 (cond
850 ((eq? count #\*)
851 (lambda ()
852 (let ((res '()))
853 (let loop ((next (read)))
854 (unless (eof-object? next)
855 (push! res next)
856 (loop (read))))
857 (reverse res))))
858 ((procedure? count)
859 (let ((unpacker (count 'unpacker)))
860 (lambda ()
861 (let ((res '()))
862 (dotimes (i (get-count (unpacker)))
863 (push! res (read)))
864 (reverse res)))))
865 (else
866 (lambda ()
867 (let ((res '()))
868 (dotimes (i count)
869 (push! res (read)))
870 (reverse res)))))
871 )))
872
873 ;; x A null byte.
874 ((#\x)
875 (let ((skipper (make-skipper count)))
876 (make-pack-dispatch
877 (if (number? count) count 0)
878 (not (number? count))
879 vlp
880 (lambda (v)
881 (unless (eq? count #\*)
882 (display (make-string count #\null)))
883 v)
884 ;; unpack means skip but return '() to append as nothing
885 (lambda () (skipper) '())
886 skipper)))
887
888 ;; @ Null fill to absolute position.
889 ((#\@)
890 (let ((var-len-param (or vlp (make-parameter 0)))
891 (var-count (- count fixed-len)))
892 (make-pack-dispatch
893 ;;(if (number? count) count 0)
894 0
895 ;;(not (number? count))
896 #f
897 var-len-param
898 (lambda (v)
899 (let ((diff (- var-count (var-len-param))))
900 (display (make-string diff #\null))
901 (var-len-param (+ (var-len-param) diff))
902 )
903 v)
904 (lambda ()
905 (let ((diff (- var-count (var-len-param))))
906 (var-len-param (+ (var-len-param) diff))
907 (or (port-seek (current-input-port) diff SEEK_CUR)
908 (read-block diff))
909 '()))
910 #f)))
911
912 ;; j A signed integer value (a Perl internal integer, IV).
913 ;; J An unsigned integer value (a Perl internal unsigned integer, UV).
914 ((#\j #\J #\F #\D)
915 (error "sorry, this isn't Perl, IV, UV and NV not supported"))
916
917 (else
918 (error "unknown pack template character: ~S" c)))
919
920 )))))
921
922 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
923 ;; high level parser
924
925 ;; read-packer that handles / formats
926 (define (read-packer fixed-len var-len? vlp)
927 (let ((p (read-one-packer fixed-len var-len? vlp #f)))
928 (skip-pack-comments)
929 (let ((c (peek-char)))
930 ;; handle the / mechanism a little more generally than Perl
931 (cond ((eq? c #\/)
932 (read-char) ;; eat the /
933 (skip-pack-comments) ;; scout ahead
934 (if (memq (peek-char) '(#\a #\A #\Z #\b #\B #\h #\H))
935 ;; make a string-type packer with dynamic length
936 (read-one-packer fixed-len var-len? vlp p)
937 ;; make an object-type dynamic repeat packer
938 (make-slash-repeat-pack-dispatch
939 (read-one-packer fixed-len var-len? vlp #f) p vlp)))
940 (else
941 ;; not a /, return the initial packer
942 p)))))
943
944 (define (read-until-token tok fixed-len var-len? var-len-param)
945 (let loop ((ls '()) (flen fixed-len) (vlen? var-len?) (vlp var-len-param))
946 (let ((next (read-packer flen vlen? vlp)))
947 (cond ((equal? next tok)
948 (reverse ls))
949 ((eof-object? next)
950 (error "encountered eof while expecting ~S" tok))
951 ((procedure? next)
952 (loop (cons next ls)
953 (+ flen (next 'length))
954 (or vlen? (next 'variable-length?))
955 (or vlp (next 'var-len-param))))
956 (else
957 (error "packer ~S not recognized" next))))))
958
959 (define (pack-merge-folder a b)
960 (if (not b)
961 a
962 (merge-pack-dispatch b a)))
963
964 (define (read-packers-until-token token :optional (fixed-len 0) (var-len? #f)
965 (vlp #f))
966 (let ((packers (read-until-token (make-pack-token token)
967 fixed-len var-len? vlp)))
968 ;; update var-len fields if we have an @
969 ;; with make-count-update-pack-dispatch
970 (let loop ((ls packers)
971 (res '()))
972 (if (null? ls)
973 (fold pack-merge-folder #f (reverse res))
974 (let ((a (car ls))
975 (rest (cdr ls)))
976 (if (null? rest)
977 (loop rest (cons a res))
978 (let ((b (car rest)))
979 (if (and (not (a 'var-len-param))
980 (b 'var-len-param))
981 (let ((b-vlp (b 'var-len-param)))
982 (loop rest
983 (list
984 (modify-pack-dispatch
985 (fold pack-merge-folder #f
986 (reverse
987 (map (lambda (p)
988 (if (p 'variable-length?)
989 (make-count-update-pack-dispatch p (b 'var-len-param))
990 p))
991 (cons a res))))
992 'packer (lambda (orig) (^v (b-vlp 0) (orig v)))
993 'unpacker (lambda (orig) (lambda () (b-vlp 0) (orig)))
994 ))))
995 (loop rest (cons a res))))))))))
996
997 (define (read-all-packers template)
998 (with-input-from-string template
999 (lambda ()
1000 (read-packers-until-token the-eof-object))))
1001
1002 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1003 ;; exported interface
1004
1005 ;; making a parameter for thread safety, maybe better shared between
1006 ;; threads with a mutex.
1007 (define make-packer
1008 (let ((cache (make-parameter (make-hash-table 'equal?))))
1009 (lambda (template :optional (cached? #t))
1010 (if cached?
1011 (let ((res (hash-table-get (cache) template #f)))
1012 (unless res
1013 (set! res (read-all-packers template))
1014 (hash-table-put! (cache) template res))
1015 res)
1016 (read-all-packers template)))))
1017
1018 (define (pack template values :key (output #f) (to-string? #f) (cached? #t))
1019 (let ((packer (make-packer template cached?))
1020 (out (or output
1021 (and to-string? (open-output-string))
1022 (current-output-port))))
1023 (with-output-to-port out
1024 (lambda ()
1025 (let ((res (packer 'pack values)))
1026 (if (pair? res)
1027 (error "pack: extra values remaining: ~S" res)
1028 (if to-string?
1029 (get-output-string out)
1030 #t)))))))
1031
1032 (define (get-input-port keys)
1033 (let-keywords keys ((input #f)
1034 (from-string #f))
1035 (or input
1036 (and from-string (open-input-string from-string))
1037 (current-input-port))))
1038
1039 (define (unpack template :key (cached? #t) :allow-other-keys rest)
1040 (let ((packer (make-packer template cached?))
1041 (in (get-input-port rest)))
1042 (with-input-from-port in
1043 (cut packer 'unpack))))
1044
1045 ;; just "skip" is too vague
1046 (define (unpack-skip template :key (cached? #t) :allow-other-keys rest)
1047 (let ((packer (make-packer template cached?))
1048 (in (get-input-port rest)))
1049 (with-input-from-port in
1050 (cut packer 'skip))))