Debian Codegraph @

ext/peg/test.scm

raw
1 ;;;
2 ;;; test/peg.scm - Unit Tests for PEG module
3 ;;;
4 ;;; Copyright (c) 2006 Rui Ueyama ([email protected])
5 ;;;
6 ;;; Redistribution and use in source and binary forms, with or without
7 ;;; modification, are permitted provided that the following conditions
8 ;;; are met:
9 ;;;
10 ;;; 1. Redistributions of source code must retain the above copyright
11 ;;; notice, this list of conditions and the following disclaimer.
12 ;;;
13 ;;; 2. Redistributions in binary form must reproduce the above copyright
14 ;;; notice, this list of conditions and the following disclaimer in the
15 ;;; documentation and/or other materials provided with the distribution.
16 ;;;
17 ;;; 3. Neither the name of the authors nor the names of its contributors
18 ;;; may be used to endorse or promote products derived from this
19 ;;; software without specific prior written permission.
20 ;;;
21 ;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 ;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 ;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 ;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 ;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 ;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
27 ;;; TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
28 ;;; PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
29 ;;; LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
30 ;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
31 ;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 ;;;
33
34 (use gauche.test)
35 (use scheme.list)
36 (use srfi.13)
37 (use gauche.generator)
38 (use gauche.lazy)
39
40 (test-start "parser.peg")
41
42 (test-section "peg")
43 (use parser.peg)
44 (test-module 'parser.peg)
45
46 ;;;============================================================
47 ;;; helper macros
48 ;;;
49 (define (%compare test-type accessors)
50 (^[e o] (and (test-type o)
51 (every equal? e (map (cut <> o) accessors)))))
52
53 (define-syntax test-succ
54 (syntax-rules ()
55 [(_ label expect parse input)
56 (test* #"~label (success)" expect
57 (peg-parse-string parse input))]))
58
59 (define-syntax test-fail
60 (syntax-rules ()
61 [(_ label expect parse input)
62 (test* #"~label (failure)" expect
63 (guard (e [(<parse-error> e)
64 (list (ref e 'position) (ref e 'objects))]
65 [else e])
66 (peg-parse-string parse input)
67 (error "test-fail failed")))]))
68
69 ;;;============================================================
70 ;;; Ropes
71 ;;;
72 (receive (make-rope rope->string)
73 (with-module parser.peg (values make-rope rope->string))
74 (let* ([rope1 (make-rope "abc")]
75 [rope2 (make-rope `("012" ,rope1 #\X))]
76 [rope3 (make-rope 'foo)])
77 (test* "rope->string" "abc" (rope->string rope1))
78 (test* "rope->string" "012abcX" (rope->string rope2))
79 (test* "rope->string" (test-error) (rope->string rope3))))
80
81
82 ;;;============================================================
83 ;;; Primitives
84 ;;;
85 (test-section "primitives")
86
87 (test-succ "$return" "xyz" ($return "xyz") "abc")
88 (test-fail "$fail" '(0 "error") ($fail "error") "foo")
89 (test-fail "$expect" '(0 "character")
90 ($expect ($fail "zzz") "character") "foo")
91
92 ;;;============================================================
93 ;;; Error handlers
94 ;;;
95 ; (test-section "error handling")
96
97 ; (let1 merge-failure (with-module peg merge-failure)
98 ; (test* "merge-failure"
99 ; '#(fail expect ("a") 1)
100 ; (merge-failure '(#(fail expect ("a") 1)
101 ; #(fail expect ("b") 0))))
102 ; (test* "merge-failure (same position)"
103 ; '#(fail expect ("a" "b") 1)
104 ; (merge-failure '(#(fail expect ("a") 1)
105 ; #(fail expect ("b") 1)))))
106
107 ;;;============================================================
108 ;;; Parsers for string
109 ;;;
110 (test-section "string")
111
112 (test-succ "$string" "abc" ($string "abc") "abcdef")
113 (test-fail "$string)" '(0 "ABC") ($string "ABC") "012")
114 (test-succ "$char" #\a ($char #\a) "abc")
115 (test-fail "$char" '(0 #\a) ($char #\a) "012")
116 (test-succ "$one-of" #\j ($one-of #[a-z]) "j")
117 (test-fail "$one-of" '(0 #[A-Z]) ($one-of #[A-Z]) "j")
118 (test-succ "$one-of" "abd" ($one-of '("abc" "abd" "acd")) "abd")
119 (test-succ "$one-of" #\c ($one-of '(#\a #\b #\c)) "c")
120 (test-succ "$one-of" #\c ($one-of '("ab" #\c)) "c")
121 (test-fail "$one-of" '(0 (#\a #\b #\c)) ($one-of '(#\a #\b #\c)) "z")
122 (test-fail "$one-of" '(0 ("abc" "abd" "acd"))
123 ($one-of '("abc" "abd" "acd")) "a")
124 (let ((var #f))
125 (set! var '("a" "b" "c")) ;suppress optimization
126 (test-succ "$one-of" "b" ($one-of var) "b"))
127 (test-succ "$none-of" #\j ($none-of #[A-Z]) "j")
128 (test-succ "$string-ci" "aBC" ($string-ci "abc") "aBCdef")
129 (test-fail "$string-ci" '(0 "abc") ($string-ci "abc") "012")
130
131 ;;;============================================================
132 ;;; Combinators
133 ;;;
134 (test-section "combinators")
135
136 (define (map-tree proc tree)
137 (if (pair? tree)
138 (cons (map-tree proc (car tree))
139 (map-tree proc (cdr tree)))
140 (proc tree)))
141
142 ;; $or
143 (test-succ "$or" "foo"
144 ($or ($string "foo") ($string "bar"))
145 "foo...")
146
147 (test-succ "$or" "bar"
148 ($or ($string "foo") ($string "bar"))
149 "bar...")
150
151 (test-fail "$or" '(0 ((fail-expect . "foo") (fail-expect . "bar")))
152 ($or ($string "foo") ($string "bar"))
153 "012345")
154
155 (test-fail "$or" '(0 "bar")
156 ($or ($string "foo") :else ($string "bar"))
157 "012345")
158
159 ;; $try
160 (test-succ "$try" "foo"
161 ($try ($string "foo"))
162 "foo...")
163 (test-succ "$try" "bar"
164 ($try ($seq ($string "foo") ($string "bar")))
165 "foobar...")
166 (test-fail "$try" '(0 "foo")
167 ($try ($string "foo"))
168 "bar...")
169 (test-fail "$try" '(0 "bar")
170 ($try ($seq ($string "foo") ($string "bar")))
171 "foobaz...")
172
173 ;; $cut and $raise
174 (test-fail "$cut" '(0 (error (fail-expect . "a")))
175 ($or ($cut ($. "a")) ($. "b"))
176 "b")
177 (test-succ "$cut" "a"
178 ($or ($cut ($. "a")) ($. "b"))
179 "a")
180
181 (test-fail "$cut" '(1 (error (fail-expect . "b")))
182 ($or ($try ($seq ($. "a") ($cut ($. "b")))) ($. "a"))
183 "ac")
184 (test-succ "$cut" "b"
185 ($or ($try ($seq ($. "a") ($cut ($. "b")))) ($. "a"))
186 "ab")
187
188 (test-fail "$raise" '(1 (error (fail-message . "oops")))
189 ($or ($seq ($. "a") ($raise "oops")) ($. "a"))
190 "a")
191 (test-fail "$raise" '(1 (error (fail-message . "oops")))
192 ($or ($try ($seq ($. "a") ($raise "oops"))) ($. "a"))
193 "a")
194
195 ;; $let, $let*
196 (test-succ "$let" "j"
197 ($let ([j ($string "j")])
198 ($return j))
199 "j")
200
201 (test-succ "$let" "j"
202 (let1 parse ($string "j")
203 ($let ([j parse]) ($return j)))
204 "j")
205
206 (test-succ "$let" '("foo" "bar")
207 ($let ([foo ($string "foo")]
208 [bar ($string "bar")])
209 ($return (list foo bar)))
210 "foobar...")
211
212 (test-succ "$let" '("foo" "baz")
213 ($let ([foo ($string "foo")]
214 [ ($string "bar")]
215 [baz ($string "baz")])
216 ($return (list foo baz)))
217 "foobarbaz$$$")
218
219 (test-succ "$let" "foo"
220 (let1 dot ($string ".")
221 ($let ([foo ($string "foo")]
222 dot)
223 ($return foo)))
224 "foo.")
225
226 (test-fail "$let" '(3 "bar")
227 ($let ([foo ($string "foo")]
228 [bar ($string "bar")])
229 ($return (list foo bar)))
230 "foo12")
231
232 (test-succ "$let" '("AB" "cd" "EF")
233 ($let ([x ($or ($string "AB") ($string "ab"))]
234 [y ($or ($string "CD") ($string "cd"))]
235 [z ($or ($string "EF") ($string "ef"))])
236 ($return (list x y z)))
237 "ABcdEF")
238
239 (test-fail "$let and $or" '(3 "foo")
240 ($let ([v ($or ($string "foo") ($string "bar"))])
241 ($string (rope-finalize v)))
242 "foobar")
243
244 (test-succ "$let" '("foo" "baz")
245 ($let ([foo ($string "foo")]
246 [ ($string "bar")]
247 [baz ($string "baz")])
248 ($return (list foo baz)))
249 "foobarbaz$$$")
250
251 (test-succ "$let" '(#\0 "zyx" #\0)
252 ($let* ([opener ($one-of #[\d])]
253 [meat ($->string ($many ($one-of #[a-z])))]
254 [closer ($char opener)])
255 ($return (list opener meat closer)))
256 "0zyx0")
257
258 ;; $parameterize
259 (define param1 (make-parameter "zzz"))
260 (define match-param1 ($let ([word ($->string ($many ($one-of #[a-z])))])
261 (if (string-contains word (param1))
262 ($fail #"word can't have '~(param1)'")
263 ($return word))))
264 (test-succ "$parameterize" "lineament"
265 ($parameterize ((param1 "cicle"))
266 match-param1)
267 "lineament")
268 (test-fail "$parameterize" '(9 "word can't have 'line'")
269 ($parameterize ((param1 "line"))
270 match-param1)
271 "lineament")
272
273 ;; $lift and $lift*
274 ;; NB: They are inline expanded, so we also call via apply
275 (let ()
276 (define-syntax test-lift
277 (syntax-rules ()
278 [(test-lift tester name expect (exp ...) input)
279 (begin (tester name expect (exp ...) input)
280 (tester #"~|name| via apply"
281 expect (apply exp ... '()) input))]))
282
283 (test-lift test-succ "$lift" '(#\a . #\b)
284 ($lift cons ($any) ($any))
285 "abc")
286 (test-lift test-fail "$lift" '(1 #\z)
287 ($lift cons ($any) ($char #\z))
288 "abc")
289 (test-lift test-succ "$lift*" "abc"
290 ($lift* list->string ($any) ($any) ($any))
291 "abc")
292 (test-lift test-fail "$lift" '(2 #\z)
293 ($lift* list->string ($any) ($any) ($char #\z))
294 "abc"))
295
296 ;; $fold-parsers and $fold-parsers-right
297 (test-succ "$fold-parsers" '() ; base case
298 ($fold-parsers cons '() '())
299 "abc")
300 (test-succ "$fold-parsers" '(#\c #\b #\a)
301 ($fold-parsers cons '() (list ($char #\a) ($char #\b) ($char #\c)))
302 "abc")
303 (test-fail "$fold-parsers" '(2 #\c)
304 ($fold-parsers cons '() (list ($char #\a) ($char #\b) ($char #\c)))
305 "abd")
306
307 (test-succ "$fold-parsers-right" '() ; base case
308 ($fold-parsers-right cons '() '())
309 "abc")
310 (test-succ "$fold-parsers-right" '(#\a #\b #\c)
311 ($fold-parsers-right cons '() (list ($char #\a) ($char #\b) ($char #\c)))
312 "abc")
313 (test-fail "$fold-parsers-right" '(2 #\c)
314 ($fold-parsers-right cons '() (list ($char #\a) ($char #\b) ($char #\c)))
315 "abd")
316
317 ;; $seq
318 (test-succ "$seq" "b"
319 ($seq ($string "a") ($string "b"))
320 "abcd")
321 (test-succ "$seq" "b"
322 ($seq ($or ($string "A") ($string "a"))
323 ($string "b"))
324 "abcd")
325 (test-fail "$seq" '(1 "b")
326 ($seq ($string "a") ($string "b"))
327 "a123")
328
329 ;; $many
330 (test-succ "$many" '("a" "b" "a" "a" "b")
331 ($many ($or ($string "a") ($string "b")))
332 "abaabcd")
333 (test-succ "$many" '()
334 ($many ($string "a")) "012")
335 (test-fail "$many" '(0 "a")
336 ($many ($string "a") 1) "012")
337 (test-succ "$many" '("a" "a")
338 ($many ($string "a") 1 2) "aaaaa")
339
340 ;; $many_
341 (test-succ "$many_" #\a
342 ($seq ($many_ ($string "a") 1 2) ($one-of #[a-z])) "aaaaa")
343 (test-succ "$many_" #\b
344 ($seq ($many_ ($string "a")) ($one-of #[a-z])) "baaaaa")
345
346 ;; $repeat
347 (test-succ "$repeat" '(#\a #\b #\a)
348 ($repeat ($any) 3) "abab")
349 (test-fail "$repeat" '(2 "end of input")
350 ($repeat ($any) 3) "ab")
351
352 ;; $optional
353 (test-succ "$optional" #\a
354 ($optional ($char #\a)) "abab")
355 (test-succ "$optional" #f
356 ($optional ($char #\a)) "ABAB")
357 (test-fail "$optional" '(1 #\b)
358 ($optional ($seq ($char #\a) ($char #\b)))
359 "ac")
360
361 ;; $sep-by
362 (test-succ "$sep-by" '(#\a #\b)
363 ($sep-by ($one-of #[ab]) ($string ","))
364 "a,b")
365
366 (test-fail "$sep-by" '(2 #[ab])
367 ($sep-by ($one-of #[ab]) ($string ","))
368 "a,c")
369
370 (test-succ "$sep-by" '(#\a #\b)
371 ($sep-by ($one-of #[abc]) ($string ",") 0 2)
372 "a,b,c,d")
373
374 (test-fail "$sep-by" '(2 #[abc])
375 ($sep-by ($one-of #[abc]) ($string ",") 2 3)
376 "a,2,3")
377
378 ;; $end-by
379 (test-succ "$end-by" '(#\a #\b)
380 ($end-by ($one-of #[a-z]) ($string ","))
381 "a,b,")
382
383 ;; $sep-end-by
384 (let1 p ($sep-end-by ($seq ($one-of #[a-z]) ($one-of #[a-z])) ($string ","))
385 (define (succ in exp) (test-succ "$sep-end-by" exp p in))
386 (define (fail in exp) (test-fail "$sep-end-by" exp p in))
387
388 (succ "ZZ" '())
389 (succ "" '())
390 (fail "aZ" '(1 #[a-z]))
391 (succ "aa" '(#\a))
392 (succ "bb," '(#\b))
393 (succ "cc,Z" '(#\c))
394 (fail "cc,dZ" '(4 #[a-z]))
395 (succ "ee,ff," '(#\e #\f))
396 (succ "ggZ" '(#\g))
397 (succ "hh,," '(#\h))
398 (succ "ii,jjZ" '(#\i #\j))
399 (fail "kk,ll,mZ" '(7 #[a-z]))
400
401 (test-succ "$sep-end-by (min)" '(#\a #\b)
402 ($sep-end-by ($seq ($one-of #[a-z]) ($one-of #[a-z]))
403 ($string ",") 2)
404 "aa,bb")
405 (test-succ "$sep-end-by (min)" '(#\a #\b)
406 ($sep-end-by ($seq ($one-of #[a-z]) ($one-of #[a-z]))
407 ($string ",") 2)
408 "aa,bb,")
409 (test-fail "$sep-end-by (min)" '(5 ",")
410 ($sep-end-by ($seq ($one-of #[a-z]) ($one-of #[a-z]))
411 ($string ",") 3)
412 "aa,bb")
413 (test-fail "$sep-end-by (min)" '(6 #[a-z])
414 ($sep-end-by ($seq ($one-of #[a-z]) ($one-of #[a-z]))
415 ($string ",") 3)
416 "aa,bb,")
417 )
418
419 ;; $between
420 (test-succ "$between" "foo"
421 ($between ($string "(")
422 ($string "foo")
423 ($string ")"))
424 "(foo)")
425
426 (test-fail "$between" '(4 ")")
427 ($between ($string "(")
428 ($string "foo")
429 ($string ")"))
430 "(foo]")
431
432 ;; $not
433 (test-succ "$not" #f
434 ($not ($one-of #[a-z]))
435 "ABC")
436 (test-fail "$not" '(0 #\a)
437 ($not ($one-of #[a-z]))
438 "abc")
439
440 ;; $many-till
441 (test-succ "$many-till" '(#\a #\b)
442 ($many-till ($one-of #[[:alnum:]]) ($one-of #[\d]))
443 "ab78")
444
445 ;; $chain-left
446 (let ([integer
447 ($let ([v ($many ($one-of #[\d]) 1)])
448 ($return (string->number (apply string v))))]
449 [op
450 ($or ($seq ($char #\*) ($return *))
451 ($seq ($char #\+) ($return +)))])
452 (test-succ "$chain-left" 9
453 ($chain-left integer op)
454 "1+2*3")
455 (test-succ "$chain-left" 3
456 ($chain-left integer op)
457 "1+2*")
458 (test-fail "$chain-left" '(0 #[0-9])
459 ($chain-left integer op)
460 "abc"))
461
462 ;; $chain-right
463 (let ([integer
464 ($let ([v ($many ($one-of #[\d]) 1)])
465 ($return (string->number (apply string v))))]
466 [op
467 ($or ($seq ($char #\*) ($return *))
468 ($seq ($char #\+) ($return +)))])
469 (test-succ "$chain-right" 7
470 ($chain-right integer op)
471 "1+2*3")
472 (test-succ "$chain-right" 3
473 ($chain-right integer op)
474 "1+2*")
475 (test-fail "$chain-right" '(0 #[0-9])
476 ($chain-right integer op)
477 "abc"))
478
479 ;; $lazy
480 (test-succ "$lazy" #\a
481 ($lazy ($char #\a)) "a")
482 (test-fail "$lazy" '(0 #\a)
483 ($lazy ($char #\a)) "b")
484
485 ;;;============================================================
486 ;;; Backtrack control
487 ;;;
488 (test-section "backtrack control")
489
490 (test-succ "$or and $try" "abc"
491 ($or ($try ($list ($string "abc") ($string "foo")))
492 ($string "abc"))
493 "abcdefg")
494
495 (let1 parser ($or ($let ([foo ($try ($let ([v ($string "abc")]
496 [($one-of #[+-])])
497 ($return v)))]
498 [bar ($string "foo")])
499 ($return (list foo bar)))
500 ($string "abc"))
501 (test-succ "$or and $try" "abc" parser
502 "abcdefg")
503 (test-fail "$or and $try" '(4 "foo") parser
504 "abc+efg")
505 )
506
507 ;;;============================================================
508 ;;; Lookahead assertion
509 ;;;
510 (test-section "lookahead assertion")
511
512 (let ([parser ($seq0
513 ($->rope ($many
514 ($try ($seq0 ($any)
515 ($assert ($. #[a-z]))))))
516 ($any))])
517 (test-succ "positive lookahead" " abc" parser
518 " abcd ")
519 )
520
521 (let ([parser1 ($between ($. "/*")
522 ($->rope ($many ($seq ($not ($. "*/"))
523 ($any))))
524 ($. "*/"))])
525 (test-succ "negative lookahead 1" " abc * def " parser1
526 "/* abc * def */ def")
527 (test-fail "negative lookahead 1" '(12 "*/") parser1
528 "/* abc * def")
529 )
530
531
532 ;;;============================================================
533 ;;; Error handling
534 ;;;
535
536 (test-section "error handling")
537
538 ;; compound error
539
540 (let* ([p1 ($or ($char #\a) ($char #\b))]
541 [p2 ($seq ($any) ($or ($char #\c) p1))])
542 (test* "nested compound error"
543 (test-error <parse-error> "expecting one of (#\\c #\\a #\\b) at 1")
544 (peg-parse-string p2 "dd")))
545
546 ;; position
547 (test* "error position handling"
548 (test-error <parse-error>
549 "expecting #[\\u000aa] at \"input.txt\":2:3, but got #\\b")
550 (peg-run-parser ($many ($. #[a\n]) 10)
551 (port->char-lseq/position
552 (open-input-string "aaaaa\naabaa")
553 :source-name "input.txt")))
554
555 (test* "error position handling (<parse-error> slots)"
556 #t
557 (guard (e [(<parse-error> e) (sequence-position? (~ e'position))])
558 (peg-run-parser ($many ($. #[a\n]) 10)
559 (port->char-lseq/position
560 (open-input-string "aaaaa\naabaa")
561 :source-name "input.txt"))))
562
563 ;;;============================================================
564 ;;; Optional arg for peg-parse-string and peg-parse-port
565 ;;;
566
567 (test-section "optional cont arg")
568
569 (test* "peg-parse-string cont" '("aaaa" #\b #\b #\c)
570 (peg-parse-string ($->rope ($many ($. #\a))) "aaaabbc" cons))
571 (test* "peg-parse-port cont" '("aaaa" #\b #\b #\c)
572 (call-with-input-string "aaaabbc"
573 (cut peg-parse-port ($->rope ($many ($. #\a))) <> cons)))
574
575 ;;;============================================================
576 ;;; Token parsing
577 ;;;
578
579 (test-section "token parsing")
580
581 (let ()
582 (define %a ($many1 ($. 'a)))
583 (define %bcd ($many1 ($one-of '(b c d))))
584 (define %p ($many ($list %a %bcd)))
585
586 (test* "token parsing"
587 '(((a) (b c)) ((a) (d)) ((a a) (b)) ((a) (c d c)))
588 (peg-run-parser %p '(a b c a d a a b a c d c)))
589 )
590
591
592 ;;;============================================================
593 ;;; Examples
594 ;;;
595 (test-section "examples")
596
597 ;; PEG walkthrough
598 (test* "walkthrough char" #\a
599 (peg-parse-string ($. #\a) "abc"))
600 (test* "walkthrough error" (test-error <parse-error>
601 "expecting #\\a at 0, but got #\\x")
602 (peg-parse-string ($char #\a) "xyz"))
603 (test* "walkthrough $many" '(#\a #\a #\a #\a #\a)
604 (peg-parse-string ($many ($. #\a)) "aaaaabc"))
605 (test* "walkthrough $many" '()
606 (peg-parse-string ($many ($. #\a)) "xxxxxyz"))
607
608 (let ()
609 (define digits ($many1 ($. #[\d])))
610 (define ws ($many_ ($. #[\s])))
611 (define separator ($seq ws ($. #\,) ws))
612 (define integer
613 ($let ([ds digits])
614 ($return (x->integer (list->string ds)))))
615 (define integers1
616 ($seq integer ($many ($seq separator integer))))
617 (define integers2
618 ($let ([n integer]
619 [ns ($many ($seq separator integer))])
620 ($return (cons n ns))))
621 (define integers3
622 ($lift cons
623 integer
624 ($many ($seq separator integer))))
625 (define integers4
626 ($or ($let ([n integer]
627 [ns ($many ($seq separator integer))])
628 ($return (cons n ns)))
629 ($return '())))
630 (define (sep-by stuff separator)
631 ($or ($let ([n stuff]
632 [ns ($many ($seq separator stuff))])
633 ($return (cons n ns)))
634 ($return '())))
635 (define integers5 (sep-by integer separator))
636 (define begin-list
637 ($seq0 ($. #[\(\[\{]) ws))
638 (define (end-list opener)
639 ($seq ws (case opener
640 [(#\() ($. #\))]
641 [(#\[) ($. #\])]
642 [(#\{) ($. #\})])))
643
644 (define int-list
645 ($let* ([opener begin-list]
646 [ints ($sep-by integer separator)]
647 [ (end-list opener) ])
648 ($return ints)))
649
650 (define nested-list
651 ($lazy
652 ($let* ([opener begin-list]
653 [ints ($sep-by elem separator)]
654 [ (end-list opener) ])
655 ($return ints))))
656 (define elem ($or integer nested-list))
657
658 (test* "integers1" '(456 789)
659 (peg-parse-string integers1 "123, 456, 789"))
660 (test* "integers2" '(123 456 789)
661 (peg-parse-string integers2 "123, 456, 789"))
662 (test* "integers3" '(123 456 789)
663 (peg-parse-string integers3 "123, 456, 789"))
664 (test* "integers4" '(123 456 789)
665 (peg-parse-string integers4 "123, 456, 789"))
666 (test* "integers4" '()
667 (peg-parse-string integers4 ""))
668 (test* "integers5" '(123 456 789)
669 (peg-parse-string integers5 "123, 456, 789"))
670 (test* "integers5" '()
671 (peg-parse-string integers5 ""))
672
673 (test* "int-list" '(123 456 789)
674 (peg-parse-string int-list "[123, 456, 789]"))
675 (test* "int-list" '(123 456 789)
676 (peg-parse-string int-list "{123, 456, 789}"))
677 (test* "int-list" (test-error <parse-error>
678 "expecting #\\) at 14, but got #\\}")
679 (peg-parse-string int-list "(123, 456, 789}"))
680
681 (test* "nested-list" '(123 (456 () 789) 987)
682 (peg-parse-string nested-list "(123, [456, {}, 789], 987)"))
683 )
684
685 (let ()
686 (define paren ($. #\())
687 (define thesis ($. #\)))
688
689 (test* "$or example"
690 (test-error <parse-error> "expecting ab at 1, but got #\\c")
691
692 (peg-parse-string ($or ($seq paren ($."ab") thesis)
693 ($seq paren ($."cd") thesis))
694 "(cd)"))
695 )
696
697 (let ()
698 (define integer ($many1 ($. #[\d])))
699 (define ws ($many_ ($. #[\s])))
700 (define int-list0 ($seq ($. #\[ )
701 ($many ($seq ws integer))
702 ws
703 ($char #\] )))
704 (define int-list
705 ($let ([ ($char #\[ )]
706 [digs ($many ($seq ws integer))]
707 [ ws ]
708 [ ($char #\] ) ])
709 ($return (map (compose x->integer list->string) digs))))
710
711 (test-succ "integer" '(#\1 #\2 #\3) integer "123")
712 (test-succ "int-list0" #\] int-list0 "[123 456 789]")
713 (test-succ "int-list" '(123 456 789) int-list "[123 456 789]")
714 )
715
716 ;; Count the maximal nesting level
717 (letrec ((nesting
718 ($lazy ($or ($let ([ ($char #\() ]
719 [n nesting]
720 [ ($char #\)) ]
721 [m nesting])
722 ($return (max (+ n 1) m)))
723 ($return 0)))))
724 (test-succ "nesting parenthesis" 3 nesting "((()))")
725 (test-succ "nesting parenthesis" 3 nesting "((()))()")
726 (test-succ "nesting parenthesis" 3 nesting "(()(()))")
727 (test-fail "nesting parenthesis" '(3 #\) ) nesting "((("))
728
729 ;; number parser (1) - simple
730 (let* ([sign? ($optional ($one-of #[-+]))]
731 [digits ($seq sign? ($many ($. #[\d]) 1))]
732 [point ($seq ($char #\.) ($many ($. #[\d]) 1))]
733 [exponent ($seq ($. #[eE]) digits)]
734 [number? ($seq digits ($optional point) ($optional exponent))]
735 [p ($seq number? ($eos) ($return #t))])
736 (define (%test str . fails?)
737 (if (null? fails?)
738 (test-succ #"number(1) \"~str\"" #t p str)
739 (test-fail #"number(1) \"~str\"" (car fails?) p str)))
740 (%test "27652") (%test "-27652") (%test "+27652")
741 (%test "" '(0 #[0-9])) (%test "-" '(1 #[0-9])) (%test "+" '(1 #[0-9]))
742 (%test "+27.46") (%test "0.46")
743 (%test ".46" '(0 #[0-9]))
744 (%test "0..3" '(2 #[0-9]))
745 (%test "0.4.2" '(3 "end of input"))
746 (%test "3e5")
747 (%test "3.0e5")
748 (%test "3.e5" '(2 #[0-9]))
749 (%test "3e-53")
750 (%test "3e+53")
751 (%test "-3.0e+53")
752 (%test "-3.0e+" '(6 #[0-9]))
753 )
754
755 ;; CSV parser
756 (let ()
757 (define ws ($many ($. #[ \t])))
758 (define comma ($seq ws ($. #\,) ws))
759 (define dquote ($. #\"))
760 (define double-dquote ($seq ($. "\"\"") ($return #\")))
761 (define quoted-body ($many ($or double-dquote ($. #[^\"]))))
762 (define quoted ($between dquote quoted-body dquote))
763 (define unquoted ($many-till ($any) ($or comma ($. #\newline))))
764 (define field ($or quoted unquoted))
765 (define record ($sep-by ($->rope field) comma))
766 (test-succ "CSV" '("a" "b" "c")
767 record "a,b,c")
768 (test-succ "CSV" '("a" "b" "c")
769 record "\"a\" , b , c")
770 (test-succ "CSV" '("a \" \n" "b" "c")
771 record "\"a \"\" \n\" , b , c"))
772
773 ;; hand-tuned version
774 (let ()
775 (define ws ($many_ ($. #[ \t])))
776 (define comma ($seq ws ($. #\,) ws))
777 (define dquote ($. #\"))
778 (define double-dquote ($seq ($. "\"\"") ($return #\")))
779 (define quoted-body ($many ($or ($. #[^\"]) double-dquote)))
780 (define quoted ($between dquote quoted-body dquote))
781 (define unquoted ($sep-end-by ($many1 ($. #[^ \t\r\n,]))
782 ($many_ ($. #[ \t]))))
783 (define field ($or quoted unquoted))
784 (define record ($sep-by ($->rope field) comma 1))
785 (define records ($sep-end-by record ($. "\r\n")))
786 ;; NB: In CSV spec there's an ambiguity of treatment of the ending CRLF.
787 ;; The CRLF after the last record can be omitted; so if the file ends with
788 ;; CRLF, it may be the end of whole records, or there may be one record
789 ;; with an empty field. Customary it is resolved to take the first
790 ;; interpretation. Since PEG is eager the straightforward implementation
791 ;; takes the second interpretation. For the sake of testing here, it
792 ;; doesn't matter either way.
793 (test-succ "CSV 2" '(("a" "b" "c") ("x" "y" "z") (""))
794 records "a,b,c\r\nx,y,z\r\n")
795 (test-succ "CSV 2" '(("a " "b" "c") ("zzz\nyyy " " w \" ") (""))
796 records "\"a \" , b , c\r\n\"zzz\nyyy \", \" w \"\" \"\r\n")
797 (test-succ "CSV 2" '(("a \" \n" "b" "c"))
798 records "\"a \"\" \n\" , b , c"))
799
800 ;; Poor-man's XML parser
801 (letrec* ([open-tag ($->rope ($between ($char #\<)
802 ($one-of #[[:alnum:]])
803 ($char #\>)))]
804 [close-tag (^[tagname]
805 ($seq ($string "</") ($string tagname) ($char #\>)))]
806 [text ($->rope ($many ($none-of #[<])))]
807 [body ($let ([t text]
808 [r ($many ($let ([e ($lazy element)]
809 [t text])
810 ($return (list e t))))])
811 ($return `(,t ,@(apply append r))))]
812 [element ($let* ([tagname ($try open-tag)]
813 [body body]
814 [ (close-tag (rope-finalize tagname)) ])
815 ($return (cons tagname body)))])
816 (test-succ "tag element" '("a" "")
817 element "<a></a>")
818 (test-succ "tag element" '("a" "foo" ("b" "bar") "baz")
819 element "<a>foo<b>bar</b>baz</a>"))
820
821 ;; Calculator
822 (letrec ([integer ($let ([v ($many ($one-of #[\d]) 1)])
823 ($return (string->number (apply string v))))]
824 [mulop ($or ($seq ($char #\*) ($return *))
825 ($seq ($char #\/) ($return /)))]
826 [addop ($or ($seq ($char #\+) ($return +))
827 ($seq ($char #\-) ($return -)))]
828 [factor ($lazy ($or ($between ($char #\() expr ($char #\)))
829 integer))]
830 [term ($chain-left factor mulop)]
831 [expr ($chain-left term addop)])
832 (test-succ "calculator" 11 expr "1+2*3+4")
833 (test-succ "calculator" 36 expr "2/2+5*(3+4)")
834 (test-succ "calculator" -1 expr "1-2"))
835
836 ;;;
837 ;;; deprecated
838 ;;;
839
840 (test-section "deprecated")
841 ;; kludge - we can't say (use parser.peg.deprecated) until peg/deprecated.scm
842 ;; is installed in place.
843 (or (load "peg/deprecated" :error-if-not-found #f)
844 (load "parser/peg/deprecated" :error-if-not-found #f))
845 (import parser.peg.deprecated)
846 (test-module 'parser.peg.deprecated)
847
848 (define-syntax test-char
849 (syntax-rules ()
850 ((_ parse expin unexpin message)
851 (begin
852 (test-succ (symbol->string 'parse)
853 (string-ref expin 0)
854 parse expin)
855 (test-fail (symbol->string 'parse)
856 '(0 message)
857 parse unexpin)))))
858
859 (test-char anychar "a" "" "character")
860 (test-char upper "A" "a" "upper case letter")
861 (test-char lower "a" "A" "lower case letter")
862 (test-char letter "a" "." "letter")
863 (test-char alphanum "0" "." "letter or digit")
864 (test-char digit "0" "." "digit")
865 (test-char hexdigit "f" "." "hexadecimal digit")
866 (test-char newline "\n" "." "newline")
867 (test-char tab "\t" "." "tab")
868 (test-char space "\t" "." "space")
869
870 (test-succ "spaces" " " spaces " ")
871 (test-succ "spaces" " \n\t" spaces " \n\tj")
872
873 (test-succ "$satisfy" #\0
874 ($satisfy char-numeric? "numeric")
875 "0")
876 (test-fail "$satisfy" '(0 "numeric")
877 ($satisfy char-numeric? "numeric")
878 "a")
879
880 (test-succ "$match1*" #\a
881 ($match1* (#\c x #\r) x)
882 "car foo")
883 (test-succ "$match1*" #\d
884 ($match1* (#\c x #\r) x)
885 "cdr foo")
886 (test-succ "$match1*" '(#\a (#\space #\f #\o #\o))
887 ($match1* (#\c x #\r . t) (list x t))
888 "car foo")
889 (test-succ "$match1* (no result arg)" '(#\c #\u #\r)
890 ($match1* (#\c x #\r))
891 "cur foo")
892 (test-fail "$match1*" '(0 "(#\\c x #\\r)")
893 ($match1* (#\c x #\r) x)
894 "zxy foo")
895 (let ((p ($match1* (#\c x #\r) (=> F)
896 (if (#[ad] x) x (F "car or cdr expected")))))
897 (test-succ "$match1* fail proc" #\a p "car")
898 (test-succ "$match1* fail proc" #\d p "cdr")
899 (test-fail "$match1* fail proc" '(0 "car or cdr expected") p "cxr"))
900 (let ((p ($match1* (#\c x #\r . t) (=> F)
901 (if (#[ad] x) (list x t) (F "car or cdr expected")))))
902 (test-succ "$match1* fail proc" '(#\a (#\z)) p "carz")
903 (test-succ "$match1* fail proc" '(#\d (#\z)) p "cdrz")
904 (test-fail "$match1* fail proc" '(0 "car or cdr expected") p "cxrz"))
905
906 (test-succ "$match1" #\a ($match1 #\a) "abc")
907 (test-succ "$match1" #\0 ($match1 (and (? #[0-9] d)) d) "0ab")
908 (test-fail "$match1" '(0 "#\\a") ($match1 #\a 'yo) "0ab")
909 (let ((p ($match1 (? char-numeric? x) (=> F)
910 (if (eqv? x #\0)
911 'ok
912 (F "0 expected")))))
913 (test-succ "$match1 fail proc" 'ok p "012")
914 (test-fail "$match1 fail proc" '(0 "0 expected") p "123")
915 (test-fail "$match1 not enough input" '(0 "(? char-numeric? x)") p ""))
916
917
918 (let ((p ($binding ($: x ($or ($seq ($: y ($."a")) ($."b"))
919 ($seq ($: z ($."c")) ($."d"))))
920 (list x y z))))
921 (test-succ "$binding" `("b" "a" ,(undefined)) p "ab")
922 (test-succ "$binding" `("d" ,(undefined) "c") p "cd")
923 (test-fail "$binding" '(1 "b") p "az")
924 (test-fail "$binding" '(1 "d") p "cy")
925 (test-fail "$binding" '(0 ((fail-expect . "a") (fail-expect . "c"))) p "pq"))
926
927 (let ((p ($binding ($: x ($or ($seq ($: y ($."a")) ($."b"))
928 ($seq ($: y ($."c")) ($."d"))))
929 (list x y))))
930 (test-succ "$binding" `("b" "a") p "ab")
931 (test-succ "$binding" `("d" "c") p "cd")
932 (test-fail "$binding" '(1 "d") p "cy"))
933
934 (let ((p ($binding ($: x ($or ($seq ($: y ($."a")) ($."b"))
935 ($seq ($: y ($."c")) ($."d"))))
936 (=> F)
937 (if (equal? y "c")
938 (F "no-no")
939 (list x y)))))
940 (test-succ "$binding fail proc" `("b" "a") p "ab")
941 (test-fail "$binding fail proc" `(2 "no-no") p "cd")
942 (test-fail "$binding fail proc" '(1 "d") p "cy"))
943
944 (let ((p ($or ($binding ($: x ($. "ab"))
945 (=> F)
946 (F 'error "ugh"))
947 ($. "ab"))))
948 (test-fail "$binding fail/error" '(2 (error (fail-message . "ugh")))
949 p "ab"))
950
951 (letrec ([p (lambda (q)
952 ($binding ($. "a")
953 ($: y q)
954 ($seq ($: x ($any)) ($. "."))
955 ($. "z")
956 (list x y)))]
957 [q ($binding ($many ($or ($: x ($. "b")) ($: y ($. "c"))))
958 (list x y))])
959 (test-succ "$binding dynamically nested" '(#\d ("b" "c"))
960 (p q) "abcbcd.z")
961 )
962
963 (test-succ "$binding -> lift optimization" '("b" "d")
964 ($binding ($. "a")
965 ($: b ($. "b"))
966 ($. "c")
967 ($: d ($. "d"))
968 ($. "e")
969 (list b d))
970 "abcde")
971
972 (test-succ "eof" (eof-object) eof "")
973 (test-fail "eof" '(0 "end of input") eof "a")
974
975 ;;;============================================================
976 ;;; rfc.json
977 ;;; Test is here since the module uses parser.peg.
978
979 (test-section "rfc.json")
980 (use rfc.json)
981 (test-module 'rfc.json)
982
983 (let ()
984 (define (t str val)
985 (test* "primitive" `(("x" . ,val)) (parse-json-string str)))
986 (t "{\"x\": 100 }" 100)
987 (t "{\"x\" : -100}" -100)
988 (t "{\"x\": +100 }" 100)
989 (t "{\"x\": 12.5} " 12.5)
990 (t "{\"x\":-12.5}" -12.5)
991 (t "{\"x\":+12.5}" 12.5)
992 (t "{\"x\": 1.25e1 }" 12.5)
993 (t "{\"x\":125e-1}" 12.5)
994 (t "{\"x\":1250.0e-2}" 12.5)
995 (t "{\"x\": false }" 'false)
996 (t "{\"x\":true}" 'true)
997 (t "{\"x\":null}" 'null)
998 (t "{\"x\": \"abc\\\"\\\\\\/\\b\\f\\n\\r\\t\\u0040abc\"}"
999 "abc\"\\/\u0008\u000c\u000a\u000d\u0009@abc")
1000 (t "{\"x\": \"\\ud800\\udc01\"}"
1001 "\x10001;")
1002 )
1003
1004 (let ()
1005 (define (t str)
1006 (test* #"parse error ~str" (test-error <json-parse-error>)
1007 (parse-json-string str)))
1008 (t "{\"x\": 100")
1009 (t "{x : 100}}")
1010 )
1011
1012 (let ()
1013 (define (t str)
1014 (test* #"parse error (unpaired surrogate) ~str"
1015 (test-error <json-parse-error> #/unpaired (low|high) surrogate/)
1016 (parse-json-string str)))
1017 (t "{\"a\":\"\\ud800abc\"}")
1018 (t "{\"a\":\"abc\\ude01xyz\"}"))
1019
1020 (test* "parsing an object"
1021 '(("Image"
1022 ("Width" . 800)
1023 ("Height" . 600)
1024 ("Title" . "View from 15th Floor")
1025 ("Thumbnail"
1026 ("Url" . "http://www.example.com/image/481989943")
1027 ("Height" . 125)
1028 ("Width" . "100"))
1029 ("IDs" . #(116 943 234 38793))))
1030 (parse-json-string "{
1031 \"Image\": {
1032 \"Width\": 800,
1033 \"Height\": 600,
1034 \"Title\": \"View from 15th Floor\",
1035 \"Thumbnail\": {
1036 \"Url\": \"http://www.example.com/image/481989943\",
1037 \"Height\": 125,
1038 \"Width\": \"100\"
1039 },
1040 \"IDs\": [116, 943, 234, 38793]
1041 }
1042 }"))
1043
1044 (test* "parsing an array containing two objects"
1045 '#((("precision" . "zip")
1046 ("Latitude" . 37.7668)
1047 ("Longitude" . -122.3959)
1048 ("Address" . "")
1049 ("City" . "SAN FRANCISCO")
1050 ("State" . "CA")
1051 ("Zip" . "94107")
1052 ("Country" . "US"))
1053 (("precision" . "zip")
1054 ("Latitude" . 37.371991)
1055 ("Longitude" . -122.026020)
1056 ("Address" . "")
1057 ("City" . "SUNNYVALE")
1058 ("State" . "CA")
1059 ("Zip" . "94085")
1060 ("Country" . "US")))
1061 (parse-json-string "[
1062 {
1063 \"precision\": \"zip\",
1064 \"Latitude\": 37.7668,
1065 \"Longitude\": -122.3959,
1066 \"Address\": \"\",
1067 \"City\": \"SAN FRANCISCO\",
1068 \"State\": \"CA\",
1069 \"Zip\": \"94107\",
1070 \"Country\": \"US\"
1071 },
1072 {
1073 \"precision\": \"zip\",
1074 \"Latitude\": 37.371991,
1075 \"Longitude\": -122.026020,
1076 \"Address\": \"\",
1077 \"City\": \"SUNNYVALE\",
1078 \"State\": \"CA\",
1079 \"Zip\": \"94085\",
1080 \"Country\": \"US\"
1081 }
1082 ]"))
1083
1084 (test* "Parsing sequence of json objects"
1085 '((("a" . 1)("b" . 2)) (("c" . 3) ("d" . 4)))
1086 (with-input-from-string "{\"a\":1, \"b\":2}{\"c\":3, \"d\":4}"
1087 parse-json*))
1088
1089 (test* "Customizing constructors"
1090 '(object ("x" array 1 2 3) ("y" array #f #t null))
1091 (parameterize ([json-array-handler (^[elts] (cons 'array elts))]
1092 [json-object-handler (^[pairs] (cons 'object pairs))]
1093 [json-special-handler (^y (case y
1094 [(false) #f]
1095 [(true) #t]
1096 [(null) 'null]))])
1097 (parse-json-string "{\"x\":[1,2,3],\"y\":[false,true,null]}")))
1098
1099 (let ()
1100 (define (test-writer name obj)
1101 (test* name obj
1102 (parse-json-string (construct-json-string obj))))
1103
1104 (test-writer "writing an object"
1105 '(("Image"
1106 ("Width" . 800)
1107 ("Height" . 600)
1108 ("Title" . "View from 15th Floor \"magnificent\"")
1109 ("Thumbnail"
1110 ("Url" . "http://www.example.com/image/481989943")
1111 ("Height" . 125)
1112 ("Width" . "100"))
1113 ("Description" . "Foo\nbackslash \\and tab\t and \u00a1")
1114 ("IDs" . #(116 943 234 38793))
1115 ("Misc" . ()))))
1116
1117 (test-writer "writing an array containing two objects"
1118 '#((("precision" . "zip")
1119 ("Latitude" . 37.7668)
1120 ("Longitude" . -122.3959)
1121 ("Address" . "")
1122 ("City" . "SAN FRANCISCO")
1123 ("State" . "CA")
1124 ("Zip" . "94107")
1125 ("Country" . "US"))
1126 (("precision" . "zip")
1127 ("Latitude" . 37.371991)
1128 ("Longitude" . -122.026020)
1129 ("Address" . "")
1130 ("City" . "SUNNYVALE")
1131 ("State" . "CA")
1132 ("Zip" . "94085")
1133 ("Country" . "US"))))
1134 )
1135
1136 (let1 data `(("[\"\\u03bb\"]" #("\x3bb;"))
1137 ("[\"\\ud800\"]" ,(test-error <json-parse-error>))
1138 ("[\"\\ud867\\ude3d\\u03bb\"]" #("\x29e3d;\x3bb;"))
1139 ("[\"\\ude3d\\ud867\"]" ,(test-error <json-parse-error>))
1140 ("[\"\\uf020\\u03bb\"]" #("\xf020;\x3bb;")))
1141 (dolist [d data]
1142 (test* (format "unicode escape reading (~s)" (car d))
1143 (cadr d)
1144 (parse-json-string (car d)))
1145 (when (vector? (cadr data))
1146 (test* (format "unicode escape writing (~s)" (cadr d))
1147 (car d)
1148 (construct-json-string (cadr d))))))
1149
1150 (let ()
1151 (define (t obj)
1152 (test* #"writer error ~obj" (test-error <json-construct-error>)
1153 (construct-json-string obj)))
1154 (t 'a)
1155 (t '#(1 2 x))
1156 (t '(("a" . 2) 9)))
1157
1158 (test* "generalized array" "[1,2,3]"
1159 (construct-json-string '#u8(1 2 3)))
1160 (test* "generalized object" (test-one-of "{\"a\":1,\"b\":2}"
1161 "{\"b\":2,\"a\":1}")
1162 (construct-json-string (hash-table 'eq? '(a . 1) '(b . 2))))
1163
1164 ;; https://sourceforge.net/p/gauche/mailman/message/36786284/
1165 (test* "read/write invariance"
1166 '#(48.529166)
1167 (parse-json-string (construct-json-string '#(48.529166))))
1168
1169 ;; json-nesting-depth-limit
1170 (test* "depth-limit" '#(#(#(#(null))))
1171 (parameterize ((json-nesting-depth-limit 4))
1172 (parse-json-string "[[[[null]]]]")))
1173 (test* "depth-limit" (test-error <json-parse-error> #/nesting is too deep/)
1174 (parameterize ((json-nesting-depth-limit 4))
1175 (parse-json-string "[[[[[null]]]]]")))
1176 (test* "depth-limit" 123
1177 (parameterize ((json-nesting-depth-limit 1))
1178 (parse-json-string "123")))
1179 (test* "depth-limit" (test-error <json-parse-error> #/nesting is too deep/)
1180 (parameterize ((json-nesting-depth-limit 1))
1181 (parse-json-string "{\"x\":123}")))
1182
1183 (include "test-srfi-180")
1184
1185 (test-end)