Debian Codegraph @

test/text.scm

raw
1 ;;
2 ;; testing text.* module
3 ;;
4
5 (use gauche.test)
6 (test-start "text utilities")
7
8 ;;-------------------------------------------------------------------
9 (test-section "csv")
10 (use text.csv)
11 (test-module 'text.csv)
12
13 (test* "csv-reader" '("abc" "def" "" "ghi")
14 (call-with-input-string "abc , def ,, ghi "
15 (make-csv-reader #\,)))
16
17 (test* "csv-reader" '("abc" "def" "" ", ghi")
18 (call-with-input-string "abc : def :: , ghi "
19 (make-csv-reader #\:)))
20
21 (test* "csv-reader" '("abc" "def" "ghi")
22 (call-with-input-string "abc , \"def\" , \"ghi\" "
23 (make-csv-reader #\,)))
24
25 (test* "csv-reader" '("abc" " de,f " "gh\ni" "jkl")
26 (call-with-input-string " abc, \" de,f \" , \"gh\ni\", \"jkl\""
27 (make-csv-reader #\,)))
28
29 (test* "csv-reader" '("ab\nc" "de \n\n \nf " "" "" "gh\"\n\"i")
30 (call-with-input-string " \"ab\nc\" , \"de \n\n \nf \" , , \"\" , \"gh\"\"\n\"\"i\""
31 (make-csv-reader #\,)))
32
33 (test* "csv-reader" '(("" "") ("a" "") ("" "b"))
34 (let1 r (make-csv-reader #\,)
35 (call-with-input-string ",\na, \n ,b"
36 (^p (let* ([a (r p)] [b (r p)] [c (r p)] [d (r p)])
37 (and (eof-object? d)
38 (list a b c)))))))
39
40 (test* "csv-reader" (test-error)
41 (call-with-input-string " abc, def , \"ghi\"\"\n\n"
42 (make-csv-reader #\,)))
43
44 (test* "csv-reader" #t
45 (eof-object?
46 (call-with-input-string "" (make-csv-reader #\,))))
47
48 (test* "csv-writer"
49 "abc,def,123,\"what's up?\",\"he said, \"\"nothing new.\"\"\"\n"
50 (call-with-output-string
51 (lambda (out)
52 ((make-csv-writer #\,)
53 out
54 '("abc" "def" "123" "what's up?" "he said, \"nothing new.\""))))
55 )
56
57 (test* "csv-writer"
58 "abc,def,123,\"what's up?\",\"he said, \"\"nothing new.\"\"\"\r\n"
59 (call-with-output-string
60 (lambda (out)
61 ((make-csv-writer #\, "\r\n")
62 out
63 '("abc" "def" "123" "what's up?" "he said, \"nothing new.\""))))
64 )
65
66 (test* "csv-writer" "\n"
67 (call-with-output-string
68 (lambda (out)
69 ((make-csv-writer #\,) out '()))))
70
71 ;; middle-level API
72
73 (let ([data '(("" "" "" "" "" "" "" "" "")
74 ("Exported data" "" "" "" "" "" "" "" "")
75 ("" "" "" "" "" "" "" "" "")
76 ("" "" "Year" "Country" "" "Population" "GDP" "" "Note")
77 ("" "" "1958" "Land of Lisp" "" "39994" "551,435,453" "" "")
78 ("" "" "1957" "United States of Formula Translators" "" "115333"
79 "4,343,225,434" "" "Estimated")
80 ("" "" "1959" "People's Republic of COBOL" ""
81 "82524" "3,357,551,143" "" "")
82 ("" "" "1970" "Kingdom of Pascal" "" "3785" "" "" "GDP missing")
83 ("" "" "" "" "" "" "" "" "")
84 ("" "" "1962" "APL Republic" "" "1545" "342,335,151" "" ""))]
85 [header-slots1 '("Country" "Year" "GDP" "Population")]
86 [header-slots2 '(#/country/i #/year/i #/gdp/i #/popu/i)])
87 (test* "make-csv-header-parser (strings)" '#(3 2 6 5)
88 (any (make-csv-header-parser header-slots1) data))
89
90 (test* "make-csv-header-parser (regexps)" '#(3 2 6 5)
91 (any (make-csv-header-parser header-slots2) data))
92
93 (test* "make-csv-record-parser (strings)"
94 '(("Land of Lisp" "1958" "551,435,453" "39994")
95 ("United States of Formula Translators" "1957" "4,343,225,434"
96 "115333")
97 ("People's Republic of COBOL" "1959" "3,357,551,143" "82524")
98 ("APL Republic" "1962" "342,335,151" "1545"))
99 (filter-map (make-csv-record-parser header-slots1 '#(3 2 6 5)
100 '(("Year" #/^\d+$/)
101 "Country" "Population" "GDP"))
102 data))
103
104 (test* "make-csv-record-parser (regexps)"
105 '(("Land of Lisp" "1958" "551,435,453" "39994")
106 ("United States of Formula Translators" "1957" "4,343,225,434"
107 "115333")
108 ("People's Republic of COBOL" "1959" "3,357,551,143" "82524")
109 ("APL Republic" "1962" "342,335,151" "1545"))
110 (filter-map (make-csv-record-parser header-slots2 '#(3 2 6 5)
111 '((#/year/i #/^\d+$/)
112 #/country/i #/popu/i #/gdp/i))
113 data))
114
115 (test* "csv-rows->tuples (allow-gap? #f)"
116 '(("Land of Lisp" "1958" "551,435,453" "39994")
117 ("United States of Formula Translators" "1957" "4,343,225,434"
118 "115333")
119 ("People's Republic of COBOL" "1959" "3,357,551,143" "82524")
120 ("Kingdom of Pascal" "1970" "" "3785"))
121 (csv-rows->tuples data header-slots1))
122
123 (test* "csv-rows->tuples (allow-gap? #t)"
124 '(("Land of Lisp" "1958" "551,435,453" "39994")
125 ("United States of Formula Translators" "1957" "4,343,225,434"
126 "115333")
127 ("People's Republic of COBOL" "1959" "3,357,551,143" "82524")
128 ("Kingdom of Pascal" "1970" "" "3785")
129 ("APL Republic" "1962" "342,335,151" "1545"))
130 (csv-rows->tuples data header-slots1 :allow-gap? #t))
131 )
132
133 ;;-------------------------------------------------------------------
134 (test-section "diff")
135 (use text.diff)
136 (use srfi.13)
137 (test-module 'text.diff)
138
139 (define diff-a "foo
140 bar
141 bar
142 baz
143 baz
144 hoge
145 ")
146 (define diff-b "foo
147 bar
148 baz
149 fuga
150 hoge
151 fuga
152 ")
153
154 (test* "diff"
155 '(((- 2 "bar")) ((- 4 "baz") (+ 3 "fuga")) ((+ 5 "fuga")))
156 (diff diff-a diff-b))
157
158 (test* "diff"
159 '(((- 2 "bar")) ((- 4 "baz") (+ 3 "FUGA")) ((+ 5 "FUGA")))
160 (diff diff-a (string-upcase diff-b) :equal string-ci=?))
161
162 (test* "diff-report"
163 " foo\n bar\n- bar\n baz\n- baz\n+ fuga\n hoge\n+ fuga\n"
164 (with-output-to-string
165 (lambda () (diff-report diff-a diff-b))))
166
167 (test* "diff-report/context"
168 (string-join '("***************"
169 "*** 1,6 ****"
170 " foo"
171 " bar"
172 "- bar"
173 " baz"
174 "! baz"
175 " hoge"
176 "--- 1,6 ----"
177 " foo"
178 " bar"
179 " baz"
180 "! fuga"
181 " hoge"
182 "+ fuga")
183 "\n" 'suffix)
184 (with-output-to-string
185 (lambda () (diff-report/context diff-a diff-b))))
186
187 (test* "diff-report/context (one-side empty)"
188 (string-join '("***************"
189 "*** 0 ****"
190 "--- 1,6 ----"
191 "+ foo"
192 "+ bar"
193 "+ baz"
194 "+ fuga"
195 "+ hoge"
196 "+ fuga")
197 "\n" 'suffix)
198 (with-output-to-string
199 (lambda () (diff-report/context "" diff-b))))
200
201 (test* "diff-report/unified"
202 (string-join '("@@ -1,6 +1,6 @@"
203 " foo"
204 " bar"
205 "-bar"
206 " baz"
207 "-baz"
208 "+fuga"
209 " hoge"
210 "+fuga")
211 "\n" 'suffix)
212 (with-output-to-string
213 (lambda () (diff-report/unified diff-a diff-b))))
214
215 (test* "diff-report/context (one-side empty)"
216 (string-join '("@@ -1,6 +0,0 @@"
217 "-foo"
218 "-bar"
219 "-bar"
220 "-baz"
221 "-baz"
222 "-hoge")
223 "\n" 'suffix)
224 (with-output-to-string
225 (lambda () (diff-report/unified diff-a ""))))
226
227 ;;-------------------------------------------------------------------
228 (test-section "edn")
229 (use text.edn)
230 (use srfi.13)
231 (use gauche.uvector)
232 (test-module 'text.edn)
233
234 ;; simple stuff
235 (let1 data `[(#t . " true ")
236 (#f . "false")
237 (nil . "nil ")
238 (symbol . " symbol ")
239 (namespace/name . " namespace/name")
240 (:keyword . ":keyword")
241 ("string" . "\"string\"")
242 ("weird \" characters \t\n\r\\"
243 . "\"weird \\\" characters \\t\\n\\r\\\\\"")
244 (#\a . "\\a")
245 (#\newline . "\\newline")
246 (#\return . "\\return")
247 (#\space . "\\space")
248 (#\tab . "\\tab")
249 (0 . "0")
250 (-1 . "-1")
251 (() . "()")
252 ((a b c) . "(a b c)")
253 (#(a b c) . "[a b c]")
254 (#((a b) (a c) #(b c)) . "[(a b) (a c) [b c]]")]
255 (test* "parser basics" (map car data)
256 (map ($ parse-edn-string $ cdr $) data))
257 (test* "writer basics" (map ($ string-trim-both $ cdr $) data)
258 (map ($ construct-edn-string $ car $) data)))
259
260 ;; numeric formats (round-trip is not guaranteed)
261 ;; NB: Edn page doesn't mention it, but Clojure reader also supports
262 ;; 0xNNNN (hexadecimal) and 0NNN (octal) representations.
263 (let1 data `[(1234 . "1234")
264 (-1234 . "-1234N")
265 (1234 . "+1234N")
266 (1234.0 . "1234M")
267 (1234.5678 . "1234.5678")
268 (12.34 . "1234e-2")
269 (-1234.5678 . "-12.345678e2")
270 (1234.5678 . "12.345678e+2")
271 (1234.5678 . "+12.345678E+2")
272 (4660 . "0x1234")
273 (-4660 . "-0x1234")
274 (668 . "01234")
275 (-668 . "-01234")]
276 (test* "parser numeric" (map car data)
277 (map ($ parse-edn-string $ cdr $) data)))
278
279 ;; other aggregates (round-trip is not guaranteed)
280 (let1 data `(("{:a 1, :b 2, :c 3}" . ,(edn-map :a 1 :b 2 :c '3))
281 ("#{a b c d e}" . ,(edn-set 'a 'b 'c 'd 'e))
282 ("#myobj{:a 1}" . ,(make-edn-object 'myobj (edn-map :a 1)))
283 ("#foo\"abc\"" . ,(make-edn-object 'foo "abc"))
284 ("#bar[1 2 3]" . ,(make-edn-object 'bar '#(1 2 3)))
285 ("#baz 2345" . ,(make-edn-object 'baz 2345))
286 )
287 (dolist [d data]
288 (test* (format "parse ~s" d) (cdr d) (parse-edn-string (car d)) edn-equal?)
289 (test* (format "write ~s" d) (parse-edn-string (car d))
290 ($ parse-edn-string $ construct-edn-string
291 $ parse-edn-string $ car d)
292 edn-equal?)))
293
294 ;; custom object
295 (register-edn-object-handler! 'u8vector
296 (^[tag vec] (vector->u8vector vec)))
297 (define-method edn-write ((x <u8vector>))
298 (display "#u8vector")
299 (edn-write (u8vector->vector x)))
300 (test* "custom parse" '#u8(1 2 3 4)
301 (parse-edn-string "#u8vector[1 2 3 4]"))
302 (test* "custom writer" "#u8vector[1 2 3 4]"
303 (construct-edn-string '#u8(1 2 3 4)))
304
305
306 ;; invalid
307 (let1 invalids '("foo/bar/baz" "/foo" "foo/")
308 (dolist [d invalids]
309 (test* (format "parse (invalid) ~s" d)
310 (test-error <edn-parse-error> #/invalid token/)
311 (parse-edn-string d))))
312 (test* (format "valid symbol name") '(#f #f #f #f #t)
313 (map edn-valid-symbol-name?
314 '("nil" "bar/nil" "true" "false" "nile")))
315
316 ;;-------------------------------------------------------------------
317 (test-section "external-editor")
318 (use text.external-editor)
319 (test-module 'text.external-editor)
320
321 ;;-------------------------------------------------------------------
322 ;; text.fill depends on gauche.unicode, and tested in ext/gauche.
323
324 ;;-------------------------------------------------------------------
325 (test-section "html-lite")
326 (use text.html-lite)
327 (use srfi.13)
328 (test-module 'text.html-lite)
329
330 (test* "html-escape-string"
331 "&lt;a href=&quot;http://abc/def?ghi&amp;jkl&quot;&gt;"
332 (html-escape-string "<a href=\"http://abc/def?ghi&jkl\">"))
333
334 (test* "html-escape-string"
335 "&lt;class&gt;"
336 (html-escape-string '<class>))
337
338 (test* "html-doctype"
339 '("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\""
340 "\"http://www.w3.org/TR/html4/strict.dtd\">" "")
341 (map string-trim-both (string-split (html-doctype) #\newline)))
342
343 (test* "html-doctype"
344 '("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\""
345 "\"http://www.w3.org/TR/html4/loose.dtd\">" "")
346 (map string-trim-both
347 (string-split (html-doctype :type :transitional) #\newline)))
348
349 (test* "html-doctype"
350 '("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Frameset//EN\""
351 "\"http://www.w3.org/TR/html4/frameset.dtd\">" "")
352 (map string-trim-both
353 (string-split (html-doctype :type :frameset) #\newline)))
354
355 (test* "html-doctype"
356 '("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\""
357 "\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">" "")
358 (map string-trim-both
359 (string-split (html-doctype :type :xhtml-1.0) #\newline)))
360
361 (use srfi.13)
362
363 (let ()
364 ;; NB: avoid using tree->string, for we haven't tested it yet.
365 (define (tos x) (string-delete (string-downcase (x->string x)) #\newline))
366 (define (flatten-rec x r)
367 (cond ((null? x) r)
368 ((not (pair? x)) (cons (tos x) r))
369 ((pair? (car x))
370 (flatten-rec (cdr x) (flatten-rec (car x) r)))
371 ((null? (car x)) (flatten-rec (cdr x) r))
372 (else (flatten-rec (cdr x) (cons (tos (car x)) r)))))
373 (define (flatten x) (string-concatenate-reverse (flatten-rec x '())))
374
375 (test* "html, head, body"
376 "<html><head><title>foo</title></head><body>foo</body></html>"
377 (flatten (html:html (html:head (html:title "foo"))
378 (html:body "foo"))))
379 (test* "attributes"
380 "<a href=\"http://foo/bar?a&amp;b\" id=\"aabb\">zzdd</a>"
381 (flatten (html:a :href "http://foo/bar?a&b" :id "aabb" "zzdd")))
382
383 (test* "empty element"
384 "<img src=\"foo\" alt=\"bar baz\" />"
385 (flatten (html:img :src "foo" :alt "bar baz")))
386 )
387
388 ;;-------------------------------------------------------------------
389 (test-section "info")
390 (use text.info)
391 (test-module 'text.info)
392
393 ;;-------------------------------------------------------------------
394 (test-section "multicolumn")
395 (use text.multicolumn)
396 (test-module 'text.multicolumn)
397
398 (let ([data '("alike" "ancient" "attend" "blood" "both"
399 "break" "bury" "but" "children's" "civil" "continuance"
400 "could" "death" "death-mark'd" "dignity" "do" "ears"
401 "end" "fair" "fatal" "fearful" "foes" "forth" "from"
402 "grudge" "hands" "here" "hours" "households" "if"
403 "in" "is" "lay" "life" "loins" "love" "lovers" "makes"
404 "mend" "misadventured" "miss" "mutiny" "new" "nought"
405 "now" "of" "our" "overthrows" "pair" "parents"
406 "passage" "patient" "piteous" "rage" "remove" "scene"
407 "shall" "stage" "star-cross'd" "strife" "strive" "take"
408 "the" "their" "these" "to" "toil" "traffic" "two" "unclean"
409 "verona" "we" "what" "where" "which" "whose" "with" "you")])
410 (define (t expect . args)
411 (test*/diff (write-to-string `(display-multicolumn data ,@args))
412 expect
413 (with-output-to-string
414 (cut apply display-multicolumn data args))))
415
416 (t '(
417 "alike fearful miss strive"
418 "ancient foes mutiny take"
419 "attend forth new the"
420 "blood from nought their"
421 "both grudge now these"
422 "break hands of to"
423 "bury here our toil"
424 "but hours overthrows traffic"
425 "children's households pair two"
426 "civil if parents unclean"
427 "continuance in passage verona"
428 "could is patient we"
429 "death lay piteous what"
430 "death-mark'd life rage where"
431 "dignity loins remove which"
432 "do love scene whose"
433 "ears lovers shall with"
434 "end makes stage you"
435 "fair mend star-cross'd"
436 "fatal misadventured strife"))
437
438 (t '(
439 "alike here piteous"
440 "ancient hours rage"
441 "attend households remove"
442 "blood if scene"
443 "both in shall"
444 "break is stage"
445 "bury lay star-cross'd"
446 "but life strife"
447 "children's loins strive"
448 "civil love take"
449 "continuance lovers the"
450 "could makes their"
451 "death mend these"
452 "death-mark'd misadventured to"
453 "dignity miss toil"
454 "do mutiny traffic"
455 "ears new two"
456 "end nought unclean"
457 "fair now verona"
458 "fatal of we"
459 "fearful our what"
460 "foes overthrows where"
461 "forth pair which"
462 "from parents whose"
463 "grudge passage with"
464 "hands patient you")
465 :minimum-width 25)
466
467 (t '("alike ears lay pair these"
468 "ancient end life parents to"
469 "attend fair loins passage toil"
470 "blood fatal love patient traffic"
471 "both fearful lovers piteous two"
472 "break foes makes rage unclean"
473 "bury forth mend remove verona"
474 "but from misadventured scene we"
475 "children's grudge miss shall what"
476 "civil hands mutiny stage where"
477 "continuance here new star-cross'd which"
478 "could hours nought strife whose"
479 "death households now strive with"
480 "death-mark'd if of take you"
481 "dignity in our the"
482 "do is overthrows their")
483 :max-columns 7)
484
485 (t '("alike ancient attend blood"
486 "both break bury but"
487 "children's civil continuance could"
488 "death death-mark'd dignity do"
489 "ears end fair fatal"
490 "fearful foes forth from"
491 "grudge hands here hours"
492 "households if in is"
493 "lay life loins love"
494 "lovers makes mend misadventured"
495 "miss mutiny new nought"
496 "now of our overthrows"
497 "pair parents passage patient"
498 "piteous rage remove scene"
499 "shall stage star-cross'd strife"
500 "strive take the their"
501 "these to toil traffic"
502 "two unclean verona we"
503 "what where which whose"
504 "with you")
505 :order 'row)
506
507 (t '(
508 "alike here piteous"
509 "ancient hours rage"
510 "attend households remove"
511 "blood if scene"
512 "both in shall"
513 "break is stage"
514 "bury lay star-cross'd"
515 "but life strife"
516 "children's loins strive"
517 "civil love take"
518 "continuance lovers the"
519 "could makes their"
520 "death mend these"
521 "death-mark'd misadventured to"
522 "dignity miss toil"
523 "do mutiny traffic"
524 "ears new two"
525 "end nought unclean"
526 "fair now verona"
527 "fatal of we"
528 "fearful our what"
529 "foes overthrows where"
530 "forth pair which"
531 "from parents whose"
532 "grudge passage with"
533 "hands patient you")
534 :width 50)
535
536 (t '(
537 " alike here piteous"
538 " ancient hours rage"
539 " attend households remove"
540 " blood if scene"
541 " both in shall"
542 " break is stage"
543 " bury lay star-cross'd"
544 " but life strife"
545 " children's loins strive"
546 " civil love take"
547 " continuance lovers the"
548 " could makes their"
549 " death mend these"
550 " death-mark'd misadventured to"
551 " dignity miss toil"
552 " do mutiny traffic"
553 " ears new two"
554 " end nought unclean"
555 " fair now verona"
556 " fatal of we"
557 " fearful our what"
558 " foes overthrows where"
559 " forth pair which"
560 " from parents whose"
561 " grudge passage with"
562 " hands patient you")
563 :width 50 :indent 5)
564
565 (t '(
566 " alike misadventured"
567 " ancient miss"
568 " attend mutiny"
569 " blood new"
570 " both nought"
571 " break now"
572 " bury of"
573 " but our"
574 " children's overthrows"
575 " civil pair"
576 " continuance parents"
577 " could passage"
578 " death patient"
579 " death-mark'd piteous"
580 " dignity rage"
581 " do remove"
582 " ears scene"
583 " end shall"
584 " fair stage"
585 " fatal star-cross'd"
586 " fearful strife"
587 " foes strive"
588 " forth take"
589 " from the"
590 " grudge their"
591 " hands these"
592 " here to"
593 " hours toil"
594 " households traffic"
595 " if two"
596 " in unclean"
597 " is verona"
598 " lay we"
599 " life what"
600 " loins where"
601 " love which"
602 " lovers whose"
603 " makes with"
604 " mend you")
605 :width 45 :indent 5)
606 )
607
608 ;;-------------------------------------------------------------------
609 (test-section "parse")
610 (use text.parse)
611 (test-module 'text.parse)
612
613 ;; a part of text data is taken from Oleg's vinput-parse.scm
614 ;; http://pobox.com/~oleg/ftp/Scheme/parsing.html
615
616 (define (test-find-string input pattern . max-chars)
617 (call-with-input-string input
618 (lambda (p)
619 (let* ((n (apply find-string-from-port? pattern p max-chars))
620 (c (read-char p)))
621 (list n (if (eof-object? c) 'eof c))))))
622
623 (test* "find-string-from-port?" '(7 #\d)
624 (test-find-string "bacacabd" "acab"))
625 (test* "find-string-from-port?" '(7 #\d)
626 (test-find-string "bacacabd" "acab" 100))
627 (test* "find-string-from-port?" '(#f eof)
628 (test-find-string "bacacabd" "acad"))
629 (test* "find-string-from-port?" '(#f eof)
630 (test-find-string "bacacabd" "acad" 100))
631 (test* "find-string-from-port?" '(#f #\a)
632 (test-find-string "bacacabd" "bd" 5))
633 (test* "find-string-from-port?" '(8 eof)
634 (test-find-string "bacacabd" "bd" 9))
635 (test* "find-string-from-port?" '(8 eof)
636 (test-find-string "bacacabd" "bd"))
637 (test* "find-string-from-port?" '(8 eof)
638 (test-find-string "bacacabd" "bd" 8))
639 (test* "find-string-from-port?" '(#f eof)
640 (test-find-string "bacacabd" "be" 20))
641
642
643 (define (test-parseutil proc input . args)
644 (call-with-input-string input
645 (lambda (p)
646 (let* ((c (apply proc (append args (list p))))
647 (n (read-char p)))
648 (list (if (eof-object? c) 'eof c)
649 (if (eof-object? n) 'eof n))))))
650
651 (define (test-assert-curr-char str clist)
652 (test-parseutil assert-curr-char str clist "zz"))
653
654 (test* "assert-curr-char" '(#\space #\a)
655 (test-assert-curr-char " abcd" '(#\a #\space)))
656 (test* "assert-curr-char" '(#\space #\a)
657 (test-assert-curr-char " abcd" #[a ]))
658 (test* "assert-curr-char" '(#\space #\a)
659 (test-assert-curr-char " abcd" #[a\s]))
660 (test* "assert-curr-char" '(#\space #\a)
661 (test-assert-curr-char " abcd" '(#\a #[\s])))
662 (test* "assert-curr-char" '(#\a #\space)
663 (test-assert-curr-char "a bcd" '(#\a #\space)))
664 (test* "assert-curr-char" '(#\a #\space)
665 (test-assert-curr-char "a bcd" #[a ]))
666 (test* "assert-curr-char" (test-error)
667 (test-assert-curr-char "bcd" #[a ]))
668 (test* "assert-curr-char" (test-error)
669 (test-assert-curr-char "" #[a ]))
670 (test* "assert-curr-char" '(eof eof)
671 (test-assert-curr-char "" '(#\a #\space *eof*)))
672
673 (test* "skip-until number" '(#f #\a)
674 (test-parseutil skip-until " abcd" 1))
675 (test* "skip-until number" (test-error)
676 (test-parseutil skip-until " abcd" 10))
677 (test* "skip-until number" '(#f eof)
678 (test-parseutil skip-until " abcd" 5))
679 (test* "skip-until cset" '(#\space #\a)
680 (test-parseutil skip-until " abcd" '(#\a #\space)))
681 (test* "skip-until cset" '(#\space #\a)
682 (test-parseutil skip-until " abcd" #[a ]))
683 (test* "skip-until cset" '(#\c #\space)
684 (test-parseutil skip-until "xxxc bcd" #[abc ]))
685 (test* "skip-until cset" '(#\c eof)
686 (test-parseutil skip-until "xxxc" #[abc ]))
687 (test* "skip-until cset" (test-error)
688 (test-parseutil skip-until "xxxc" #[def]))
689 (test* "skip-until cset" '(eof eof)
690 (test-parseutil skip-until "xxxc" '(#[def] *eof*)))
691 (test* "skip-until cset" '(#\c eof)
692 (test-parseutil skip-until "xxxc" '(#[c-f] *eof*)))
693 (test* "skip-until proc" '(#\c #\space)
694 (test-parseutil skip-until "xxxc bcd"
695 (^x (not (eqv? x #\x)))))
696 (test* "skip-until proc" '(eof eof)
697 (test-parseutil skip-until "xxx"
698 (^x (not (eqv? x #\x)))))
699 (test* "skip-until proc" (test-error)
700 (test-parseutil skip-until "yyyy"
701 (^x (eqv? x #\x))))
702 (test* "skip-while" '(#\d #\d)
703 (test-parseutil skip-while "xxxd" '(#\a #\space #\x)))
704 (test* "skip-while" '(#\d #\d)
705 (test-parseutil skip-while "xxxd" #[ax ]))
706 (test* "skip-while" '(#\y #\y)
707 (test-parseutil skip-while "yxxxd" #[ax ]))
708 (test* "skip-while" '(eof eof)
709 (test-parseutil skip-while "xxxa" #[ax ]))
710 (test* "skip-while" '(#\d #\d)
711 (test-parseutil skip-while "xxxd"
712 (^x (eqv? x #\x))))
713 (test* "skip-while" '(#\y #\y)
714 (test-parseutil skip-while "yxxxd"
715 (^x (eqv? x #\x))))
716 (test* "skip-while" '(eof eof)
717 (test-parseutil skip-while "yxxxd"
718 (^x (and (char? x)
719 (char-alphabetic? x)))))
720
721 (test* "next-token" '("" #\d)
722 (test-parseutil next-token "xxxd" #[ax ] #[d] "next token"))
723 (test* "next-token" '("bc" #\d)
724 (test-parseutil next-token "xxxabcd" #[ax ] #[d] "next token"))
725 (test* "next-token" '("aeio" #\tab)
726 (test-parseutil next-token " aeio\tnjj" #[\s] #[\s] "next token"))
727 (test* "next-token" (test-error)
728 (test-parseutil next-token " aeio" #[\s] #[\s] "next token"))
729 (test* "next-token" '("aeio" eof)
730 (test-parseutil next-token " aeio" #[\s] '(#[\s] *eof*) "next token"))
731 (test* "next-token" '("aeio" #\tab)
732 (test-parseutil next-token " aeio\tnjj"
733 (^x (and (char? x)
734 (char-whitespace? x)))
735 (^x (or (eof-object? x)
736 (char-whitespace? x)))
737 "next token"
738 ))
739
740 (test* "next-token-of" '("" #\x)
741 (test-parseutil next-token-of "xxxd" #[a-c]))
742 (test* "next-token-of" '("" #\x)
743 (test-parseutil next-token-of "xxxd" #[a-d]))
744 (test* "next-token-of" '("xxx" #\d)
745 (test-parseutil next-token-of "xxxd" #[ax]))
746 (test* "next-token-of" '("anmb" #\-)
747 (test-parseutil next-token-of "anmb-runge" #[\w]))
748 (test* "next-token-of" '("rnge!rg0#$@" #\space)
749 (test-parseutil next-token-of "rnge!rg0#$@ bag" #[\S]))
750 (test* "next-token-of" '("xxx" #\d)
751 (test-parseutil next-token-of "xxxd"
752 (^x (eqv? x #\x))))
753 (test* "next-token-of" '("xxxx" eof)
754 (test-parseutil next-token-of "xxxx"
755 (^x (eqv? x #\x))))
756
757 (test* "read-string" '("aaaa" #\a)
758 (test-parseutil read-string "aaaaa" 4))
759 (test* "read-string" '("aaaaa" eof)
760 (test-parseutil read-string "aaaaa" 5))
761 (test* "read-string" '("aaaaa" eof)
762 (test-parseutil read-string "aaaaa" 6))
763 (test* "read-string" '("" #\a)
764 (test-parseutil read-string "aaaaa" 0))
765 (test* "read-string" '("" #\a)
766 (test-parseutil read-string "aaaaa" -1))
767 (test* "read-string" '("" eof)
768 (test-parseutil read-string "" 7))
769
770 ;;-------------------------------------------------------------------
771 (test-section "progress")
772 (use text.progress)
773 (test-module 'text.progress)
774
775 ;; WRITEME
776
777 ;;-------------------------------------------------------------------
778 (test-section "sh")
779 (use text.sh)
780 (test-module 'text.sh)
781
782 (let ()
783 (define (t input expected flavor)
784 (test* #"shell-escape-string(~flavor) - ~input" expected
785 (shell-escape-string input flavor)))
786
787 (t "" "\"\"" 'windows)
788 (t "abc" "abc" 'windows)
789 (t "a b c" "\"a b c\"" 'windows)
790 (t "a \"b\" c" "\"a \\\"b\\\" c\"" 'windows)
791 (t "a\\ \\\"" "\"a\\ \\\\\\\"\"" 'windows)
792 (t "" "''" 'posix)
793 (t "abc" "abc" 'posix)
794 (t "a b c" "'a b c'" 'posix)
795 (t "$abc" "'$abc'" 'posix)
796 (t "[abc]" "'[abc]'" 'posix)
797 (t "\\abc" "'\\abc'" 'posix)
798 (t ">abc" "'>abc'" 'posix)
799 (t "<abc" "'<abc'" 'posix)
800 (t "a\"b\"c" "'a\"b\"c'" 'posix)
801 (t "a(b)c" "'a(b)c'" 'posix)
802 (t "a{b}c" "'a{b}c'" 'posix)
803 (t "a'c" "'a'\"'\"'c'" 'posix)
804 )
805
806 (let ()
807 (define (t input expected flavor)
808 (test* #"shell-tokenize-string(~flavor) - ~input" expected
809 (shell-tokenize-string input flavor)))
810
811 (t "" '() 'posix)
812 (t " " '() 'posix)
813 (t "abc" '("abc") 'posix)
814 (t " abc " '("abc") 'posix)
815 (t "ab c d" '("ab" "c" "d") 'posix)
816 (t "ab\\ c\\ d" '("ab c d") 'posix)
817 (t "ab\\'c\\\"d" '("ab'c\"d") 'posix)
818 (t "'ab c d' ef \"g h\"" '("ab c d" "ef" "g h") 'posix)
819 (t "'ab\\\\cd' \"ab\\\\cd\"" '("ab\\\\cd" "ab\\cd") 'posix)
820 (t "'ab\\xcd' \"ab\\xcd\"" '("ab\\xcd" "ab\\xcd") 'posix)
821 (t "a\"\"b" '("ab") 'posix)
822 (t "a''b" '("ab") 'posix)
823 (t "a'\"'b" '("a\"b") 'posix)
824 (t "a\"'\"b" '("a'b") 'posix)
825
826 (t "$abc" (test-error) 'posix)
827 (t "\"$abc\"" (test-error) 'posix)
828 (t "'$abc'" '("$abc") 'posix)
829 (t "abc >z" (test-error) 'posix)
830 (t "abc \">z\"" '("abc" ">z") 'posix)
831 (t "abc \"`z`\"" (test-error) 'posix)
832 (t "abc '`z`'" '("abc" "`z`") 'posix)
833
834 (t "" '() 'windows)
835 (t " " '() 'windows)
836 (t "abc" '("abc") 'windows)
837 (t " abc " '("abc") 'windows)
838 (t "ab c d" '("ab" "c" "d") 'windows)
839 (t "\"a b c\" d e" '("a b c" "d" "e") 'windows)
840 (t "\"ab\\\"c\" \"\\\\\" d" '("ab\"c" "\\" "d") 'windows)
841 (t "a\\\\\\b d\"e f\"g h" '("a\\\\\\b" "de fg" "h") 'windows)
842 (t "a\\\\\\\"b c d" '("a\\\"b" "c" "d") 'windows)
843 (t "a\\\\\\\\\"b c\" d e" '("a\\\\b c" "d" "e") 'windows)
844 (t "\"" '("") 'windows)
845 (t "\"\"" '("") 'windows)
846 (t "\"\"\"" '("\"") 'windows)
847 (t "\"\"\"\"" '("\"") 'windows)
848 (t "\"\"\"\"\"" '("\"") 'windows)
849 (t "\"\"\"\"\"\"" '("\"\"") 'windows)
850 (t "\"\"\"\"\"\"\"" '("\"\"") 'windows)
851 (t "a\"b\"\" c" '("ab\"" "c") 'windows)
852 (t "a\"b\"\"\" c" '("ab\" c") 'windows)
853 (t "a\"b\"\"\"\" c" '("ab\"" "c") 'windows)
854 )
855
856 (let ()
857 (define (t-case input expect)
858 (test* #"shell-case: ~input"
859 expect
860 ($ shell-case input
861 ["foo" 'foo]
862 ["foo?" 'foo?]
863 ["foo*" 'foo*]
864 ["*foo" '*foo]
865 [("bar*" "baz*") 'barz]
866 [else 'else])))
867 (t-case "foo" 'foo)
868 (t-case "foof" 'foo?)
869 (t-case "foogh" 'foo*)
870 (t-case "fufoo" '*foo)
871 (t-case "bazooka" 'barz)
872 (t-case "zbar" 'else)
873 (t-case "zfooz" 'else)
874 )
875
876 ;;-------------------------------------------------------------------
877 (test-section "sql")
878 (use text.sql)
879 (test-module 'text.sql)
880
881 (test* "sql-tokenize" '("select" "tab" #\. "x" #\, "tab" #\. "y" "as" "foo"
882 "from" "tab" "where" "tab" #\. "z" < (number "30"))
883 (sql-tokenize "select tab.x, tab.y as foo from tab\nwhere tab.z<30"))
884
885 (test* "sql-tokenize (literal numberes)" '((number "0")
886 (number "-12")
887 (number "+12")
888 (number ".123")
889 (number "123.")
890 (number "123.45")
891 (number "-.123")
892 (number "-123.")
893 (number "-123.45")
894 (number "+.123")
895 (number "+123.")
896 (number "+123.45")
897 (number "0E0")
898 (number "-1E3")
899 (number "-1.E3")
900 (number "-.1E3")
901 (number "-1.2E3")
902 (number "1E-3")
903 (number "1.E-3")
904 (number ".1E-3")
905 - #\. "E" (number "-3")
906 (number "1.2") (number ".3")
907 )
908
909 (sql-tokenize "0 -12 +12 .123 123. 123.45 -.123 -123. -123.45
910 +.123 +123. +123.45 0E0 -1E3 -1.E3 -.1E3
911 -1.2E3 1E-3 1.E-3 .1E-3 -.E-3 1.2.3"))
912
913 (test* "sql-tokenize (literal strings)" '((string "abc")
914 (string "ab'c")
915 (string "'abc")
916 (string "abc'")
917 (string "")
918 (string "'")
919 (string "a'b'c'"))
920 (sql-tokenize "'abc' 'ab''c' '''abc' 'abc''' '' '''' 'a''b''c'''"))
921
922 (test* "sql-tokenize (unterminated literal)" (test-error <sql-parse-error>)
923 (sql-tokenize "'abc def"))
924
925 (test* "sql-tokenize (unterminated literal)" (test-error <sql-parse-error>)
926 (sql-tokenize "'abc''def"))
927
928 (test* "sql-tokenize (other stuff)" '((bitstring "0")
929 (bitstring "010101")
930 (hexstring "0")
931 (hexstring "1aBc9")
932 (delimited "run \"run\" run"))
933 (sql-tokenize "B'0' B'010101' X'0' X'1aBc9' \"run \"\"run\"\" run\""))
934
935 (test* "sql-tokenize (parameters)" '((parameter 0) #\,
936 (parameter 1) #\,
937 (parameter "foo") #\,
938 (parameter "bar") #\,
939 (parameter 2))
940 (sql-tokenize "?,?,:foo, :bar , ?"))
941
942 ;;-------------------------------------------------------------------
943 (test-section "pager")
944 (use text.pager)
945 (test-module 'text.pager)
946
947 ;;-------------------------------------------------------------------
948 (test-section "segmented-match")
949 (use text.segmented-match)
950 (test-module 'text.segmented-match)
951
952 (let ([data '(;; pattern (success-word ...) (fail-word ...)
953 ("a-b-c"
954 ("a-b-c" "a-b-c-d" "a-b-c-d-e" "a-b-c-"
955 "abc-bcd-cde" "abc-bcd-cde-def")
956 ("a-b" "ab-c" "abc-cde" "a-b-"))
957 ("abc"
958 ("abc" "abcd" "abc-abc" "abc-")
959 ("ab" "-abc" "a-bc"))
960 ("a--b"
961 ("a--b" "aaa--bbb")
962 ("a-b")))])
963 (dolist [pattest data]
964 (let ([pattern (car pattest)])
965 (dolist [succ (cadr pattest)]
966 (test* `(segmented-prefix ,pattern ,succ) #t
967 (segmented-prefix? pattern succ)))
968 (dolist [fail (caddr pattest)]
969 (test* `(segmented-prefix ,pattern ,fail) #f
970 (segmented-prefix? pattern fail)))
971 ))
972 )
973
974 (test* "segmented match (alternative separator)" #t
975 (segmented-prefix? "/u/b/g" "/usr/bin/gosh" #\/))
976
977 ;;-------------------------------------------------------------------
978 (test-section "template")
979 (use text.template)
980 (test-module 'text.template)
981
982 (test* "expand-template-string"
983 "abc 9 ghi mno3e8"
984 (expand-template-string
985 "abc ~def ghi ~(jkl \"mno~x\" pqr)"
986 (make-template-environment
987 :bindings `(abc 3 def 9 jkl ,format pqr 1000))))
988
989 (test*/diff "expand-template-file"
990 "<html><head><title>Test</title></head><body><p><a href=\"foo\">bar</a\n>baz</p\n></body></html>\n"
991 (expand-template-file
992 "template.txt"
993 (make-template-environment
994 :imports '(text.tree)
995 :bindings `(title "Test"
996 body ,(html:p (html:a :href "foo" "bar") "baz")))
997 '("../test/data")))
998
999 ;;-------------------------------------------------------------------
1000 (test-section "tree")
1001 (use text.tree)
1002 (test-module 'text.tree)
1003
1004 (test* "tree->string" "" (tree->string '()))
1005 (test* "tree->string" "" (tree->string ""))
1006 (test* "tree->string" "ab" (tree->string "ab"))
1007 (test* "tree->string" "ab" (tree->string 'ab))
1008 (test* "tree->string" "ab" (tree->string '(a . b)))
1009 (test* "tree->string" "ab" (tree->string '(a b)))
1010 (test* "tree->string" "ab" (tree->string '((((() ())) . a) ((((b)))))))
1011 (test* "tree->string"
1012 (if (symbol? :b) "A:b" "Ab") ; transient during symbol-keyword integration
1013 (tree->string '(|A| . :b)))
1014
1015 ;;-------------------------------------------------------------------
1016 (test-section "unicode.ucd")
1017 (use text.unicode.ucd)
1018 (test-module 'text.unicode.ucd)
1019
1020 (test-section "unicode.codeset")
1021 (use text.unicode.codeset)
1022 (test-module 'text.unicode.codeset)
1023
1024 (let ([cs (make <char-code-set>)])
1025 ;; testing overlapped ranges
1026 (add-code-range! cs 64 127)
1027 (add-code-range! cs 8 80)
1028 (add-code-range! cs 100 1000)
1029 (add-code-range! cs 200 210) ; totally subsumed
1030 (add-code-range! cs 1500 2000) ; independent
1031
1032 (test* "add-code-range! (bitmap)"
1033 #xffff_ffff_ffff_ffff_ffff_ffff_ffff_ff00
1034 (~ cs'small-map))
1035
1036 (test* "add-code-range! (subsumed)"
1037 '((128 . 1000) (1500 . 2000))
1038 (tree-map->alist (~ cs'large-map)))
1039
1040 (add-code-range! cs 1800 2100) ; partially overlap
1041 (add-code-range! cs 1400 1600)
1042 (test* "add-code-range! (partially overlap)"
1043 '((128 . 1000) (1400 . 2100))
1044 (tree-map->alist (~ cs'large-map)))
1045
1046 (add-code-range! cs 1300 1400) ; adjacent
1047 (add-code-range! cs 1000 1100) ; adjacent
1048 (add-code-range! cs 2100 2200) ; adjacent
1049 (test* "add-code-range! (adjacent)"
1050 '((128 . 1100) (1300 . 2200))
1051 (tree-map->alist (~ cs'large-map)))
1052
1053 (add-code-range! cs 2500 2600)
1054 (add-code-range! cs 2700 2800)
1055 (add-code-range! cs 2900 3000)
1056 (add-code-range! cs 3100 3200)
1057 (add-code-range! cs 3300 3400)
1058 (add-code-range! cs 2550 3050)
1059 (test* "add-code-range! (multiple)"
1060 '((128 . 1100) (1300 . 2200) (2500 . 3050) (3100 . 3200) (3300 . 3400))
1061 (tree-map->alist (~ cs'large-map)))
1062
1063 (add-code-range! cs 3090 3300)
1064 (test* "add-code-range! (multiple)"
1065 '((128 . 1100) (1300 . 2200) (2500 . 3050) (3090 . 3400))
1066 (tree-map->alist (~ cs'large-map)))
1067
1068 (add-code! cs 3401)
1069 (add-code! cs 3089)
1070 (test* "add-code! (next)"
1071 '((128 . 1100) (1300 . 2200) (2500 . 3050) (3089 . 3401))
1072 (tree-map->alist (~ cs'large-map)))
1073
1074 (add-code-range! cs 3402 3500)
1075 (test* "add-code-range! (next)"
1076 '((128 . 1100) (1300 . 2200) (2500 . 3050) (3089 . 3500))
1077 (tree-map->alist (~ cs'large-map)))
1078
1079 (add-code-range! cs 3052 3088)
1080 (test* "add-code-range! (next)"
1081 '((128 . 1100) (1300 . 2200) (2500 . 3050) (3052 . 3500))
1082 (tree-map->alist (~ cs'large-map)))
1083
1084 (add-code! cs 3051)
1085 (test* "add-code! (fill)"
1086 '((128 . 1100) (1300 . 2200) (2500 . 3500))
1087 (tree-map->alist (~ cs'large-map)))
1088
1089 (add-code-range! cs 1101 1299)
1090 (test* "add-code-range! (fill)"
1091 '((128 . 2200) (2500 . 3500))
1092 (tree-map->alist (~ cs'large-map)))
1093
1094 (add-code-range! cs 4000 4500)
1095 (add-code-range! cs 2300 3999)
1096 (test* "add-code-range! (fill)"
1097 '((128 . 2200) (2300 . 4500))
1098 (tree-map->alist (~ cs'large-map)))
1099 )
1100
1101 (let ([cs1 (make <char-code-set>)]
1102 [cs2 (make <char-code-set>)])
1103 (add-code-range! cs1 8 15)
1104 (add-code-range! cs1 120 200)
1105 (add-code-range! cs1 300 400)
1106 (add-code! cs1 500)
1107 (add-code! cs1 600)
1108 (add-code-range! cs1 1000 2000)
1109 (add-code-range! cs1 3000 4000)
1110 (add-code-range! cs1 5000 6000)
1111 (add-code-range! cs1 7000 8000)
1112
1113 (add-code-range! cs2 24 31)
1114 (add-code-range! cs2 128 250)
1115 (add-code-range! cs2 300 500)
1116 (add-code-range! cs2 600 900)
1117 (add-code-range! cs2 1100 5100)
1118 (add-code-range! cs2 5800 9000)
1119
1120 (let1 cs (code-set-union #f cs1 cs2)
1121 (test* "code-set-union (bitmap)"
1122 #xff00_0000_0000_0000_0000_0000_ff00_ff00
1123 (~ cs'small-map))
1124 (test* "code-set-union (large-map"
1125 '((128 . 250) (300 . 500) (600 . 900) (1000 . 9000))
1126 (tree-map->alist (~ cs'large-map))))
1127 )
1128
1129 (test-end)