Debian Codegraph @

src/gen-unicode.scm

raw
1 ;; -*- coding:utf-8 -*-
2 ;;;
3 ;;; gen-unicode.scm - generate unicode-handling tables
4 ;;;
5 ;;; Originally written by Shiro Kawai, 2011
6 ;;; Public Domain - use as you like.
7 ;;;
8
9 ;; Reads Unicode data tables and generates various source files.
10
11 ;; This script can serve three operations.
12 ;;
13 ;; (1) Generate unicode-data.scm from Unicode character database
14 ;; It is only necessary when new version of Unicode is published,
15 ;; and the resulting unicode-data.scm is checked in to the source
16 ;; tree so that other developers don't need UCD.
17 ;;
18 ;; gosh ./gen-unicode.scm --import <unicode-database-directory> unicode-data.scm [unicode-test.scm]
19 ;;
20 ;; (2) Generate source files from unicode-data.scm
21 ;; To reduce runtime overhead, unicode properties are saved in binary
22 ;; tables, within the following two generated source files:
23 ;;
24 ;; char_attr.c - General category and case mappings.
25 ;; gauche/priv/unicode_attr.h - Grapheme break, word break, East-asian width
26 ;; ext/gauche/unicode-attr.scm - Break property symbols
27 ;;
28 ;; This is done when you build from git source tree.
29 ;;
30 ;; gosh ./gen-unicode.scm --compile unicode-data.scm
31 ;;
32 ;; (3) Fetch Unicode character database files. This is for developer's
33 ;; convenience.
34 ;;
35 ;; gosh ./gen-unicode.scm --fetch <unicode-database-directory> [<unicode-version>
36 ;; Note that the original content in <unicode-database-directory> is
37 ;; overwritten.
38 ;; You might need to adjust code for newer version of Unicode.
39 ;; See doc/HOWTO-unicode.txt for the details.
40
41 (use file.util)
42 (use gauche.charconv)
43 (use gauche.dictionary)
44 (use gauche.generator)
45 (use gauche.record)
46 (use gauche.sequence)
47 (use gauche.uvector)
48 (use scheme.list)
49 (use srfi.13)
50 (use srfi.42)
51 (use text.csv)
52 (use util.match)
53
54 ;; needs to load from source
55 (add-load-path "../lib" :relative)
56 (use text.unicode.ucd)
57 (use text.unicode.codeset)
58
59 ;; rfc.http is only required by '--fetch' operation. We don't want to load
60 ;; it during build process, so let's autoload it.
61 (autoload rfc.http http-get)
62
63
64 ;; We generate four kind of lookup structures. Each structure consists
65 ;; of various types of tables in order to reduce the size.
66 ;;
67 ;; * General categories and case bits
68 ;;
69 ;; This structure maps Unicode codepoint to a byte containing the
70 ;; following info:
71 ;;
72 ;; bit 7-6 - Alphabetic and case info
73 ;; 00 : non-alphabetic char
74 ;; 01 : lowercase alphabetic char
75 ;; 10 : uppercase alphabetic char
76 ;; 11 : caseless or titlecase alphabetic char
77 ;; bit 5 - Reserved
78 ;; bit 4-0 - General category (Lu, Nd, Cc, etc.)
79 ;; See ucd-general-categories
80 ;; and the enum in src/gauche/char_attr.h
81 ;;
82 ;; For U+0000 to U+1ffff, we have a single table that directly maps
83 ;; the codepoint to the above byte.
84 ;;
85 ;; unsigned char ucs_general_category_00000[0x20000] (131072 bytes)
86 ;;
87 ;; Above U+20000 the mappings are sparse and mostly contiguous, so we
88 ;; generate a C code that performs a binary search.
89 ;;
90 ;; unsigned char ucs_general_category_20000(ScmChar code)
91 ;;
92 ;; * Case mapping
93 ;;
94 ;; Almost all characters that needs case mappings are below #xffff. In
95 ;; Unicode 6.0, the only exceptions are 80 characters in U+104xx range
96 ;; (Deseret).
97 ;;
98 ;; Among case-mapped characters, almost all of them requires a simple
99 ;; rule---that is, either a character can be lowercased by adding some
100 ;; small value to the codepoint, or a character can be upcased and
101 ;; titlecased by adding some small value to the codepoint. These have
102 ;; simple casemap entries. Characters that requires more complex handling
103 ;; have extended casemap entries.
104 ;;
105 ;; The tables first maps a codepoint into 16bit entry. The entry
106 ;; represents either a simple casemap entry, or an index to an
107 ;; extended casemap entry.
108 ;;
109 ;; bit15 == 0:
110 ;; This is a simple entry. uppercase and titlecase is the same,
111 ;; no special case mappings, and conversion between
112 ;; (upper,title) <-> lower is simply done by adding the given offset.
113 ;;
114 ;; If bit14 is 0, this letter is uppercase. Converting to uppercase and
115 ;; titlecase is noop. Converting to lowercase is to add
116 ;; the offset.
117 ;; If bit14 is 1, this letter is lowercase. Converting to lowercase is
118 ;; noop. Converting to uppercase and titlecase is to add
119 ;; the offset.
120 ;; bit13-bit0, singned integer offset [-8192, 8191]
121 ;;
122 ;; bit15 == 1:
123 ;; This is an extended entry (except #xffff).
124 ;; bit14-bit0 is an index to an extended entry table.
125 ;;
126 ;; #xffff indicates empty entry.
127 ;;
128 ;; Characters that require case mapping tend to cluster, so we use two-staged
129 ;; table to lookup the 16-bit entry from the codepoint below U+10000.
130 ;;
131 ;; The bit 15-8 is the index of this table:
132 ;;
133 ;; static unsiged char casemap_000[256]
134 ;;
135 ;; If the value is 255, the character doesn't have case mappings.
136 ;; Otherwise, let V be the value of the above lookup, the entry
137 ;; can be looked up by the following table:
138 ;;
139 ;; static unsiged char casemap_subtable[V][<lower 8 bit of codepoint>];
140 ;;
141 ;; We only need to have 18 subtables.
142 ;;
143 ;; * Digit values
144 ;;
145 ;; Characters with categrory Nd has associated numeric value 0..9.
146 ;; A set of decimal numeric characters for 0..9 are always contiguous.
147 ;; As of Unicode 6.2, we have 42 of such sets.
148 ;;
149 ;; * Break properties
150 ;;
151 ;; This structure maps codepoint to the Grapheme_Cluster_Break,
152 ;; Word_Break and Sentence_Break properties.
153 ;;
154 ;; See http://www.unicode.org/reports/tr29/
155 ;; http://www.unicode.org/Public/UNIDATA/auxiliary/GraphemeBreakProperty.txt
156 ;; http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt
157 ;; http://www.unicode.org/Public/UNIDATA/auxiliary/SentenceBreakProperty.txt
158 ;;
159 ;; NB: As of Unicode 13, we also need Extended_Poctographic (ExtPict)
160 ;; Emoji property to do clustering properly. For grapheme break
161 ;; properties, all ExtPict codepoints are GB_Other, so we simply name
162 ;; them GB_ExtPict. For word break properties, ExtPict falls on WB_Other
163 ;; and WB_ALetter. We Name the former WB_ExtPict, and the latter
164 ;; WB_ExtPict_ALetter.
165 ;;
166 ;; The definitive list of properties are in text.unicode.ucd,
167 ;; ucd-grapheme-break-properties and ucd-word-break-properties
168 ;;
169 ;; Codepoints below U+20000 are looked up by two-staged tables.
170 ;; First, look up this table with (codepoint >> 8).
171 ;;
172 ;; static unsigned char {GB|WB}_break_table[0x200]
173 ;;
174 ;; If the value is up to single-property-offset (224), the value is
175 ;; the index to the secondary table.
176 ;;
177 ;; static unsigned char {GB|WB}_break_subtable[index][256]
178 ;;
179 ;; The value of this table encodes GB|WB properties.
180 ;; If th evalue is equal to or more than single-property-offset,
181 ;; it indicates the entire block of 256 codepoints shared the same
182 ;; property, (- value single-property-offset).
183 ;;
184 ;; Codepoints on or above U+20000 are all 'Other', except the following
185 ;; ranges. They are handled specially in the lookup procedure.
186 ;;
187 ;; E0001 GB_Control, WB_Format
188 ;; E0020..E007F GB_Control, WB_Format
189 ;; E0100..E01EF GB_Extend, WB_Extend
190 ;;
191 ;;
192 ;; * Emoji Properties
193 ;;
194 ;; Also http://unicode.org/reports/tr51
195 ;; https://www.unicode.org/Public/13.0.0/ucd/emoji/emoji-data.txt
196 ;;
197
198 ;;;
199 ;;; Main entry
200 ;;;
201
202 (define (main args)
203 (match (cdr args)
204 [("--fetch" dir . maybe-version) (apply fetch-ucd dir maybe-version)]
205 [("--import" dir ucdfile) (import-data dir ucdfile #f)]
206 [("--import" dir ucdfile testfile) (import-data dir ucdfile testfile)]
207 [("--analyze" ucdfile lobits)
208 (analyze-break-tables (call-with-input-file ucdfile ucd-load-db)
209 (string->number lobits))]
210 [("--compile" ucdfile)
211 (unless (file-exists? ucdfile)
212 (exit 1 "Couldn't open unicode data file: ~a" ucdfile))
213 (generate-tables (call-with-input-file ucdfile ucd-load-db))]
214 [else
215 (exit 1 "Usage:\n\
216 gen-unicode.scm --fetch <unicode-database-dir> [<unicode-version>]\n\
217 gen-unicode.scm --import <unicode-database-dir> <ucd-file>\n\
218 gen-unicode.scm --compile <unicode-data.scm>")])
219 0)
220
221 ;;;
222 ;;; Fetching
223 ;;;
224 (define (fetch-ucd dir :optional (version #f))
225 (let ([path (if version
226 #"/Public/~|version|/ucd"
227 "/Public/UCD/latest/ucd")]
228 [datafiles '("UnicodeData.txt"
229 "SpecialCasing.txt"
230 "PropList.txt"
231 "EastAsianWidth.txt"
232 "auxiliary/GraphemeBreakProperty.txt"
233 "auxiliary/SentenceBreakProperty.txt"
234 "auxiliary/WordBreakProperty.txt"
235 "auxiliary/GraphemeBreakTest.txt"
236 "auxiliary/SentenceBreakTest.txt"
237 "auxiliary/WordBreakTest.txt"
238 "emoji/emoji-data.txt")])
239 (make-directory* (build-path dir "auxiliary"))
240 (make-directory* (build-path dir "emoji"))
241 (dolist [f datafiles]
242 (let1 p (build-path dir f)
243 (display #"Getting ~|f|... ") (flush)
244 (call-with-output-file p
245 (^o (http-get "www.unicode.org" (build-path path f)
246 :sink o :flusher (^ _ #t))))
247 (display "done\n")))))
248
249 ;;;
250 ;;; Importing
251 ;;;
252
253 ;; Dump unicode character data in ucdfile.
254 (define (import-data dir ucdfile testfile)
255 (unless (file-is-directory? dir)
256 (exit 1 "Directory required, but got: ~a" dir))
257 (let1 ucd-db (ucd-parse-files dir)
258 (with-output-to-file ucdfile
259 (cut ucd-save-db ucd-db))
260 (when testfile
261 (with-output-to-file testfile
262 (^[] (import-break-test ucd-db dir))))))
263
264 (define (import-break-test ucd-db dir)
265 ;; break test file format:
266 ;; <line>: <sep> (<code> <sep>)+ '#' <comment>
267 ;; <sep> : '÷' | '×'
268 ;; <code> : #[[:xdigit:]]+
269 ;;
270 ;; returns (<count> <seq> <comment>)
271 ;; where
272 ;; <count> : integer, to make it easy to find tests
273 ;; <seq> : <bool> (<integer> <bool>)+
274 ;; <bool> : #t as breakable (÷), #f as unbreakable (×)
275 (define (parse-1 line)
276 (and-let1 m (#/^([÷×])\s+/ line)
277 (let loop ([line (m'after)] [r (list (break-flag (m 1)))])
278 (rxmatch-case line
279 [#/^([[:xdigit:]]+)\s+([÷×])\s+(.*)/ (_ cc ff rr)
280 (loop rr (list* (break-flag ff) (string->number cc 16) r))]
281 [#/^#\s+(.*)/ (_ comment) (list (reverse r) comment)]
282 [else (list (reverse r) "")]))))
283 (define (break-flag s) (equal? s "÷"))
284 (define (parse name file)
285 (let1 path (build-path dir file)
286 (print)
287 (print ";; " file)
288 (print #"(define ~name '(")
289 (generator-for-each (^[r k] (pprint (cons k r)))
290 (gfilter-map parse-1 (file->line-generator path))
291 (giota))
292 (print "))")))
293 (print ";; -*- coding:utf-8 -*-")
294 (print ";; Generated from Unicode " (ucd-version ucd-db) " test data files")
295 (parse '*grapheme-break-tests* "auxiliary/GraphemeBreakTest.txt")
296 (parse '*word-break-tests* "auxiliary/WordBreakTest.txt")
297 (parse '*sentence-break-tests* "auxiliary/SentenceBreakTest.txt")
298 )
299
300 ;;;
301 ;;; Generate source file
302 ;;;
303
304 (define (generate-tables db)
305 (let-values ([(c.p char_attr.c) (sys-mkstemp "char_attr.c.")]
306 [(h.p unicode_attr.h) (sys-mkstemp "unicode_attr.h.")]
307 [(scm.p unicode-attr.scm) (sys-mkstemp "unicode-attr.scm.")])
308 (with-output-to-port c.p
309 (^() (preamble db "/*" "*/")
310 (generate-category-tables db)
311 (generate-case-tables db)
312 (generate-digit-value-tables db)))
313 (with-output-to-port h.p
314 (^() (preamble db "/*" "*/")
315 (generate-break-tables db)
316 (generate-width-tables db)))
317 (with-output-to-port scm.p
318 (^() (preamble db ";;" "")
319 (generate-break-symbols db)))
320 (close-port c.p)
321 (close-port h.p)
322 (close-port scm.p)
323 (make-directory* "gauche/priv") ; in case we're building out-of-tree
324 (sys-rename char_attr.c "char_attr.c")
325 (sys-rename unicode_attr.h "gauche/priv/unicode_attr.h")
326 (sys-rename unicode-attr.scm "../ext/gauche/unicode-attr.scm")))
327
328 (define (preamble db op cl)
329 (print #"~|op| Generated automatically from Unicode character database ~|cl|")
330 (print #"~|op| See src/gen-unicode.scm for the description of data structures. ~|cl|")
331 (print #"~|op| Unicode version ~(ucd-version db). Do not edit. ~|cl|")
332 )
333
334 (define (generate-category-tables db)
335 (define (emit-entry e)
336 (if e
337 (cond
338 [(ucd-entry-uppercase e) (format #t " U(~a)," (ucd-entry-category e))]
339 [(ucd-entry-lowercase e) (format #t " L(~a)," (ucd-entry-category e))]
340 [(ucd-entry-alphabetic e) (format #t " A(~a)," (ucd-entry-category e))]
341 [else (format #t " ~a," (ucd-entry-category e))])
342 (format #t " Cn,")))
343 (define (emit-columns start end p)
344 (do-ec (:parallel (: k start end)
345 (:integers i))
346 (begin
347 (cond [(zero? (mod i 256)) (format #t "\n /* ~5,'0x - */\n" k)]
348 [(zero? (mod i 8)) (newline)])
349 (p k)))
350 (print))
351 (define (ucs->entry ucs) (ucd-get-entry db ucs))
352
353 ;; Setup
354 (dolist [c (ucd-general-categories)]
355 (format #t "#define ~a SCM_CHAR_CATEGORY_~a\n" c c))
356 (print "#undef A")
357 (print "#undef U")
358 (print "#undef L")
359 (print "#define A(x) ((x)|SCM_CHAR_ALPHABETIC_BITS)")
360 (print "#define U(x) ((x)|SCM_CHAR_UPPERCASE_BITS)")
361 (print "#define L(x) ((x)|SCM_CHAR_LOWERCASE_BITS)")
362
363 (let ([code-sets (build-code-sets db (ucd-general-categories))])
364 ;; U+0000 - U+1ffff : direct table lookup
365 (display "static unsigned char ucs_general_category_00000[] = {")
366 (emit-columns 0 #x20000 (^i (emit-entry (ucs->entry i))))
367 (print "};")
368 ;; Over U+20000 - binary search
369 (print)
370 (print "static unsigned char ucs_general_category_20000(ScmChar code)")
371 (print "{")
372 (print " /*")
373 (for-each (^e (format #t " ~6x ~a\n" (car e) (cdr e)))
374 (ucd-get-category-ranges db))
375 (print " */")
376 (generate-bisect (coerce-to <vector> (ucd-get-category-ranges db))
377 (^e (format "return ~a;"
378 (case (cdr e)
379 [(Lo) "A(Lo)"]
380 [(#f) "Cn"]
381 [else => identity]))))
382 (print "}")
383 (for-each
384 (^s (dump-code-set-in-C (cdr s)))
385 (sort (hash-table->alist code-sets) string<? (^x (x->string (car x)))))
386 (print))
387
388 ;; Bind predefined charset
389 ;; The CAT entry can be just a symbol or (alias symbol).
390 (print "static void init_predefined_charsets() {")
391 (dolist [cat (append (ucd-general-categories)
392 '(L LC M N P S Z C
393 (LETTER L) ASCII_LETTER
394 (DIGIT Nd) ASCII_DIGIT
395 LETTER_DIGIT ASCII_LETTER_DIGIT
396 (LOWER Ll) ASCII_LOWER
397 (UPPER Lu) ASCII_UPPER
398 (TITLE Lt)
399 GRAPHIC ASCII_GRAPHIC
400 PRINTING ASCII_PRINTING
401 (PUNCTUATION P) ASCII_PUNCTUATION
402 (SYMBOL S) ASCII_SYMBOL
403 (ISO_CONTROL Cc) ASCII_CONTROL
404 HEX_DIGIT
405 WHITESPACE ASCII_WHITESPACE
406 BLANK ASCII_BLANK
407 ASCII EMPTY
408 ASCII_WORD (WORD ASCII_WORD)))]
409 (let1 setname (if (pair? cat) (car cat) cat)
410 (print #" predef_sets[SCM_CHAR_SET_~|setname|] =")
411 (if (pair? cat)
412 (print #" predef_sets[SCM_CHAR_SET_~(cadr cat)];")
413 (print #" make_charset_~|setname|();"))))
414 (print "}")
415
416 ;; Teardown
417 (dolist [c (ucd-general-categories)]
418 (format #t "#undef ~a\n" c))
419 (format #t "#undef A\n")
420 (format #t "#undef U\n")
421 (format #t "#undef L\n")
422 (print))
423
424 (define (generate-case-tables db)
425 (define subtables '())
426 (define extended '())
427 ;; returns a list of (code . ucd-entry-case-map) with U+HHHxx where HHH
428 ;; is given to the hi arg.
429 (define (gather-entries hb)
430 (fold-ec '() (: lb 256) (ucd-get-entry db (+ (* hb 256) lb))
431 (^(e seed) (if (or (not e) (not (ucd-entry-case-map e)))
432 seed
433 (acons (+ (* hb 256) lb) (ucd-entry-case-map e) seed)))))
434 ;; ucd-entry-alist :: ((code . ucd-entry-case-map) ...)
435 ;; subtable :: (start-code . #(....))
436 ;; returns (start-code . subtable-number)
437 (define (gen-subtable start-code ucd-entry-alist)
438 (if (null? ucd-entry-alist)
439 `(,start-code . 255)
440 (let* ([table-num (length subtables)]
441 [vec (make-vector 256 #f)])
442 (dolist [p ucd-entry-alist]
443 (when (ucd-extended-case-map? (cdr p))
444 (push! extended (cdr p)))
445 (set! (~ vec (logand (car p) #xff)) (cdr p)))
446 (push! subtables (cons start-code vec))
447 (cons start-code table-num))))
448 ;;
449 (define (gather-toptable)
450 (list-ec (: hi 256) (gen-subtable (* hi 256) (gather-entries hi))))
451
452 ;;
453 (define (emit-subtable subtable)
454 (format #t " /* ~4,'0x - ~4,'0x */\n"
455 (car subtable)
456 (+ (car subtable) 255))
457 (format #t " {\n")
458 (dotimes [n 256]
459 (format #t " ~20a, /* 0x~4,'0x */\n"
460 (if-let1 cmap (~ (cdr subtable) n)
461 (if (ucd-simple-case-map? cmap)
462 (format "~a(~a)"
463 (if (eq? (ucd-simple-case-map-case cmap) 'upper)
464 "TOLOWER"
465 "TOUPPER")
466 (ucd-simple-case-map-offset cmap))
467 (format "EXTENDED(~a)"
468 (find-index (cut eq? cmap <>) extended)))
469 "NO_CASE_MAPPING")
470 (+ (car subtable) n)))
471 (format #t " },\n"))
472
473 (define (emit-subtables)
474 (format #t "static unsigned short casemap_subtable[][256] = {\n")
475 (for-each emit-subtable (reverse subtables))
476 (format #t "};\n\n"))
477
478 (define (emit-toptable subtable-num-alist)
479 (let1 vec (make-vector 256 255)
480 (dolist [s subtable-num-alist]
481 (set! (~ vec (div (car s) 256)) (cdr s)))
482 (format #t "static unsigned char casemap_000[] = {")
483 (dotimes [i 256]
484 (when (zero? (mod i 16)) (newline))
485 (format #t " ~3d," (~ vec i)))
486 (format #t "\n};\n")))
487
488 (define (emit-extended-case-maps)
489 (format #t "static ScmCharCaseMap extended_casemaps[] = {\n")
490 (dolist [e extended]
491 (let ([simple (ucd-extended-case-map-simple-map e)]
492 [special (ucd-extended-case-map-special-map e)])
493 (format #t " { ~s,~s,~s,"
494 (or (~ simple 0) 0)
495 (or (~ simple 1) 0)
496 (or (~ simple 2) 0))
497 (if special
498 (format #t "~a,~a,~a"
499 (extended-special-array (~ special 0))
500 (extended-special-array (~ special 1))
501 (extended-special-array (~ special 2)))
502 (format #t "{-1},{-1},{-1}"))
503 (format #t "}, /* ~4,'0x */\n" (ucd-extended-case-map-code e))))
504 (format #t "};\n\n"))
505 (define (extended-special-array lis)
506 (apply format "{~a,~a,~a,-1}" (map (^i (list-ref lis i -1)) '(0 1 2))))
507
508 ;; body of generate-case-tables
509 (print "#define NO_CASE_MAPPING SCM_CHAR_NO_CASE_MAPPING")
510 (print "#define TOLOWER(x) SCM_CHAR_CASEMAP_TOLOWER(x)")
511 (print "#define TOUPPER(x) SCM_CHAR_CASEMAP_TOUPPER(x)")
512 (print "#define EXTENDED(x) SCM_CHAR_CASEMAP_EXTENDED(x)")
513 (let1 subtable-num-alist (gather-toptable)
514 (emit-extended-case-maps)
515 (emit-subtables)
516 (emit-toptable subtable-num-alist))
517 )
518
519 (define (generate-bisect entries gen-value)
520 (define (bisect lo hi indent)
521 (if (= (+ lo 1) hi)
522 (format #t "~v,a~a\n" (* 2 indent) " " (gen-value (~ entries lo)))
523 (let1 mid (div (+ lo hi) 2)
524 (format #t "~v,aif (code < 0x~x) {\n"
525 (* 2 indent) " " (car (~ entries mid)))
526 (bisect lo mid (+ indent 1))
527 (format #t "~v,a} else {\n" (* 2 indent) " ")
528 (bisect mid hi (+ indent 1))
529 (format #t "~v,a}\n" (* 2 indent) " "))))
530 (bisect 0 (size-of entries) 1))
531
532 ;; Digit-value tables. Note that we'll have a shortcut for the first
533 ;; chunk [0x30, 0x39], so we only generate for the second chunk and after
534 (define (generate-digit-value-tables db)
535 (let1 ranges ($ reverse
536 $ fold (^[p s] (match s
537 [() `(,p (0 . #f))]
538 [((_ . last) . rest)
539 (if (= (car p) last)
540 `(,p ,@s)
541 `(,p (,last . #f) ,@s))]))
542 '()
543 $ cdr $ (cut sort-by <> car) $ filter identity
544 $ ucd-map-entries db
545 (^[code entry]
546 (and (eq? (ucd-entry-category entry) 'Nd)
547 (zero? (ucd-entry-digit-value entry))
548 (cons code (+ code 10)))))
549
550 (print)
551 (print "static int ucs_digit_value(ScmChar code)")
552 (print "{")
553 (dolist [r ranges]
554 (format #t " /* ~5,'0x- ~a */\n" (car r) (if (cdr r) "Nd" "* ")))
555 (generate-bisect (coerce-to <vector> ranges)
556 (^e (if (cdr e)
557 (format "return (code - 0x~x);" (car e))
558 "return -1;")))
559 (print "}")))
560
561 ;; Predefined character sets
562
563 (define (for-each-char-code proc)
564 (dotimes [n #x2ffff] (proc n n))
565 (proc #xe0000 #xe007f) ; Language tags; Cf
566 (proc #xe0100 #xe01ef) ; Variation selectors; Mn
567 (proc #xf0000 #xffffd) ; Private use; Co
568 (proc #x100000 #x10fffd) ; Private use; Co
569 )
570
571 (define (build-code-sets db categories)
572 (define sets (make-hash-table 'eq?))
573 (define (register set cat n m)
574 (and-let1 e (ucd-get-entry db n)
575 (if (eq? cat (ucd-entry-category e))
576 (if (= n m)
577 (add-code! set n)
578 (add-code-range! set n m)))))
579 ;; general category charsets (char-set:Lt etc.)
580 (dolist [cat categories]
581 (let1 cs (make <char-code-set> :name cat)
582 (for-each-char-code (cut register cs cat <> <>))
583 (hash-table-put! sets cat cs)))
584 ;; general category class charsets (char-set:L etc.)
585 (dolist [gcats (group-collection categories
586 :key (^c (string-ref (symbol->string c) 0)))]
587 (let1 cs
588 (make <char-code-set>
589 :name (string->symbol (substring (symbol->string (car gcats)) 0 1)))
590 (for-each (^c (for-each-char-code (cut register cs c <> <>))) gcats)
591 (hash-table-put! sets (~ cs'name) cs)))
592 ;; SRFI-14 charsets (the ones that has equivalent set in general category set
593 ;; is handled implicitly.
594 (hash-table-put! sets 'ASCII_UPPER
595 (rlet1 cs (make <char-code-set> :name 'ASCII_UPPER)
596 (add-code-range! cs
597 (char->integer #\A)
598 (char->integer #\Z))))
599 (hash-table-put! sets 'ASCII_LOWER
600 (rlet1 cs (make <char-code-set> :name 'ASCII_LOWER)
601 (add-code-range! cs
602 (char->integer #\a)
603 (char->integer #\z))))
604 (hash-table-put! sets 'ASCII_LETTER
605 (code-set-union 'ASCII_LETTER
606 (hash-table-ref sets 'ASCII_UPPER)
607 (hash-table-ref sets 'ASCII_LOWER)))
608 (hash-table-put! sets 'ASCII_DIGIT
609 (rlet1 cs (make <char-code-set> :name 'ASCII_DIGIT)
610 (add-code-range! cs
611 (char->integer #\0)
612 (char->integer #\9))))
613 (hash-table-put! sets 'LETTER_DIGIT
614 (code-set-union 'LETTER_DIGIT
615 (hash-table-ref sets 'L)
616 (hash-table-ref sets 'Nd)))
617 (hash-table-put! sets 'ASCII_LETTER_DIGIT
618 (code-set-union 'ASCII_LETTER_DIGIT
619 (hash-table-ref sets 'ASCII_LETTER)
620 (hash-table-ref sets 'ASCII_DIGIT)))
621 (hash-table-put! sets 'ASCII_WHITESPACE
622 (rlet1 cs (make <char-code-set> :name 'ASCII_WHITESPACE)
623 (add-code-range! cs 9 13) ;TAB,LF,LTAB,FF,CR
624 (add-code! cs (char->integer #\space))))
625 (hash-table-put! sets 'WHITESPACE
626 (code-set-union 'WHITESPACE
627 (hash-table-ref sets 'ASCII_WHITESPACE)
628 (hash-table-ref sets 'Z)))
629 (hash-table-put! sets 'ASCII_BLANK
630 (rlet1 cs (make <char-code-set> :name 'ASCII_BLANK)
631 (add-code! cs 9) ;TAB
632 (add-code! cs (char->integer #\space))))
633 (hash-table-put! sets 'BLANK
634 (code-set-union 'BLANK
635 (hash-table-ref sets 'ASCII_BLANK)
636 (hash-table-ref sets 'Zs)))
637 (hash-table-put! sets 'ASCII_PUNCTUATION
638 (rlet1 cs (make <char-code-set> :name 'ASCII_PUNCTUATION)
639 (add-code! cs 33) ;!
640 (add-code! cs 34) ;"
641 (add-code! cs 35) ;#
642 (add-code! cs 37) ;%
643 (add-code! cs 38) ;&
644 (add-code! cs 39) ;'
645 (add-code! cs 40) ;(
646 (add-code! cs 41) ;)
647 (add-code! cs 42) ;*
648 (add-code! cs 44) ;,
649 (add-code! cs 45) ;-
650 (add-code! cs 46) ;.
651 (add-code! cs 47) ;/
652 (add-code! cs 58) ;:
653 (add-code! cs 59) ;;
654 (add-code! cs 63) ;?
655 (add-code! cs 64) ;@
656 (add-code! cs 91) ;[
657 (add-code! cs 92) ;\
658 (add-code! cs 93) ;)
659 (add-code! cs 95) ;_
660 (add-code! cs 123) ;{
661 (add-code! cs 125) ;}
662 ))
663 (hash-table-put! sets 'ASCII_SYMBOL
664 (rlet1 cs (make <char-code-set> :name 'ASCII_SYMBOL)
665 (add-code! cs 36) ;$
666 (add-code! cs 43) ;+
667 (add-code! cs 60) ;<
668 (add-code! cs 61) ;=
669 (add-code! cs 62) ;>
670 (add-code! cs 94) ;^
671 (add-code! cs 96) ;`
672 (add-code! cs 124) ;|
673 (add-code! cs 126) ;~
674 ))
675 (hash-table-put! sets 'GRAPHIC
676 (code-set-union 'GRAPHIC
677 (hash-table-ref sets 'L)
678 (hash-table-ref sets 'N)
679 (hash-table-ref sets 'P)
680 (hash-table-ref sets 'S)))
681 (hash-table-put! sets 'ASCII_GRAPHIC
682 (code-set-union 'ASCII_GRAPHIC
683 (hash-table-ref sets 'ASCII_LETTER)
684 (hash-table-ref sets 'ASCII_DIGIT)
685 (hash-table-ref sets 'ASCII_PUNCTUATION)
686 (hash-table-ref sets 'ASCII_SYMBOL)))
687 (hash-table-put! sets 'PRINTING
688 (code-set-union 'PRINTING
689 (hash-table-ref sets 'GRAPHIC)
690 (hash-table-ref sets 'WHITESPACE)))
691 (hash-table-put! sets 'ASCII_PRINTING
692 (code-set-union 'ASCII_PRINTING
693 (hash-table-ref sets 'ASCII_GRAPHIC)
694 (hash-table-ref sets 'ASCII_WHITESPACE)))
695 (hash-table-put! sets 'ASCII_CONTROL
696 (rlet1 cs (make <char-code-set> :name 'ASCII_CONTROL)
697 (add-code-range! cs #x00 #x1f)
698 (add-code! cs #x7f)))
699 (hash-table-put! sets 'HEX_DIGIT
700 (rlet1 cs (make <char-code-set> :name 'HEX_DIGIT)
701 (add-code-range! cs
702 (char->integer #\0)
703 (char->integer #\9))
704 (add-code-range! cs
705 (char->integer #\A)
706 (char->integer #\F))
707 (add-code-range! cs
708 (char->integer #\a)
709 (char->integer #\f))))
710 (hash-table-put! sets 'ASCII
711 (rlet1 cs (make <char-code-set> :name 'ASCII)
712 (add-code-range! cs 0 127)))
713 (hash-table-put! sets 'ASCII_WORD
714 (rlet1 cs (make <char-code-set> :name 'ASCII_WORD)
715 (add-code-range! cs
716 (char->integer #\0)
717 (char->integer #\9))
718 (add-code-range! cs
719 (char->integer #\A)
720 (char->integer #\Z))
721 (add-code-range! cs
722 (char->integer #\a)
723 (char->integer #\z))
724 (add-code! cs (char->integer #\_))))
725 (hash-table-put! sets 'LC
726 (code-set-union 'LC
727 (hash-table-ref sets 'Lt)
728 (hash-table-ref sets 'Ll)
729 (hash-table-ref sets 'Lu)))
730 (hash-table-put! sets 'EMPTY (make <char-code-set> :name 'EMPTY))
731 sets)
732
733 ;; Break property values
734 ;; This goes to src/gauche/priv/unicode_attr.h and ext/gauche/unicode-attr.scm
735 ;;
736 ;; The data structure is two-level tables that maps codepoint 0-1FFFFF to
737 ;; an octet.
738 ;;
739 ;; The first level, {GB|WB}_break_table, is 512-length byte vector.
740 ;; Upper 9bit of codepoint is used to index this table.
741 ;; If the entry is [0..224], it's the index to the {GB|WB}_break_subtable[N][],
742 ;; which is indexed with the lower 8 bit of the codepoint.
743 ;; If the entry is above that, it indicates the entire block shares the
744 ;; same property.
745
746 (define (generate-break-tables db)
747 (define single-property-offset 224)
748 (define subtable-count 0)
749
750 (define (get-property entry type); type : GB | WB
751 (if entry
752 (case type
753 [(GB) (ucd-break-property-grapheme entry)]
754 [(WB) (ucd-break-property-word entry)])
755 'Other))
756
757 (define (gen-entry code type) ; type : GB | WB
758 (format #t " ~a_~a,\n" type
759 (get-property (ucd-get-break-property db code) type)))
760
761 ;; returns subtable-number
762 (define (gen-subtable start-code type)
763 (define kind #f)
764 (do-ec (: lb 256)
765 (let* ([e (ucd-get-break-property db (+ start-code lb))]
766 [p (get-property e type)])
767 (cond [(not kind) (set! kind p)]
768 [(eqv? kind p)]
769 [else (set! kind #t)])))
770 (if (symbol? kind)
771 ;; same property in the entire block
772 (format "(BOFF+~a_~a)" type kind)
773 (rlet1 subtable-num subtable-count
774 (format #t " {\n")
775 (do-ec (: lb 256) (gen-entry (+ start-code lb) type))
776 (format #t " },\n")
777 (inc! subtable-count))))
778
779 (define (gen-main-table type)
780 (set! subtable-count 0)
781 (print)
782 (print #"static unsigned char ~|type|_break_subtable[][256] = {")
783 (let1 nlist (list-ec (: n 512) (gen-subtable (* n 256) type))
784 (print "};")
785 (when (>= subtable-count single-property-offset)
786 (error #"~|type| subtable index overflow (~|subtable-count|)"))
787 (print)
788 (format #t "static unsigned char ~a_break_table[] = {" type)
789 (do-ec (:parallel (: n nlist) (:integers i))
790 (begin (when (zero? (mod i 4)) (format #t "\n "))
791 (format #t " ~3d," n)))
792 (format #t "\n};\n")))
793
794 ;; Generate table of symbols
795 (define (gen-symbol-table constants type)
796 (for-each-with-index (^[i c] (format #t "#define ~a_~a ~a\n" type c i))
797 constants))
798
799 (print)
800 (gen-symbol-table (ucd-grapheme-break-properties) 'GB)
801 (gen-symbol-table (ucd-word-break-properties) 'WB)
802 (print)
803 (print #"#define BOFF ~|single-property-offset|")
804 (gen-main-table 'GB)
805 (gen-main-table 'WB)
806 )
807
808 ;; Auxiliary stuff to find out optimal table configuration
809 ;; NB: It looks like 3-level table doesn't save at all.
810 (define (analyze-break-tables db :optional (lobits 4))
811 (define num-leaf-tables 0)
812 (define num-intermediate-tables 0)
813
814 ;; Check break properties of 2^lobits consecutive codepoints.
815 ;; Returns (<codepoint-string> . <property-or-#t>). If properties of
816 ;; all codeponds are the same, cdr is the property; otherwise it's #t.
817 (define (probe-lo start accessor)
818 (define kind #f)
819 (define (get-1 i)
820 (if-let1 e (ucd-get-break-property db (+ start i))
821 (accessor e)
822 'Other))
823 (do-ec (: i (expt 2 lobits))
824 (let1 kk (get-1 i)
825 (cond [(not kind) (set! kind kk)]
826 [(eqv? kind #t)]
827 [(eqv? kind kk)]
828 [else (set! kind #t)])))
829 (when (eqv? kind #t) (inc! num-leaf-tables))
830 (cons (format "~5,'0x" start) kind))
831 ;; Check break properties of 16 consecutive 16 codepoint blocks.
832 (define (probe-hi start accessor)
833 (define kind #f)
834 (define ps
835 (list-ec (: ii (expt 2 (- 8 lobits)))
836 (rlet1 p (probe-lo (+ start (ash ii lobits)) accessor)
837 (cond [(not kind) (set! kind (cdr p))]
838 [(eqv? kind (cdr p))]
839 [else (set! kind #t)]))))
840 (case kind
841 [(#t)
842 (format #t "~3,'0xXX -------------\n" (ash start -8))
843 (for-each (^p (format #t " ~a ~a\n" (car p) (cdr p)))
844 (remove (^p (eq? (cdr p) 'Other)) ps))
845 (inc! num-intermediate-tables)]
846 [(Other)]
847 [else
848 (format #t "~3,'0xXX ~a\n" (ash start -8) kind)]))
849
850 (define (probe-all name accessor)
851 (format #t "~a distribution\n" name)
852 (set! num-leaf-tables 0)
853 (set! num-intermediate-tables 0)
854 (do-ec (: u #x200)
855 (probe-hi (ash u 8) accessor))
856 (list name num-leaf-tables num-intermediate-tables))
857
858 (dolist [rr (list (probe-all "GB" ucd-break-property-grapheme)
859 (probe-all "WB" ucd-break-property-word))]
860 (format #t "~a # int. tables = ~d, # leaf tables = ~d\n"
861 (car rr) (cadr rr) (caddr rr)))
862 )
863
864 (define (generate-break-symbols db)
865 (define (gen-symbols constants type)
866 (for-each-with-index
867 (^[i c] (pprint `(define-constant ,(symbol-append type '_ c) ,i)))
868 constants)
869 (pprint `(define-constant ,(symbol-append '* type '-properties*)
870 ',(map-with-index (^[i c] (cons c i)) constants))))
871
872 (gen-symbols (ucd-grapheme-break-properties) 'GB)
873 (gen-symbols (ucd-word-break-properties) 'WB)
874 )
875
876 ;; EastAsianWidth property values
877 ;; This goes to src/gauche/priv/unicode_attr.h
878 ;;
879 ;; The data structure is two-level tables that maps codepoint 0-1FFFFF to
880 ;; an octet.
881 ;;
882 ;; The first level, width_table, is 512-length byte vector.
883 ;; Upper 9bit of codepoint is used to index this table. If the entry
884 ;; is one of WIDTH_x values, all codepoints of that range has that property
885 ;; value.
886 ;;
887 ;; Otherwise, the entry value - NUM_WIDTH_PROPERTIES points to the subtable,
888 ;; width_subtable[N][]. It is a vecto of nibbles; to look up, first take
889 ;; 1-7bit of the codepoint and look up an octet; if the LSB of codepoint
890 ;; is 0, take lower nibble; otherwise, take upper nibble.
891
892 (define (generate-width-tables db)
893 (define prop-count (length (ucd-east-asian-widths)))
894 (define subtable-count prop-count)
895
896 ;; returns subtable-number
897 (define (gen-subtable start-code)
898 (let1 e (ucd-get-east-asian-width db start-code)
899 (if (any?-ec (: lb 256)
900 (not (eq? (ucd-get-east-asian-width db (+ start-code lb)) e)))
901 (rlet1 subtable-num subtable-count
902 (print " {")
903 (do-ec (: c 128)
904 (let ([w0 (ucd-get-east-asian-width db (+ start-code (* c 2)))]
905 [w1 (ucd-get-east-asian-width db (+ start-code (* c 2) 1))])
906 (print #" WIDTH_ENTRY(WIDTH_~|w1|, WIDTH_~|w0|),")))
907 (print " },")
908 (inc! subtable-count))
909 ;; all codepoints of this range shares the value.
910 (find-index (cut eqv? e <>) (ucd-east-asian-widths)))))
911
912 ;; Generate table of symbols
913 (define (gen-symbol-table)
914 (for-each-with-index (^(i c) (format #t "#define WIDTH_~a ~a\n" c i))
915 (ucd-east-asian-widths))
916 (print #"#define NUM_WIDTH_PROPERTIES ~|prop-count|")
917 (print)
918 (print "static void init_WIDTH_symbols(ScmModule *mod) {")
919 (print #" ScmObj v = Scm_MakeVector(~|prop-count|, SCM_FALSE);")
920 (for-each-with-index
921 (^[i c]
922 (print #" SCM_VECTOR_ELEMENT(v, ~|i|) = SCM_INTERN(\"~|c|\");"))
923 (ucd-east-asian-widths))
924 (print " Scm_DefineConst(mod, \
925 SCM_SYMBOL(SCM_INTERN(\"*east-asian-widths*\")), \
926 v);")
927 (print "}"))
928
929 (print)
930 (gen-symbol-table)
931 (print)
932 (print "#define WIDTH_ENTRY(a, b) (((a)<<4) | (b))")
933 (print)
934 (print "static unsigned char width_subtable[][128] = {")
935 (let1 nlist (list-ec (: n 512) (gen-subtable (* n 256)))
936 (print "};")
937 (print)
938 (format #t "static unsigned char width_table[] = {")
939 (do-ec (:parallel (: n nlist) (:integers i))
940 (begin (when (zero? (mod i 8)) (format #t "\n "))
941 (format #t " ~3d," n)))
942 (format #t "\n};\n"))
943 )