| 1 |
;; -*- coding:utf-8 -*- |
| 2 |
;;; |
| 3 |
;;; gen-unicode.scm - generate unicode-handling tables |
| 4 |
;;; |
| 5 |
;;; Originally written by Shiro Kawai, 2011 |
| 6 |
;;; Public Domain - use as you like. |
| 7 |
;;; |
| 8 |
|
| 9 |
;; Reads Unicode data tables and generates various source files. |
| 10 |
|
| 11 |
;; This script can serve three operations. |
| 12 |
;; |
| 13 |
;; (1) Generate unicode-data.scm from Unicode character database |
| 14 |
;; It is only necessary when new version of Unicode is published, |
| 15 |
;; and the resulting unicode-data.scm is checked in to the source |
| 16 |
;; tree so that other developers don't need UCD. |
| 17 |
;; |
| 18 |
;; gosh ./gen-unicode.scm --import <unicode-database-directory> unicode-data.scm [unicode-test.scm] |
| 19 |
;; |
| 20 |
;; (2) Generate source files from unicode-data.scm |
| 21 |
;; To reduce runtime overhead, unicode properties are saved in binary |
| 22 |
;; tables, within the following two generated source files: |
| 23 |
;; |
| 24 |
;; char_attr.c - General category and case mappings. |
| 25 |
;; gauche/priv/unicode_attr.h - Grapheme break, word break, East-asian width |
| 26 |
;; ext/gauche/unicode-attr.scm - Break property symbols |
| 27 |
;; |
| 28 |
;; This is done when you build from git source tree. |
| 29 |
;; |
| 30 |
;; gosh ./gen-unicode.scm --compile unicode-data.scm |
| 31 |
;; |
| 32 |
;; (3) Fetch Unicode character database files. This is for developer's |
| 33 |
;; convenience. |
| 34 |
;; |
| 35 |
;; gosh ./gen-unicode.scm --fetch <unicode-database-directory> [<unicode-version> |
| 36 |
;; Note that the original content in <unicode-database-directory> is |
| 37 |
;; overwritten. |
| 38 |
;; You might need to adjust code for newer version of Unicode. |
| 39 |
;; See doc/HOWTO-unicode.txt for the details. |
| 40 |
|
| 41 |
(use file.util) |
| 42 |
(use gauche.charconv) |
| 43 |
(use gauche.dictionary) |
| 44 |
(use gauche.generator) |
| 45 |
(use gauche.record) |
| 46 |
(use gauche.sequence) |
| 47 |
(use gauche.uvector) |
| 48 |
(use scheme.list) |
| 49 |
(use srfi.13) |
| 50 |
(use srfi.42) |
| 51 |
(use text.csv) |
| 52 |
(use util.match) |
| 53 |
|
| 54 |
;; needs to load from source |
| 55 |
(add-load-path "../lib" :relative) |
| 56 |
(use text.unicode.ucd) |
| 57 |
(use text.unicode.codeset) |
| 58 |
|
| 59 |
;; rfc.http is only required by '--fetch' operation. We don't want to load |
| 60 |
;; it during build process, so let's autoload it. |
| 61 |
(autoload rfc.http http-get) |
| 62 |
|
| 63 |
|
| 64 |
;; We generate four kind of lookup structures. Each structure consists |
| 65 |
;; of various types of tables in order to reduce the size. |
| 66 |
;; |
| 67 |
;; * General categories and case bits |
| 68 |
;; |
| 69 |
;; This structure maps Unicode codepoint to a byte containing the |
| 70 |
;; following info: |
| 71 |
;; |
| 72 |
;; bit 7-6 - Alphabetic and case info |
| 73 |
;; 00 : non-alphabetic char |
| 74 |
;; 01 : lowercase alphabetic char |
| 75 |
;; 10 : uppercase alphabetic char |
| 76 |
;; 11 : caseless or titlecase alphabetic char |
| 77 |
;; bit 5 - Reserved |
| 78 |
;; bit 4-0 - General category (Lu, Nd, Cc, etc.) |
| 79 |
;; See ucd-general-categories |
| 80 |
;; and the enum in src/gauche/char_attr.h |
| 81 |
;; |
| 82 |
;; For U+0000 to U+1ffff, we have a single table that directly maps |
| 83 |
;; the codepoint to the above byte. |
| 84 |
;; |
| 85 |
;; unsigned char ucs_general_category_00000[0x20000] (131072 bytes) |
| 86 |
;; |
| 87 |
;; Above U+20000 the mappings are sparse and mostly contiguous, so we |
| 88 |
;; generate a C code that performs a binary search. |
| 89 |
;; |
| 90 |
;; unsigned char ucs_general_category_20000(ScmChar code) |
| 91 |
;; |
| 92 |
;; * Case mapping |
| 93 |
;; |
| 94 |
;; Almost all characters that needs case mappings are below #xffff. In |
| 95 |
;; Unicode 6.0, the only exceptions are 80 characters in U+104xx range |
| 96 |
;; (Deseret). |
| 97 |
;; |
| 98 |
;; Among case-mapped characters, almost all of them requires a simple |
| 99 |
;; rule---that is, either a character can be lowercased by adding some |
| 100 |
;; small value to the codepoint, or a character can be upcased and |
| 101 |
;; titlecased by adding some small value to the codepoint. These have |
| 102 |
;; simple casemap entries. Characters that requires more complex handling |
| 103 |
;; have extended casemap entries. |
| 104 |
;; |
| 105 |
;; The tables first maps a codepoint into 16bit entry. The entry |
| 106 |
;; represents either a simple casemap entry, or an index to an |
| 107 |
;; extended casemap entry. |
| 108 |
;; |
| 109 |
;; bit15 == 0: |
| 110 |
;; This is a simple entry. uppercase and titlecase is the same, |
| 111 |
;; no special case mappings, and conversion between |
| 112 |
;; (upper,title) <-> lower is simply done by adding the given offset. |
| 113 |
;; |
| 114 |
;; If bit14 is 0, this letter is uppercase. Converting to uppercase and |
| 115 |
;; titlecase is noop. Converting to lowercase is to add |
| 116 |
;; the offset. |
| 117 |
;; If bit14 is 1, this letter is lowercase. Converting to lowercase is |
| 118 |
;; noop. Converting to uppercase and titlecase is to add |
| 119 |
;; the offset. |
| 120 |
;; bit13-bit0, singned integer offset [-8192, 8191] |
| 121 |
;; |
| 122 |
;; bit15 == 1: |
| 123 |
;; This is an extended entry (except #xffff). |
| 124 |
;; bit14-bit0 is an index to an extended entry table. |
| 125 |
;; |
| 126 |
;; #xffff indicates empty entry. |
| 127 |
;; |
| 128 |
;; Characters that require case mapping tend to cluster, so we use two-staged |
| 129 |
;; table to lookup the 16-bit entry from the codepoint below U+10000. |
| 130 |
;; |
| 131 |
;; The bit 15-8 is the index of this table: |
| 132 |
;; |
| 133 |
;; static unsiged char casemap_000[256] |
| 134 |
;; |
| 135 |
;; If the value is 255, the character doesn't have case mappings. |
| 136 |
;; Otherwise, let V be the value of the above lookup, the entry |
| 137 |
;; can be looked up by the following table: |
| 138 |
;; |
| 139 |
;; static unsiged char casemap_subtable[V][<lower 8 bit of codepoint>]; |
| 140 |
;; |
| 141 |
;; We only need to have 18 subtables. |
| 142 |
;; |
| 143 |
;; * Digit values |
| 144 |
;; |
| 145 |
;; Characters with categrory Nd has associated numeric value 0..9. |
| 146 |
;; A set of decimal numeric characters for 0..9 are always contiguous. |
| 147 |
;; As of Unicode 6.2, we have 42 of such sets. |
| 148 |
;; |
| 149 |
;; * Break properties |
| 150 |
;; |
| 151 |
;; This structure maps codepoint to the Grapheme_Cluster_Break, |
| 152 |
;; Word_Break and Sentence_Break properties. |
| 153 |
;; |
| 154 |
;; See http://www.unicode.org/reports/tr29/ |
| 155 |
;; http://www.unicode.org/Public/UNIDATA/auxiliary/GraphemeBreakProperty.txt |
| 156 |
;; http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt |
| 157 |
;; http://www.unicode.org/Public/UNIDATA/auxiliary/SentenceBreakProperty.txt |
| 158 |
;; |
| 159 |
;; NB: As of Unicode 13, we also need Extended_Poctographic (ExtPict) |
| 160 |
;; Emoji property to do clustering properly. For grapheme break |
| 161 |
;; properties, all ExtPict codepoints are GB_Other, so we simply name |
| 162 |
;; them GB_ExtPict. For word break properties, ExtPict falls on WB_Other |
| 163 |
;; and WB_ALetter. We Name the former WB_ExtPict, and the latter |
| 164 |
;; WB_ExtPict_ALetter. |
| 165 |
;; |
| 166 |
;; The definitive list of properties are in text.unicode.ucd, |
| 167 |
;; ucd-grapheme-break-properties and ucd-word-break-properties |
| 168 |
;; |
| 169 |
;; Codepoints below U+20000 are looked up by two-staged tables. |
| 170 |
;; First, look up this table with (codepoint >> 8). |
| 171 |
;; |
| 172 |
;; static unsigned char {GB|WB}_break_table[0x200] |
| 173 |
;; |
| 174 |
;; If the value is up to single-property-offset (224), the value is |
| 175 |
;; the index to the secondary table. |
| 176 |
;; |
| 177 |
;; static unsigned char {GB|WB}_break_subtable[index][256] |
| 178 |
;; |
| 179 |
;; The value of this table encodes GB|WB properties. |
| 180 |
;; If th evalue is equal to or more than single-property-offset, |
| 181 |
;; it indicates the entire block of 256 codepoints shared the same |
| 182 |
;; property, (- value single-property-offset). |
| 183 |
;; |
| 184 |
;; Codepoints on or above U+20000 are all 'Other', except the following |
| 185 |
;; ranges. They are handled specially in the lookup procedure. |
| 186 |
;; |
| 187 |
;; E0001 GB_Control, WB_Format |
| 188 |
;; E0020..E007F GB_Control, WB_Format |
| 189 |
;; E0100..E01EF GB_Extend, WB_Extend |
| 190 |
;; |
| 191 |
;; |
| 192 |
;; * Emoji Properties |
| 193 |
;; |
| 194 |
;; Also http://unicode.org/reports/tr51 |
| 195 |
;; https://www.unicode.org/Public/13.0.0/ucd/emoji/emoji-data.txt |
| 196 |
;; |
| 197 |
|
| 198 |
;;; |
| 199 |
;;; Main entry |
| 200 |
;;; |
| 201 |
|
| 202 |
(define (main args) |
| 203 |
(match (cdr args) |
| 204 |
[("--fetch" dir . maybe-version) (apply fetch-ucd dir maybe-version)] |
| 205 |
[("--import" dir ucdfile) (import-data dir ucdfile #f)] |
| 206 |
[("--import" dir ucdfile testfile) (import-data dir ucdfile testfile)] |
| 207 |
[("--analyze" ucdfile lobits) |
| 208 |
(analyze-break-tables (call-with-input-file ucdfile ucd-load-db) |
| 209 |
(string->number lobits))] |
| 210 |
[("--compile" ucdfile) |
| 211 |
(unless (file-exists? ucdfile) |
| 212 |
(exit 1 "Couldn't open unicode data file: ~a" ucdfile)) |
| 213 |
(generate-tables (call-with-input-file ucdfile ucd-load-db))] |
| 214 |
[else |
| 215 |
(exit 1 "Usage:\n\ |
| 216 |
gen-unicode.scm --fetch <unicode-database-dir> [<unicode-version>]\n\ |
| 217 |
gen-unicode.scm --import <unicode-database-dir> <ucd-file>\n\ |
| 218 |
gen-unicode.scm --compile <unicode-data.scm>")]) |
| 219 |
0) |
| 220 |
|
| 221 |
;;; |
| 222 |
;;; Fetching |
| 223 |
;;; |
| 224 |
(define (fetch-ucd dir :optional (version #f)) |
| 225 |
(let ([path (if version |
| 226 |
#"/Public/~|version|/ucd" |
| 227 |
"/Public/UCD/latest/ucd")] |
| 228 |
[datafiles '("UnicodeData.txt" |
| 229 |
"SpecialCasing.txt" |
| 230 |
"PropList.txt" |
| 231 |
"EastAsianWidth.txt" |
| 232 |
"auxiliary/GraphemeBreakProperty.txt" |
| 233 |
"auxiliary/SentenceBreakProperty.txt" |
| 234 |
"auxiliary/WordBreakProperty.txt" |
| 235 |
"auxiliary/GraphemeBreakTest.txt" |
| 236 |
"auxiliary/SentenceBreakTest.txt" |
| 237 |
"auxiliary/WordBreakTest.txt" |
| 238 |
"emoji/emoji-data.txt")]) |
| 239 |
(make-directory* (build-path dir "auxiliary")) |
| 240 |
(make-directory* (build-path dir "emoji")) |
| 241 |
(dolist [f datafiles] |
| 242 |
(let1 p (build-path dir f) |
| 243 |
(display #"Getting ~|f|... ") (flush) |
| 244 |
(call-with-output-file p |
| 245 |
(^o (http-get "www.unicode.org" (build-path path f) |
| 246 |
:sink o :flusher (^ _ #t)))) |
| 247 |
(display "done\n"))))) |
| 248 |
|
| 249 |
;;; |
| 250 |
;;; Importing |
| 251 |
;;; |
| 252 |
|
| 253 |
;; Dump unicode character data in ucdfile. |
| 254 |
(define (import-data dir ucdfile testfile) |
| 255 |
(unless (file-is-directory? dir) |
| 256 |
(exit 1 "Directory required, but got: ~a" dir)) |
| 257 |
(let1 ucd-db (ucd-parse-files dir) |
| 258 |
(with-output-to-file ucdfile |
| 259 |
(cut ucd-save-db ucd-db)) |
| 260 |
(when testfile |
| 261 |
(with-output-to-file testfile |
| 262 |
(^[] (import-break-test ucd-db dir)))))) |
| 263 |
|
| 264 |
(define (import-break-test ucd-db dir) |
| 265 |
;; break test file format: |
| 266 |
;; <line>: <sep> (<code> <sep>)+ '#' <comment> |
| 267 |
;; <sep> : '÷' | '×' |
| 268 |
;; <code> : #[[:xdigit:]]+ |
| 269 |
;; |
| 270 |
;; returns (<count> <seq> <comment>) |
| 271 |
;; where |
| 272 |
;; <count> : integer, to make it easy to find tests |
| 273 |
;; <seq> : <bool> (<integer> <bool>)+ |
| 274 |
;; <bool> : #t as breakable (÷), #f as unbreakable (×) |
| 275 |
(define (parse-1 line) |
| 276 |
(and-let1 m (#/^([÷×])\s+/ line) |
| 277 |
(let loop ([line (m'after)] [r (list (break-flag (m 1)))]) |
| 278 |
(rxmatch-case line |
| 279 |
[#/^([[:xdigit:]]+)\s+([÷×])\s+(.*)/ (_ cc ff rr) |
| 280 |
(loop rr (list* (break-flag ff) (string->number cc 16) r))] |
| 281 |
[#/^#\s+(.*)/ (_ comment) (list (reverse r) comment)] |
| 282 |
[else (list (reverse r) "")])))) |
| 283 |
(define (break-flag s) (equal? s "÷")) |
| 284 |
(define (parse name file) |
| 285 |
(let1 path (build-path dir file) |
| 286 |
(print) |
| 287 |
(print ";; " file) |
| 288 |
(print #"(define ~name '(") |
| 289 |
(generator-for-each (^[r k] (pprint (cons k r))) |
| 290 |
(gfilter-map parse-1 (file->line-generator path)) |
| 291 |
(giota)) |
| 292 |
(print "))"))) |
| 293 |
(print ";; -*- coding:utf-8 -*-") |
| 294 |
(print ";; Generated from Unicode " (ucd-version ucd-db) " test data files") |
| 295 |
(parse '*grapheme-break-tests* "auxiliary/GraphemeBreakTest.txt") |
| 296 |
(parse '*word-break-tests* "auxiliary/WordBreakTest.txt") |
| 297 |
(parse '*sentence-break-tests* "auxiliary/SentenceBreakTest.txt") |
| 298 |
) |
| 299 |
|
| 300 |
;;; |
| 301 |
;;; Generate source file |
| 302 |
;;; |
| 303 |
|
| 304 |
(define (generate-tables db) |
| 305 |
(let-values ([(c.p char_attr.c) (sys-mkstemp "char_attr.c.")] |
| 306 |
[(h.p unicode_attr.h) (sys-mkstemp "unicode_attr.h.")] |
| 307 |
[(scm.p unicode-attr.scm) (sys-mkstemp "unicode-attr.scm.")]) |
| 308 |
(with-output-to-port c.p |
| 309 |
(^() (preamble db "/*" "*/") |
| 310 |
(generate-category-tables db) |
| 311 |
(generate-case-tables db) |
| 312 |
(generate-digit-value-tables db))) |
| 313 |
(with-output-to-port h.p |
| 314 |
(^() (preamble db "/*" "*/") |
| 315 |
(generate-break-tables db) |
| 316 |
(generate-width-tables db))) |
| 317 |
(with-output-to-port scm.p |
| 318 |
(^() (preamble db ";;" "") |
| 319 |
(generate-break-symbols db))) |
| 320 |
(close-port c.p) |
| 321 |
(close-port h.p) |
| 322 |
(close-port scm.p) |
| 323 |
(make-directory* "gauche/priv") ; in case we're building out-of-tree |
| 324 |
(sys-rename char_attr.c "char_attr.c") |
| 325 |
(sys-rename unicode_attr.h "gauche/priv/unicode_attr.h") |
| 326 |
(sys-rename unicode-attr.scm "../ext/gauche/unicode-attr.scm"))) |
| 327 |
|
| 328 |
(define (preamble db op cl) |
| 329 |
(print #"~|op| Generated automatically from Unicode character database ~|cl|") |
| 330 |
(print #"~|op| See src/gen-unicode.scm for the description of data structures. ~|cl|") |
| 331 |
(print #"~|op| Unicode version ~(ucd-version db). Do not edit. ~|cl|") |
| 332 |
) |
| 333 |
|
| 334 |
(define (generate-category-tables db) |
| 335 |
(define (emit-entry e) |
| 336 |
(if e |
| 337 |
(cond |
| 338 |
[(ucd-entry-uppercase e) (format #t " U(~a)," (ucd-entry-category e))] |
| 339 |
[(ucd-entry-lowercase e) (format #t " L(~a)," (ucd-entry-category e))] |
| 340 |
[(ucd-entry-alphabetic e) (format #t " A(~a)," (ucd-entry-category e))] |
| 341 |
[else (format #t " ~a," (ucd-entry-category e))]) |
| 342 |
(format #t " Cn,"))) |
| 343 |
(define (emit-columns start end p) |
| 344 |
(do-ec (:parallel (: k start end) |
| 345 |
(:integers i)) |
| 346 |
(begin |
| 347 |
(cond [(zero? (mod i 256)) (format #t "\n /* ~5,'0x - */\n" k)] |
| 348 |
[(zero? (mod i 8)) (newline)]) |
| 349 |
(p k))) |
| 350 |
(print)) |
| 351 |
(define (ucs->entry ucs) (ucd-get-entry db ucs)) |
| 352 |
|
| 353 |
;; Setup |
| 354 |
(dolist [c (ucd-general-categories)] |
| 355 |
(format #t "#define ~a SCM_CHAR_CATEGORY_~a\n" c c)) |
| 356 |
(print "#undef A") |
| 357 |
(print "#undef U") |
| 358 |
(print "#undef L") |
| 359 |
(print "#define A(x) ((x)|SCM_CHAR_ALPHABETIC_BITS)") |
| 360 |
(print "#define U(x) ((x)|SCM_CHAR_UPPERCASE_BITS)") |
| 361 |
(print "#define L(x) ((x)|SCM_CHAR_LOWERCASE_BITS)") |
| 362 |
|
| 363 |
(let ([code-sets (build-code-sets db (ucd-general-categories))]) |
| 364 |
;; U+0000 - U+1ffff : direct table lookup |
| 365 |
(display "static unsigned char ucs_general_category_00000[] = {") |
| 366 |
(emit-columns 0 #x20000 (^i (emit-entry (ucs->entry i)))) |
| 367 |
(print "};") |
| 368 |
;; Over U+20000 - binary search |
| 369 |
(print) |
| 370 |
(print "static unsigned char ucs_general_category_20000(ScmChar code)") |
| 371 |
(print "{") |
| 372 |
(print " /*") |
| 373 |
(for-each (^e (format #t " ~6x ~a\n" (car e) (cdr e))) |
| 374 |
(ucd-get-category-ranges db)) |
| 375 |
(print " */") |
| 376 |
(generate-bisect (coerce-to <vector> (ucd-get-category-ranges db)) |
| 377 |
(^e (format "return ~a;" |
| 378 |
(case (cdr e) |
| 379 |
[(Lo) "A(Lo)"] |
| 380 |
[(#f) "Cn"] |
| 381 |
[else => identity])))) |
| 382 |
(print "}") |
| 383 |
(for-each |
| 384 |
(^s (dump-code-set-in-C (cdr s))) |
| 385 |
(sort (hash-table->alist code-sets) string<? (^x (x->string (car x))))) |
| 386 |
(print)) |
| 387 |
|
| 388 |
;; Bind predefined charset |
| 389 |
;; The CAT entry can be just a symbol or (alias symbol). |
| 390 |
(print "static void init_predefined_charsets() {") |
| 391 |
(dolist [cat (append (ucd-general-categories) |
| 392 |
'(L LC M N P S Z C |
| 393 |
(LETTER L) ASCII_LETTER |
| 394 |
(DIGIT Nd) ASCII_DIGIT |
| 395 |
LETTER_DIGIT ASCII_LETTER_DIGIT |
| 396 |
(LOWER Ll) ASCII_LOWER |
| 397 |
(UPPER Lu) ASCII_UPPER |
| 398 |
(TITLE Lt) |
| 399 |
GRAPHIC ASCII_GRAPHIC |
| 400 |
PRINTING ASCII_PRINTING |
| 401 |
(PUNCTUATION P) ASCII_PUNCTUATION |
| 402 |
(SYMBOL S) ASCII_SYMBOL |
| 403 |
(ISO_CONTROL Cc) ASCII_CONTROL |
| 404 |
HEX_DIGIT |
| 405 |
WHITESPACE ASCII_WHITESPACE |
| 406 |
BLANK ASCII_BLANK |
| 407 |
ASCII EMPTY |
| 408 |
ASCII_WORD (WORD ASCII_WORD)))] |
| 409 |
(let1 setname (if (pair? cat) (car cat) cat) |
| 410 |
(print #" predef_sets[SCM_CHAR_SET_~|setname|] =") |
| 411 |
(if (pair? cat) |
| 412 |
(print #" predef_sets[SCM_CHAR_SET_~(cadr cat)];") |
| 413 |
(print #" make_charset_~|setname|();")))) |
| 414 |
(print "}") |
| 415 |
|
| 416 |
;; Teardown |
| 417 |
(dolist [c (ucd-general-categories)] |
| 418 |
(format #t "#undef ~a\n" c)) |
| 419 |
(format #t "#undef A\n") |
| 420 |
(format #t "#undef U\n") |
| 421 |
(format #t "#undef L\n") |
| 422 |
(print)) |
| 423 |
|
| 424 |
(define (generate-case-tables db) |
| 425 |
(define subtables '()) |
| 426 |
(define extended '()) |
| 427 |
;; returns a list of (code . ucd-entry-case-map) with U+HHHxx where HHH |
| 428 |
;; is given to the hi arg. |
| 429 |
(define (gather-entries hb) |
| 430 |
(fold-ec '() (: lb 256) (ucd-get-entry db (+ (* hb 256) lb)) |
| 431 |
(^(e seed) (if (or (not e) (not (ucd-entry-case-map e))) |
| 432 |
seed |
| 433 |
(acons (+ (* hb 256) lb) (ucd-entry-case-map e) seed))))) |
| 434 |
;; ucd-entry-alist :: ((code . ucd-entry-case-map) ...) |
| 435 |
;; subtable :: (start-code . #(....)) |
| 436 |
;; returns (start-code . subtable-number) |
| 437 |
(define (gen-subtable start-code ucd-entry-alist) |
| 438 |
(if (null? ucd-entry-alist) |
| 439 |
`(,start-code . 255) |
| 440 |
(let* ([table-num (length subtables)] |
| 441 |
[vec (make-vector 256 #f)]) |
| 442 |
(dolist [p ucd-entry-alist] |
| 443 |
(when (ucd-extended-case-map? (cdr p)) |
| 444 |
(push! extended (cdr p))) |
| 445 |
(set! (~ vec (logand (car p) #xff)) (cdr p))) |
| 446 |
(push! subtables (cons start-code vec)) |
| 447 |
(cons start-code table-num)))) |
| 448 |
;; |
| 449 |
(define (gather-toptable) |
| 450 |
(list-ec (: hi 256) (gen-subtable (* hi 256) (gather-entries hi)))) |
| 451 |
|
| 452 |
;; |
| 453 |
(define (emit-subtable subtable) |
| 454 |
(format #t " /* ~4,'0x - ~4,'0x */\n" |
| 455 |
(car subtable) |
| 456 |
(+ (car subtable) 255)) |
| 457 |
(format #t " {\n") |
| 458 |
(dotimes [n 256] |
| 459 |
(format #t " ~20a, /* 0x~4,'0x */\n" |
| 460 |
(if-let1 cmap (~ (cdr subtable) n) |
| 461 |
(if (ucd-simple-case-map? cmap) |
| 462 |
(format "~a(~a)" |
| 463 |
(if (eq? (ucd-simple-case-map-case cmap) 'upper) |
| 464 |
"TOLOWER" |
| 465 |
"TOUPPER") |
| 466 |
(ucd-simple-case-map-offset cmap)) |
| 467 |
(format "EXTENDED(~a)" |
| 468 |
(find-index (cut eq? cmap <>) extended))) |
| 469 |
"NO_CASE_MAPPING") |
| 470 |
(+ (car subtable) n))) |
| 471 |
(format #t " },\n")) |
| 472 |
|
| 473 |
(define (emit-subtables) |
| 474 |
(format #t "static unsigned short casemap_subtable[][256] = {\n") |
| 475 |
(for-each emit-subtable (reverse subtables)) |
| 476 |
(format #t "};\n\n")) |
| 477 |
|
| 478 |
(define (emit-toptable subtable-num-alist) |
| 479 |
(let1 vec (make-vector 256 255) |
| 480 |
(dolist [s subtable-num-alist] |
| 481 |
(set! (~ vec (div (car s) 256)) (cdr s))) |
| 482 |
(format #t "static unsigned char casemap_000[] = {") |
| 483 |
(dotimes [i 256] |
| 484 |
(when (zero? (mod i 16)) (newline)) |
| 485 |
(format #t " ~3d," (~ vec i))) |
| 486 |
(format #t "\n};\n"))) |
| 487 |
|
| 488 |
(define (emit-extended-case-maps) |
| 489 |
(format #t "static ScmCharCaseMap extended_casemaps[] = {\n") |
| 490 |
(dolist [e extended] |
| 491 |
(let ([simple (ucd-extended-case-map-simple-map e)] |
| 492 |
[special (ucd-extended-case-map-special-map e)]) |
| 493 |
(format #t " { ~s,~s,~s," |
| 494 |
(or (~ simple 0) 0) |
| 495 |
(or (~ simple 1) 0) |
| 496 |
(or (~ simple 2) 0)) |
| 497 |
(if special |
| 498 |
(format #t "~a,~a,~a" |
| 499 |
(extended-special-array (~ special 0)) |
| 500 |
(extended-special-array (~ special 1)) |
| 501 |
(extended-special-array (~ special 2))) |
| 502 |
(format #t "{-1},{-1},{-1}")) |
| 503 |
(format #t "}, /* ~4,'0x */\n" (ucd-extended-case-map-code e)))) |
| 504 |
(format #t "};\n\n")) |
| 505 |
(define (extended-special-array lis) |
| 506 |
(apply format "{~a,~a,~a,-1}" (map (^i (list-ref lis i -1)) '(0 1 2)))) |
| 507 |
|
| 508 |
;; body of generate-case-tables |
| 509 |
(print "#define NO_CASE_MAPPING SCM_CHAR_NO_CASE_MAPPING") |
| 510 |
(print "#define TOLOWER(x) SCM_CHAR_CASEMAP_TOLOWER(x)") |
| 511 |
(print "#define TOUPPER(x) SCM_CHAR_CASEMAP_TOUPPER(x)") |
| 512 |
(print "#define EXTENDED(x) SCM_CHAR_CASEMAP_EXTENDED(x)") |
| 513 |
(let1 subtable-num-alist (gather-toptable) |
| 514 |
(emit-extended-case-maps) |
| 515 |
(emit-subtables) |
| 516 |
(emit-toptable subtable-num-alist)) |
| 517 |
) |
| 518 |
|
| 519 |
(define (generate-bisect entries gen-value) |
| 520 |
(define (bisect lo hi indent) |
| 521 |
(if (= (+ lo 1) hi) |
| 522 |
(format #t "~v,a~a\n" (* 2 indent) " " (gen-value (~ entries lo))) |
| 523 |
(let1 mid (div (+ lo hi) 2) |
| 524 |
(format #t "~v,aif (code < 0x~x) {\n" |
| 525 |
(* 2 indent) " " (car (~ entries mid))) |
| 526 |
(bisect lo mid (+ indent 1)) |
| 527 |
(format #t "~v,a} else {\n" (* 2 indent) " ") |
| 528 |
(bisect mid hi (+ indent 1)) |
| 529 |
(format #t "~v,a}\n" (* 2 indent) " ")))) |
| 530 |
(bisect 0 (size-of entries) 1)) |
| 531 |
|
| 532 |
;; Digit-value tables. Note that we'll have a shortcut for the first |
| 533 |
;; chunk [0x30, 0x39], so we only generate for the second chunk and after |
| 534 |
(define (generate-digit-value-tables db) |
| 535 |
(let1 ranges ($ reverse |
| 536 |
$ fold (^[p s] (match s |
| 537 |
[() `(,p (0 . #f))] |
| 538 |
[((_ . last) . rest) |
| 539 |
(if (= (car p) last) |
| 540 |
`(,p ,@s) |
| 541 |
`(,p (,last . #f) ,@s))])) |
| 542 |
'() |
| 543 |
$ cdr $ (cut sort-by <> car) $ filter identity |
| 544 |
$ ucd-map-entries db |
| 545 |
(^[code entry] |
| 546 |
(and (eq? (ucd-entry-category entry) 'Nd) |
| 547 |
(zero? (ucd-entry-digit-value entry)) |
| 548 |
(cons code (+ code 10))))) |
| 549 |
|
| 550 |
(print) |
| 551 |
(print "static int ucs_digit_value(ScmChar code)") |
| 552 |
(print "{") |
| 553 |
(dolist [r ranges] |
| 554 |
(format #t " /* ~5,'0x- ~a */\n" (car r) (if (cdr r) "Nd" "* "))) |
| 555 |
(generate-bisect (coerce-to <vector> ranges) |
| 556 |
(^e (if (cdr e) |
| 557 |
(format "return (code - 0x~x);" (car e)) |
| 558 |
"return -1;"))) |
| 559 |
(print "}"))) |
| 560 |
|
| 561 |
;; Predefined character sets |
| 562 |
|
| 563 |
(define (for-each-char-code proc) |
| 564 |
(dotimes [n #x2ffff] (proc n n)) |
| 565 |
(proc #xe0000 #xe007f) ; Language tags; Cf |
| 566 |
(proc #xe0100 #xe01ef) ; Variation selectors; Mn |
| 567 |
(proc #xf0000 #xffffd) ; Private use; Co |
| 568 |
(proc #x100000 #x10fffd) ; Private use; Co |
| 569 |
) |
| 570 |
|
| 571 |
(define (build-code-sets db categories) |
| 572 |
(define sets (make-hash-table 'eq?)) |
| 573 |
(define (register set cat n m) |
| 574 |
(and-let1 e (ucd-get-entry db n) |
| 575 |
(if (eq? cat (ucd-entry-category e)) |
| 576 |
(if (= n m) |
| 577 |
(add-code! set n) |
| 578 |
(add-code-range! set n m))))) |
| 579 |
;; general category charsets (char-set:Lt etc.) |
| 580 |
(dolist [cat categories] |
| 581 |
(let1 cs (make <char-code-set> :name cat) |
| 582 |
(for-each-char-code (cut register cs cat <> <>)) |
| 583 |
(hash-table-put! sets cat cs))) |
| 584 |
;; general category class charsets (char-set:L etc.) |
| 585 |
(dolist [gcats (group-collection categories |
| 586 |
:key (^c (string-ref (symbol->string c) 0)))] |
| 587 |
(let1 cs |
| 588 |
(make <char-code-set> |
| 589 |
:name (string->symbol (substring (symbol->string (car gcats)) 0 1))) |
| 590 |
(for-each (^c (for-each-char-code (cut register cs c <> <>))) gcats) |
| 591 |
(hash-table-put! sets (~ cs'name) cs))) |
| 592 |
;; SRFI-14 charsets (the ones that has equivalent set in general category set |
| 593 |
;; is handled implicitly. |
| 594 |
(hash-table-put! sets 'ASCII_UPPER |
| 595 |
(rlet1 cs (make <char-code-set> :name 'ASCII_UPPER) |
| 596 |
(add-code-range! cs |
| 597 |
(char->integer #\A) |
| 598 |
(char->integer #\Z)))) |
| 599 |
(hash-table-put! sets 'ASCII_LOWER |
| 600 |
(rlet1 cs (make <char-code-set> :name 'ASCII_LOWER) |
| 601 |
(add-code-range! cs |
| 602 |
(char->integer #\a) |
| 603 |
(char->integer #\z)))) |
| 604 |
(hash-table-put! sets 'ASCII_LETTER |
| 605 |
(code-set-union 'ASCII_LETTER |
| 606 |
(hash-table-ref sets 'ASCII_UPPER) |
| 607 |
(hash-table-ref sets 'ASCII_LOWER))) |
| 608 |
(hash-table-put! sets 'ASCII_DIGIT |
| 609 |
(rlet1 cs (make <char-code-set> :name 'ASCII_DIGIT) |
| 610 |
(add-code-range! cs |
| 611 |
(char->integer #\0) |
| 612 |
(char->integer #\9)))) |
| 613 |
(hash-table-put! sets 'LETTER_DIGIT |
| 614 |
(code-set-union 'LETTER_DIGIT |
| 615 |
(hash-table-ref sets 'L) |
| 616 |
(hash-table-ref sets 'Nd))) |
| 617 |
(hash-table-put! sets 'ASCII_LETTER_DIGIT |
| 618 |
(code-set-union 'ASCII_LETTER_DIGIT |
| 619 |
(hash-table-ref sets 'ASCII_LETTER) |
| 620 |
(hash-table-ref sets 'ASCII_DIGIT))) |
| 621 |
(hash-table-put! sets 'ASCII_WHITESPACE |
| 622 |
(rlet1 cs (make <char-code-set> :name 'ASCII_WHITESPACE) |
| 623 |
(add-code-range! cs 9 13) ;TAB,LF,LTAB,FF,CR |
| 624 |
(add-code! cs (char->integer #\space)))) |
| 625 |
(hash-table-put! sets 'WHITESPACE |
| 626 |
(code-set-union 'WHITESPACE |
| 627 |
(hash-table-ref sets 'ASCII_WHITESPACE) |
| 628 |
(hash-table-ref sets 'Z))) |
| 629 |
(hash-table-put! sets 'ASCII_BLANK |
| 630 |
(rlet1 cs (make <char-code-set> :name 'ASCII_BLANK) |
| 631 |
(add-code! cs 9) ;TAB |
| 632 |
(add-code! cs (char->integer #\space)))) |
| 633 |
(hash-table-put! sets 'BLANK |
| 634 |
(code-set-union 'BLANK |
| 635 |
(hash-table-ref sets 'ASCII_BLANK) |
| 636 |
(hash-table-ref sets 'Zs))) |
| 637 |
(hash-table-put! sets 'ASCII_PUNCTUATION |
| 638 |
(rlet1 cs (make <char-code-set> :name 'ASCII_PUNCTUATION) |
| 639 |
(add-code! cs 33) ;! |
| 640 |
(add-code! cs 34) ;" |
| 641 |
(add-code! cs 35) ;# |
| 642 |
(add-code! cs 37) ;% |
| 643 |
(add-code! cs 38) ;& |
| 644 |
(add-code! cs 39) ;' |
| 645 |
(add-code! cs 40) ;( |
| 646 |
(add-code! cs 41) ;) |
| 647 |
(add-code! cs 42) ;* |
| 648 |
(add-code! cs 44) ;, |
| 649 |
(add-code! cs 45) ;- |
| 650 |
(add-code! cs 46) ;. |
| 651 |
(add-code! cs 47) ;/ |
| 652 |
(add-code! cs 58) ;: |
| 653 |
(add-code! cs 59) ;; |
| 654 |
(add-code! cs 63) ;? |
| 655 |
(add-code! cs 64) ;@ |
| 656 |
(add-code! cs 91) ;[ |
| 657 |
(add-code! cs 92) ;\ |
| 658 |
(add-code! cs 93) ;) |
| 659 |
(add-code! cs 95) ;_ |
| 660 |
(add-code! cs 123) ;{ |
| 661 |
(add-code! cs 125) ;} |
| 662 |
)) |
| 663 |
(hash-table-put! sets 'ASCII_SYMBOL |
| 664 |
(rlet1 cs (make <char-code-set> :name 'ASCII_SYMBOL) |
| 665 |
(add-code! cs 36) ;$ |
| 666 |
(add-code! cs 43) ;+ |
| 667 |
(add-code! cs 60) ;< |
| 668 |
(add-code! cs 61) ;= |
| 669 |
(add-code! cs 62) ;> |
| 670 |
(add-code! cs 94) ;^ |
| 671 |
(add-code! cs 96) ;` |
| 672 |
(add-code! cs 124) ;| |
| 673 |
(add-code! cs 126) ;~ |
| 674 |
)) |
| 675 |
(hash-table-put! sets 'GRAPHIC |
| 676 |
(code-set-union 'GRAPHIC |
| 677 |
(hash-table-ref sets 'L) |
| 678 |
(hash-table-ref sets 'N) |
| 679 |
(hash-table-ref sets 'P) |
| 680 |
(hash-table-ref sets 'S))) |
| 681 |
(hash-table-put! sets 'ASCII_GRAPHIC |
| 682 |
(code-set-union 'ASCII_GRAPHIC |
| 683 |
(hash-table-ref sets 'ASCII_LETTER) |
| 684 |
(hash-table-ref sets 'ASCII_DIGIT) |
| 685 |
(hash-table-ref sets 'ASCII_PUNCTUATION) |
| 686 |
(hash-table-ref sets 'ASCII_SYMBOL))) |
| 687 |
(hash-table-put! sets 'PRINTING |
| 688 |
(code-set-union 'PRINTING |
| 689 |
(hash-table-ref sets 'GRAPHIC) |
| 690 |
(hash-table-ref sets 'WHITESPACE))) |
| 691 |
(hash-table-put! sets 'ASCII_PRINTING |
| 692 |
(code-set-union 'ASCII_PRINTING |
| 693 |
(hash-table-ref sets 'ASCII_GRAPHIC) |
| 694 |
(hash-table-ref sets 'ASCII_WHITESPACE))) |
| 695 |
(hash-table-put! sets 'ASCII_CONTROL |
| 696 |
(rlet1 cs (make <char-code-set> :name 'ASCII_CONTROL) |
| 697 |
(add-code-range! cs #x00 #x1f) |
| 698 |
(add-code! cs #x7f))) |
| 699 |
(hash-table-put! sets 'HEX_DIGIT |
| 700 |
(rlet1 cs (make <char-code-set> :name 'HEX_DIGIT) |
| 701 |
(add-code-range! cs |
| 702 |
(char->integer #\0) |
| 703 |
(char->integer #\9)) |
| 704 |
(add-code-range! cs |
| 705 |
(char->integer #\A) |
| 706 |
(char->integer #\F)) |
| 707 |
(add-code-range! cs |
| 708 |
(char->integer #\a) |
| 709 |
(char->integer #\f)))) |
| 710 |
(hash-table-put! sets 'ASCII |
| 711 |
(rlet1 cs (make <char-code-set> :name 'ASCII) |
| 712 |
(add-code-range! cs 0 127))) |
| 713 |
(hash-table-put! sets 'ASCII_WORD |
| 714 |
(rlet1 cs (make <char-code-set> :name 'ASCII_WORD) |
| 715 |
(add-code-range! cs |
| 716 |
(char->integer #\0) |
| 717 |
(char->integer #\9)) |
| 718 |
(add-code-range! cs |
| 719 |
(char->integer #\A) |
| 720 |
(char->integer #\Z)) |
| 721 |
(add-code-range! cs |
| 722 |
(char->integer #\a) |
| 723 |
(char->integer #\z)) |
| 724 |
(add-code! cs (char->integer #\_)))) |
| 725 |
(hash-table-put! sets 'LC |
| 726 |
(code-set-union 'LC |
| 727 |
(hash-table-ref sets 'Lt) |
| 728 |
(hash-table-ref sets 'Ll) |
| 729 |
(hash-table-ref sets 'Lu))) |
| 730 |
(hash-table-put! sets 'EMPTY (make <char-code-set> :name 'EMPTY)) |
| 731 |
sets) |
| 732 |
|
| 733 |
;; Break property values |
| 734 |
;; This goes to src/gauche/priv/unicode_attr.h and ext/gauche/unicode-attr.scm |
| 735 |
;; |
| 736 |
;; The data structure is two-level tables that maps codepoint 0-1FFFFF to |
| 737 |
;; an octet. |
| 738 |
;; |
| 739 |
;; The first level, {GB|WB}_break_table, is 512-length byte vector. |
| 740 |
;; Upper 9bit of codepoint is used to index this table. |
| 741 |
;; If the entry is [0..224], it's the index to the {GB|WB}_break_subtable[N][], |
| 742 |
;; which is indexed with the lower 8 bit of the codepoint. |
| 743 |
;; If the entry is above that, it indicates the entire block shares the |
| 744 |
;; same property. |
| 745 |
|
| 746 |
(define (generate-break-tables db) |
| 747 |
(define single-property-offset 224) |
| 748 |
(define subtable-count 0) |
| 749 |
|
| 750 |
(define (get-property entry type); type : GB | WB |
| 751 |
(if entry |
| 752 |
(case type |
| 753 |
[(GB) (ucd-break-property-grapheme entry)] |
| 754 |
[(WB) (ucd-break-property-word entry)]) |
| 755 |
'Other)) |
| 756 |
|
| 757 |
(define (gen-entry code type) ; type : GB | WB |
| 758 |
(format #t " ~a_~a,\n" type |
| 759 |
(get-property (ucd-get-break-property db code) type))) |
| 760 |
|
| 761 |
;; returns subtable-number |
| 762 |
(define (gen-subtable start-code type) |
| 763 |
(define kind #f) |
| 764 |
(do-ec (: lb 256) |
| 765 |
(let* ([e (ucd-get-break-property db (+ start-code lb))] |
| 766 |
[p (get-property e type)]) |
| 767 |
(cond [(not kind) (set! kind p)] |
| 768 |
[(eqv? kind p)] |
| 769 |
[else (set! kind #t)]))) |
| 770 |
(if (symbol? kind) |
| 771 |
;; same property in the entire block |
| 772 |
(format "(BOFF+~a_~a)" type kind) |
| 773 |
(rlet1 subtable-num subtable-count |
| 774 |
(format #t " {\n") |
| 775 |
(do-ec (: lb 256) (gen-entry (+ start-code lb) type)) |
| 776 |
(format #t " },\n") |
| 777 |
(inc! subtable-count)))) |
| 778 |
|
| 779 |
(define (gen-main-table type) |
| 780 |
(set! subtable-count 0) |
| 781 |
(print) |
| 782 |
(print #"static unsigned char ~|type|_break_subtable[][256] = {") |
| 783 |
(let1 nlist (list-ec (: n 512) (gen-subtable (* n 256) type)) |
| 784 |
(print "};") |
| 785 |
(when (>= subtable-count single-property-offset) |
| 786 |
(error #"~|type| subtable index overflow (~|subtable-count|)")) |
| 787 |
(print) |
| 788 |
(format #t "static unsigned char ~a_break_table[] = {" type) |
| 789 |
(do-ec (:parallel (: n nlist) (:integers i)) |
| 790 |
(begin (when (zero? (mod i 4)) (format #t "\n ")) |
| 791 |
(format #t " ~3d," n))) |
| 792 |
(format #t "\n};\n"))) |
| 793 |
|
| 794 |
;; Generate table of symbols |
| 795 |
(define (gen-symbol-table constants type) |
| 796 |
(for-each-with-index (^[i c] (format #t "#define ~a_~a ~a\n" type c i)) |
| 797 |
constants)) |
| 798 |
|
| 799 |
(print) |
| 800 |
(gen-symbol-table (ucd-grapheme-break-properties) 'GB) |
| 801 |
(gen-symbol-table (ucd-word-break-properties) 'WB) |
| 802 |
(print) |
| 803 |
(print #"#define BOFF ~|single-property-offset|") |
| 804 |
(gen-main-table 'GB) |
| 805 |
(gen-main-table 'WB) |
| 806 |
) |
| 807 |
|
| 808 |
;; Auxiliary stuff to find out optimal table configuration |
| 809 |
;; NB: It looks like 3-level table doesn't save at all. |
| 810 |
(define (analyze-break-tables db :optional (lobits 4)) |
| 811 |
(define num-leaf-tables 0) |
| 812 |
(define num-intermediate-tables 0) |
| 813 |
|
| 814 |
;; Check break properties of 2^lobits consecutive codepoints. |
| 815 |
;; Returns (<codepoint-string> . <property-or-#t>). If properties of |
| 816 |
;; all codeponds are the same, cdr is the property; otherwise it's #t. |
| 817 |
(define (probe-lo start accessor) |
| 818 |
(define kind #f) |
| 819 |
(define (get-1 i) |
| 820 |
(if-let1 e (ucd-get-break-property db (+ start i)) |
| 821 |
(accessor e) |
| 822 |
'Other)) |
| 823 |
(do-ec (: i (expt 2 lobits)) |
| 824 |
(let1 kk (get-1 i) |
| 825 |
(cond [(not kind) (set! kind kk)] |
| 826 |
[(eqv? kind #t)] |
| 827 |
[(eqv? kind kk)] |
| 828 |
[else (set! kind #t)]))) |
| 829 |
(when (eqv? kind #t) (inc! num-leaf-tables)) |
| 830 |
(cons (format "~5,'0x" start) kind)) |
| 831 |
;; Check break properties of 16 consecutive 16 codepoint blocks. |
| 832 |
(define (probe-hi start accessor) |
| 833 |
(define kind #f) |
| 834 |
(define ps |
| 835 |
(list-ec (: ii (expt 2 (- 8 lobits))) |
| 836 |
(rlet1 p (probe-lo (+ start (ash ii lobits)) accessor) |
| 837 |
(cond [(not kind) (set! kind (cdr p))] |
| 838 |
[(eqv? kind (cdr p))] |
| 839 |
[else (set! kind #t)])))) |
| 840 |
(case kind |
| 841 |
[(#t) |
| 842 |
(format #t "~3,'0xXX -------------\n" (ash start -8)) |
| 843 |
(for-each (^p (format #t " ~a ~a\n" (car p) (cdr p))) |
| 844 |
(remove (^p (eq? (cdr p) 'Other)) ps)) |
| 845 |
(inc! num-intermediate-tables)] |
| 846 |
[(Other)] |
| 847 |
[else |
| 848 |
(format #t "~3,'0xXX ~a\n" (ash start -8) kind)])) |
| 849 |
|
| 850 |
(define (probe-all name accessor) |
| 851 |
(format #t "~a distribution\n" name) |
| 852 |
(set! num-leaf-tables 0) |
| 853 |
(set! num-intermediate-tables 0) |
| 854 |
(do-ec (: u #x200) |
| 855 |
(probe-hi (ash u 8) accessor)) |
| 856 |
(list name num-leaf-tables num-intermediate-tables)) |
| 857 |
|
| 858 |
(dolist [rr (list (probe-all "GB" ucd-break-property-grapheme) |
| 859 |
(probe-all "WB" ucd-break-property-word))] |
| 860 |
(format #t "~a # int. tables = ~d, # leaf tables = ~d\n" |
| 861 |
(car rr) (cadr rr) (caddr rr))) |
| 862 |
) |
| 863 |
|
| 864 |
(define (generate-break-symbols db) |
| 865 |
(define (gen-symbols constants type) |
| 866 |
(for-each-with-index |
| 867 |
(^[i c] (pprint `(define-constant ,(symbol-append type '_ c) ,i))) |
| 868 |
constants) |
| 869 |
(pprint `(define-constant ,(symbol-append '* type '-properties*) |
| 870 |
',(map-with-index (^[i c] (cons c i)) constants)))) |
| 871 |
|
| 872 |
(gen-symbols (ucd-grapheme-break-properties) 'GB) |
| 873 |
(gen-symbols (ucd-word-break-properties) 'WB) |
| 874 |
) |
| 875 |
|
| 876 |
;; EastAsianWidth property values |
| 877 |
;; This goes to src/gauche/priv/unicode_attr.h |
| 878 |
;; |
| 879 |
;; The data structure is two-level tables that maps codepoint 0-1FFFFF to |
| 880 |
;; an octet. |
| 881 |
;; |
| 882 |
;; The first level, width_table, is 512-length byte vector. |
| 883 |
;; Upper 9bit of codepoint is used to index this table. If the entry |
| 884 |
;; is one of WIDTH_x values, all codepoints of that range has that property |
| 885 |
;; value. |
| 886 |
;; |
| 887 |
;; Otherwise, the entry value - NUM_WIDTH_PROPERTIES points to the subtable, |
| 888 |
;; width_subtable[N][]. It is a vecto of nibbles; to look up, first take |
| 889 |
;; 1-7bit of the codepoint and look up an octet; if the LSB of codepoint |
| 890 |
;; is 0, take lower nibble; otherwise, take upper nibble. |
| 891 |
|
| 892 |
(define (generate-width-tables db) |
| 893 |
(define prop-count (length (ucd-east-asian-widths))) |
| 894 |
(define subtable-count prop-count) |
| 895 |
|
| 896 |
;; returns subtable-number |
| 897 |
(define (gen-subtable start-code) |
| 898 |
(let1 e (ucd-get-east-asian-width db start-code) |
| 899 |
(if (any?-ec (: lb 256) |
| 900 |
(not (eq? (ucd-get-east-asian-width db (+ start-code lb)) e))) |
| 901 |
(rlet1 subtable-num subtable-count |
| 902 |
(print " {") |
| 903 |
(do-ec (: c 128) |
| 904 |
(let ([w0 (ucd-get-east-asian-width db (+ start-code (* c 2)))] |
| 905 |
[w1 (ucd-get-east-asian-width db (+ start-code (* c 2) 1))]) |
| 906 |
(print #" WIDTH_ENTRY(WIDTH_~|w1|, WIDTH_~|w0|),"))) |
| 907 |
(print " },") |
| 908 |
(inc! subtable-count)) |
| 909 |
;; all codepoints of this range shares the value. |
| 910 |
(find-index (cut eqv? e <>) (ucd-east-asian-widths))))) |
| 911 |
|
| 912 |
;; Generate table of symbols |
| 913 |
(define (gen-symbol-table) |
| 914 |
(for-each-with-index (^(i c) (format #t "#define WIDTH_~a ~a\n" c i)) |
| 915 |
(ucd-east-asian-widths)) |
| 916 |
(print #"#define NUM_WIDTH_PROPERTIES ~|prop-count|") |
| 917 |
(print) |
| 918 |
(print "static void init_WIDTH_symbols(ScmModule *mod) {") |
| 919 |
(print #" ScmObj v = Scm_MakeVector(~|prop-count|, SCM_FALSE);") |
| 920 |
(for-each-with-index |
| 921 |
(^[i c] |
| 922 |
(print #" SCM_VECTOR_ELEMENT(v, ~|i|) = SCM_INTERN(\"~|c|\");")) |
| 923 |
(ucd-east-asian-widths)) |
| 924 |
(print " Scm_DefineConst(mod, \ |
| 925 |
SCM_SYMBOL(SCM_INTERN(\"*east-asian-widths*\")), \ |
| 926 |
v);") |
| 927 |
(print "}")) |
| 928 |
|
| 929 |
(print) |
| 930 |
(gen-symbol-table) |
| 931 |
(print) |
| 932 |
(print "#define WIDTH_ENTRY(a, b) (((a)<<4) | (b))") |
| 933 |
(print) |
| 934 |
(print "static unsigned char width_subtable[][128] = {") |
| 935 |
(let1 nlist (list-ec (: n 512) (gen-subtable (* n 256))) |
| 936 |
(print "};") |
| 937 |
(print) |
| 938 |
(format #t "static unsigned char width_table[] = {") |
| 939 |
(do-ec (:parallel (: n nlist) (:integers i)) |
| 940 |
(begin (when (zero? (mod i 8)) (format #t "\n ")) |
| 941 |
(format #t " ~3d," n))) |
| 942 |
(format #t "\n};\n")) |
| 943 |
) |