Debian Codegraph @

lib/text/unicode/ucd.scm

raw
1 ;;;
2 ;;; text.unicode.ucd - Unicode character database library
3 ;;;
4 ;;; Originally written by Shiro Kawai, 2011
5 ;;; Public Domain - use as you like.
6 ;;;
7
8 ;; Read Unicode data tables and generates Scheme data structures
9 ;; for ease of use.
10 ;;
11 ;; This data structures are not intended to be lightweight for
12 ;; in-application use; they are intermediate format to generate
13 ;; more efficient structure suitable for applications.
14 ;; The src/gen-unicode.scm script uses this module.
15 ;;
16 ;; The applications should never 'use' this module. The info is
17 ;; available from Gauche core API and gauche.unicode modules.
18
19 (define-module text.unicode.ucd
20 (use gauche.record)
21 (use gauche.dictionary)
22 (use gauche.collection)
23 (use gauche.uvector)
24 (use gauche.charconv)
25 (use gauche.generator)
26 (use srfi.13)
27 (use srfi.42)
28 (use util.match)
29 (use file.util)
30 (use text.csv)
31 (export ucd-parse-files
32 ucd-get-entry ucd-get-category-ranges
33 ucd-get-break-property ucd-get-east-asian-width
34 ucd-map-entries
35
36 ucd-save-db ucd-load-db ucd-version
37
38 ucd-entry ucd-entry-category
39 ucd-entry-case-info ucd-entry-special-case-info
40 ucd-entry-case-map ucd-entry-alphabetic
41 ucd-entry-uppercase ucd-entry-lowercase
42 ucd-entry-digit-value
43
44 ucd-simple-case-map?
45 ucd-simple-case-map-case
46 ucd-simple-case-map-offset
47 ucd-extended-case-map?
48 ucd-extended-case-map-simple-map
49 ucd-extended-case-map-special-map
50 ucd-extended-case-map-code
51
52 ucd-break-property
53 ucd-break-property-grapheme
54 ucd-break-property-word
55
56 ucd-general-categories
57 ucd-grapheme-break-properties
58 ucd-word-break-properties
59
60 ucd-east-asian-widths
61
62 eucjp->ucd-entry sjis->ucd-entry
63 )
64 )
65 (select-module text.unicode.ucd)
66
67 ;;;
68 ;;; The database record
69 ;;;
70
71 ;; We can't simply have codepoint -> entry table for all the codepoints,
72 ;; since it'll be too huge. We use the fact that the region above
73 ;; U+20000 has many consecutive characters that shares some of the
74 ;; properties.
75 ;;
76 ;; The slots of unichar-db should be considered private. They may be
77 ;; changed in future versions.
78
79 ;; UnicodeData.txt format
80 ;; code
81 ;; name
82 ;; category
83 ;; combining-class
84 ;; bidi-class
85 ;; decomposition
86 ;; numeric-value-1 (for decimal only)
87 ;; numeric-value-2 (for decimal & digits)
88 ;; numeric-value-3 (for decimal, digits & numeric)
89 ;; bidi-mirrored
90 ;; old-name
91 ;; notes
92 ;; simple-uppercase
93 ;; simple-lowercase
94 ;; simple-titlecase
95
96 (define-record-type unichar-db %make-unichar-db #f
97 ;; Unicode version, in string, e.g. "6.0.3"
98 (version)
99 ;; codepoint -> ucd-entry
100 ;; The 'table' maps characters below U+20000 to the ucd-entry record.
101 ;; It is a hashtable keyed by codepoint.
102 table
103 ;; start-codepoint -> Maybe category
104 ;; This is only used for chars over U+20000.
105 ranges
106 ;; codepoint -> ucd-break-property
107 break-table
108 ;; codepoint -> east-asian-width
109 width-table
110 )
111
112 (define (make-unichar-db)
113 (%make-unichar-db #f
114 (make-hash-table 'eqv?)
115 (make-hash-table 'eqv?)
116 (make-hash-table 'eqv?)
117 (make-hash-table 'eqv?)))
118
119 ;; equal? method is mainly for testing
120 (define-method object-equal? ((a unichar-db) (b unichar-db))
121 (define (dict-equal? ta tb)
122 (and (= (size-of ta) (size-of tb))
123 (every (^k (equal? (dict-get ta k) (dict-get tb k)))
124 (dict-keys ta))))
125 (and (dict-equal? (unichar-db-table a) (unichar-db-table b))
126 (dict-equal? (unichar-db-ranges a) (unichar-db-ranges b))
127 (dict-equal? (unichar-db-break-table a) (unichar-db-break-table b))))
128
129 (define-record-type ucd-entry %make-ucd-entry #f
130 (category)
131 (case-info) ; (upper lower title) codepoints
132 (special-case-info) ; ((upper ...) (lower ...) (title ...)) codepoints
133 (case-map) ; either ucd-simple-case-map or ucd-extended-case-map
134 ; set by assign-case-mapping
135 (alphabetic) ; #t if Alphabetic
136 (uppercase) ; #t if Uppercase
137 (lowercase) ; #t if Lowercase
138 (digit-value) ; 0..9 for Nd chars, #f otherwise
139 )
140
141 (define-method object-equal? ((a ucd-entry) (b ucd-entry))
142 (and (eq? (ucd-entry-category a) (ucd-entry-category b))
143 (equal? (ucd-entry-case-map a) (ucd-entry-case-map b))
144 (eq? (ucd-entry-alphabetic a) (ucd-entry-alphabetic b))
145 (eq? (ucd-entry-uppercase a) (ucd-entry-uppercase b))
146 (eq? (ucd-entry-lowercase a) (ucd-entry-lowercase b))
147 (eq? (ucd-entry-digit-value a) (ucd-entry-digit-value b))))
148
149 (define-record-type ucd-break-property %make-ucd-break-property #f
150 (grapheme) ; Grapheme_Break category
151 (word) ; Word_Break category
152 )
153
154 (define-method object-equal? ((a ucd-break-property) (b ucd-break-property))
155 (and (eq? (ucd-break-property-grapheme a) (ucd-break-property-grapheme b))
156 (eq? (ucd-break-property-word a) (ucd-break-property-word b))))
157
158 (define (make-ucd-entry category case-info digit-value)
159 (%make-ucd-entry category case-info #f #f #f #f #f digit-value))
160
161 (define (make-ucd-break-property)
162 (%make-ucd-break-property 'Other 'Other))
163
164 ;; API
165 (define (ucd-get-entry db ucs)
166 (dict-get (unichar-db-table db) ucs #f))
167
168 ;; API
169 ;; Call (proc code ucd-entry) for all entries in unichar-db-table
170 ;; NB: The db table only contains entries up to U+20000. Higher entries
171 ;; has to be dealt with separately.
172 (define (ucd-map-entries db proc)
173 (dict-map (unichar-db-table db) proc))
174
175 ;; API
176 ;; ((<start-codepoint> . <category-symbol>) ...)
177 (define (ucd-get-category-ranges db)
178 (sort (coerce-to <list> (unichar-db-ranges db)) < car))
179
180 ;; API
181 ;; return break property
182 (define (ucd-get-break-property db code)
183 (dict-get (unichar-db-break-table db) code #f))
184
185 ;; API
186 ;; return East Asian Width
187 (define (ucd-get-east-asian-width db code)
188 (dict-get (unichar-db-width-table db) code 'N))
189
190 (define-record-type ucd-simple-case-map #t #t
191 case ; upper or lower, for this character itself (title==upper)
192 offset) ; offset to convert to the opposite case
193
194 (define-method object-equal? ((a ucd-simple-case-map)
195 (b ucd-simple-case-map))
196 (and (eq? (ucd-simple-case-map-case a) (ucd-simple-case-map-case b))
197 (eqv? (ucd-simple-case-map-offset a) (ucd-simple-case-map-offset b))))
198
199 (define-record-type ucd-extended-case-map #t #t
200 code ; original code
201 simple-map ; (upper-off lower-off title-off) or #f
202 special-map) ; ((upper ... ) (lower ...) (title ...)) or #f
203
204 (define-method object-equal? ((a ucd-extended-case-map)
205 (b ucd-extended-case-map))
206 (and (eqv? (ucd-extended-case-map-code a)
207 (ucd-extended-case-map-code b))
208 (equal? (ucd-extended-case-map-simple-map a)
209 (ucd-extended-case-map-simple-map b))
210 (equal? (ucd-extended-case-map-special-map a)
211 (ucd-extended-case-map-special-map b))))
212
213 ;; These order must match with the enum in src/gauche/char_attr.h
214 (define (ucd-general-categories)
215 '(Lu Ll Lt Lm Lo Mn Mc Me Nd Nl No Pc Pd Ps Pe Pi Pf Po
216 Sm Sc Sk So Zs Zl Zp Cc Cf Cs Co Cn))
217 (define (ucd-east-asian-widths)
218 '(A F H N Na W))
219
220 ;; Break properties. The actual property values are in
221 ;; src/gauche/priv/unicode_attr.h, which is generated by src/gen-unicode.scm.
222 ;; NB: We use 'Other instead of 'Any in UAX#29.
223 ;; NB: 'ExtPict is not defined in Grapheme Break Properties, but in
224 ;; Emoji Properties. It is required to implement GB11 and WB3c rule
225 ;; (see ext/gauche/unicode.scm)
226 (define (ucd-grapheme-break-properties)
227 '(CR LF Control Extend ZWJ Regional_Indicator Prepend SpacingMark
228 L V T LV LVT ExtPict Other))
229
230 (define (ucd-word-break-properties)
231 '(CR LF Newline Extend ZWJ Regional_Indicator
232 Format Katakana Hebrew_Letter ALetter Single_Quote Double_Quote
233 MidLetter MidNum MidNumLet Numeric ExtendNumLet WSegSpace
234 ExtPict ExtPict_ALetter Other))
235
236 ;;
237 ;; CES encoding conversions.
238 ;;
239
240 ;; We don't have utf8->ucs4 yet in 0.9.1, so we need to roll our own.
241 (define (utf8vec->ucs4 vec)
242 (let1 b0 (~ vec 0)
243 (cond [(< b0 #x80) b0]
244 [(< b0 #xe0) (+ (ash (logand b0 #x1f) 6)
245 (logand (~ vec 1) #x3f))]
246 [(< b0 #xf0) (+ (ash (logand b0 #x0f) 12)
247 (ash (logand (~ vec 1) #x3f) 6)
248 (logand (~ vec 2) #x3f))]
249 [else (+ (ash (logand b0 #x07) 18)
250 (ash (logand (~ vec 1) #x3f) 12)
251 (ash (logand (~ vec 2) #x3f) 6)
252 (logand (~ vec 3) #x3f))])))
253
254 (define (get-char/ces vec ces)
255 (guard (e [else #f])
256 (utf8vec->ucs4
257 (string->u8vector
258 (ces-convert (u8vector->string vec) ces 'utf-8)))))
259
260 (define (2b->u8vector code)
261 (u8vector (ash code -8) (logand code #xff)))
262 (define (3b->u8vector code)
263 (u8vector (ash code -16) (logand (ash code -8) #xff) (logand code #xff)))
264
265 ;; can return #f
266 (define (eucjp->ucs code)
267 ;; NB: The codes in *jisx0213-extras* should be filtered before
268 ;; calling this.
269 (and-let* ([v (cond [(< code #x80) (u8vector code)]
270 [(<= #x8ea1 code #x8ef2) (2b->u8vector code)]
271 [(<= #xa1a1 code #xfefe) (2b->u8vector code)]
272 [(<= #x8fa1a1 code #x8ffefe) (3b->u8vector code)]
273 [else #f])])
274 (get-char/ces v 'euc-jp)))
275
276 ;; can return #f
277 (define (sjis->ucs code)
278 ;; NB: The codes in *jisx0213-extras* should be filtered before
279 ;; calling this.
280 (and-let* ([v (cond [(< code #x80) (u8vector code)]
281 [(<= #xa0 code #xdf) (u8vector code)]
282 [(<= #x8040 code #x8ffc) (2b->u8vector code)]
283 [(<= #x9040 code #x9ffc) (2b->u8vector code)]
284 [(<= #xe040 code #xeffc) (2b->u8vector code)]
285 [(<= #xf040 code #xfffc) (2b->u8vector code)]
286 [else #f])])
287 (get-char/ces v 'sjis)))
288
289 ;; NB: JISX0213:2004 has 25 letters that aren't included in Unicode 6.0.
290 ;; We define general categories of them by our own.
291 (define *jisx0213-extras*
292 '(;; sjis euc-jp category
293 (#x82F5 #xA4F7 Lo) ; U+304B U+309A Kana+semi voice mark
294 (#x82F6 #xA4F8 Lo) ; U+304D U+309A
295 (#x82F7 #xA4F9 Lo) ; U+304F U+309A
296 (#x82F8 #xA4FA Lo) ; U+3051 U+309A
297 (#x82F9 #xA4FB Lo) ; U+3053 U+309A
298 (#x8397 #xA5F7 Lo) ; U+30AB U+309A
299 (#x8398 #xA5F8 Lo) ; U+30AD U+309A
300 (#x8399 #xA5F9 Lo) ; U+30AF U+309A
301 (#x839A #xA5FA Lo) ; U+30B1 U+309A
302 (#x839B #xA5FB Lo) ; U+30B3 U+309A
303 (#x839C #xA5FC Lo) ; U+30BB U+309A
304 (#x839D #xA5FD Lo) ; U+30C4 U+309A
305 (#x839E #xA5FE Lo) ; U+30C8 U+309A
306 (#x83F6 #xA6F8 Lo) ; U+31F7 U+309A
307 (#x8663 #xABC4 Ll) ; U+00E6 U+0300 IPA+accent
308 (#x8667 #xABC8 Ll) ; U+0254 U+0300
309 (#x8668 #xABC9 Ll) ; U+0254 U+0301
310 (#x8669 #xABCA Ll) ; U+028C U+0300
311 (#x866A #xABCB Ll) ; U+028C U+0301
312 (#x866B #xABCC Ll) ; U+0259 U+0300
313 (#x866C #xABCD Ll) ; U+0259 U+0301
314 (#x866D #xABCE Ll) ; U+025A U+0300
315 (#x866E #xABCF Ll) ; U+025A U+0301
316 (#x8685 #xABE5 Sk) ; U+02E9 U+02E5 Modifier
317 (#x8686 #xABE6 Sk) ; U+02E5 U+02E9 Modifier
318 ))
319
320 (define (xxx->entry db code get-extra-code ->ucs)
321 (cond [(find (^l (eqv? code (get-extra-code l))) *jisx0213-extras*)
322 => (^l (let1 cat (caddr l)
323 (rlet1 e (make-ucd-entry cat '(#f #f #f) #f)
324 (when (eq? cat 'Ll) (set! (ucd-entry-lowercase e) #t))
325 (when (eq? cat 'Lo) (set! (ucd-entry-alphabetic e) #t)))))]
326 [(->ucs code)
327 => (^c (dict-get (unichar-db-table db) c #f))]
328 [else #f]))
329
330 ;; The following APIs are needed since we have to handle jisx0213 extra
331 ;; characters.
332
333 ;; API
334 ;; can return #f if CODE is 'hole' (unassigned)
335 (define (eucjp->ucd-entry db code)
336 (xxx->entry db code cadr eucjp->ucs))
337
338 ;; API
339 ;; can return #f if CODE is 'hole' (unassigned)
340 (define (sjis->ucd-entry db code)
341 (xxx->entry db code car sjis->ucs))
342
343 ;;;
344 ;;; Parsing UnicodeData and other tables.
345 ;;;
346
347 (define (ucd-parse-files datadir)
348 (check-data-files datadir)
349 (let1 db (unicode-data (build-path datadir "UnicodeData.txt"))
350 (check-assumptions db)
351 (set-unicode-version! db (build-path datadir "SpecialCasing.txt"))
352 (special-case-mapping (build-path datadir "SpecialCasing.txt") db)
353 (additional-case-entries (build-path datadir "PropList.txt") db)
354 (assign-case-mapping db)
355 (grapheme-break-property
356 (ucd-grapheme-break-properties)
357 (build-path datadir "auxiliary/GraphemeBreakProperty.txt") db)
358 (word-break-property
359 (ucd-word-break-properties)
360 (build-path datadir "auxiliary/WordBreakProperty.txt") db)
361 (augument-break-property
362 (build-path datadir "emoji/emoji-data.txt") db)
363 (east-asian-width (build-path datadir "EastAsianWidth.txt") db)
364 db))
365
366 (define-constant +required-files+
367 '("UnicodeData.txt" "SpecialCasing.txt" "PropList.txt" "EastAsianWidth.txt"
368 "auxiliary/GraphemeBreakProperty.txt" "auxiliary/WordBreakProperty.txt"))
369
370 (define (check-data-files dir)
371 (dolist [f +required-files+ #t]
372 (unless (file-exists? (build-path dir f))
373 (errorf "Couldn't find datafile `~a' in ~a; the directory should \
374 contain these files: ~a" f dir +required-files+))))
375
376 ;; Process UnicodeData
377 (define (unicode-data file)
378 (define db (make-unichar-db))
379 ;; state := (prev-code prev-cat range?)
380 (call-with-input-file file
381 (^p (port-fold (unicode-data-1 db) '(#f #f #f)
382 (cut (make-csv-reader #\;) p))))
383 db)
384
385 (define (dict-fill! dic start end ent)
386 (dotimes [i (- end start)]
387 (dict-put! dic (+ start i) ent)))
388
389 (define (unicode-data-1 db)
390 (define table (unichar-db-table db))
391 (define ranges (unichar-db-ranges db))
392 (^[entry state]
393 (match-let ([(scode name scat _ _ _ snum _ _ _ _ _ sup slo sti) entry]
394 [(prev-code prev-cat range?) state])
395 (let* ([code (parse-code scode)]
396 [range-start? (#/First>/ name)]
397 [cat (string->symbol scat)]
398 [case-info `(,(parse-code sup) ,(parse-code slo) ,(parse-code sti))]
399 [entry (make-ucd-entry cat case-info (string->number snum))])
400 ;; register table entry
401 (cond
402 [(not prev-code) ; initial loop
403 (dict-put! table code entry)]
404 [(and range? (not (= code (+ prev-code 1)))) ;gap
405 (dict-fill! table (+ prev-code 1) (+ code 1) entry)]
406 [else
407 (dict-put! table code entry)])
408 ;; insert category range over U+20000
409 (cond [(= code #x20000) (dict-put! ranges code cat)] ; start
410 [(= code #x10fffd) (dict-put! ranges #x10fffe #f)] ;end
411 [(> code #x20000)
412 (cond [(not (= code (+ prev-code 1)))
413 (unless range?
414 (dict-put! ranges (+ prev-code 1) #f)
415 (dict-put! ranges code cat))]
416 [(not (eq? cat prev-cat))
417 (dict-put! ranges code cat)])])
418 `(,code ,cat ,range-start?)))))
419
420 (define (check-assumptions db)
421 (define table (unichar-db-table db))
422 ;; Check if U+2xxxx range only has Lo and no case mappings.
423 (dotimes [i #x10000]
424 (let1 e (dict-get table (+ i #x20000) #f)
425 (when (and e (or (not (eq? (ucd-entry-category e) 'Lo))
426 (not (equal? (ucd-entry-case-info e) '(#f #f #f)))))
427 (errorf "Unicode assumption failed: U+~x ~s ~s ~s"
428 (+ i #x20000)
429 (ucd-entry-category e)
430 (ucd-entry-case-info e)
431 (ucd-entry-special-case-info e)))))
432 )
433
434 (define (parse-code str) (string->number str 16))
435
436 ;; Get Unicode version number from a data file
437 ;; UCD data files (except UnicodeData.txt) begins with a comment line
438 ;; like this:
439 ;; # FileName-X.X.X.txt
440 ;; where the actual file name is FileName.txt, and X.X.X is the version
441 ;; number. We use it to determine unicode version heuristically.
442 (define (get-unicode-version file)
443 (rxmatch-let (#/^(\w+)\.txt$/ (sys-basename file)) [_ base]
444 (let* ([rx (string->regexp #"~|base|-(\\d+\\.\\d+\\.\\d+)\\.txt")]
445 [line (with-input-from-file file read-line)]
446 [m (rx line)])
447 (and m (m 1)))))
448
449 ;; API
450 (define (ucd-version db) (unichar-db-version db))
451
452 (define (set-unicode-version! db . files)
453 (set! (unichar-db-version db) (any get-unicode-version files)))
454
455
456 ;; Augument the db with SpecialCasing.txt.
457 ;; We ignore entries with conditional mappings. The only language-insensitive
458 ;; conditional mapping is the final form of sigma:
459 ;; 03A3; 03C2; 03A3; 03A3; Final_Sigma; # GREEK CAPITAL LETTER SIGMA
460 ;; We handle this one explicitly.
461 (define (special-case-mapping file db)
462 (define table (unichar-db-table db))
463 (define (handle line)
464 (if-let1 sp (rxmatch->string #/^([0-9a-fA-F][^#]*)/ line 1)
465 (match-let1 (scode slo sti sup sconditions . _) (string-split sp #/\;\s*/)
466 (when (string-null? sconditions)
467 (if-let1 code (parse-code scode)
468 (let ([lo (map parse-code (string-split slo #[\s]))]
469 [ti (map parse-code (string-split sti #[\s]))]
470 [up (map parse-code (string-split sup #[\s]))]
471 [e (dict-get table code)])
472 (set! (ucd-entry-special-case-info e) (list up lo ti))))))))
473 (with-input-from-file file
474 (cut generator-for-each handle read-line)))
475
476 ;; Assign case-map slot by either simple-case-map or extended-case-map
477 (define (assign-case-mapping db)
478 (define (handle code entry)
479 (set! (ucd-entry-case-map entry)
480 (if (ucd-entry-special-case-info entry)
481 (make-ucd-extended-case-map code
482 (map (^s (and s (- s code)))
483 (ucd-entry-case-info entry))
484 (ucd-entry-special-case-info entry))
485 (match (ucd-entry-case-info entry)
486 [#f #f]
487 [(#f #f #f) #f]
488 [(#f (? number? lower) #f) (=> fail)
489 (if (<= -8192 (- lower code) 8191)
490 (make-ucd-simple-case-map 'upper (- lower code))
491 (fail))]
492 [((? number? upper) #f title) (=> fail)
493 (if (and (eqv? upper title)
494 (<= -8192 (- upper code) 8191))
495 (make-ucd-simple-case-map 'lower (- upper code))
496 (fail))]
497 [_ ($ make-ucd-extended-case-map code
498 (map (^s (and s (- s code)))
499 (ucd-entry-case-info entry))
500 (ucd-entry-special-case-info entry))]))))
501 (hash-table-for-each (unichar-db-table db) handle))
502
503 (define (additional-case-entries file db)
504 (define table (unichar-db-table db))
505 (define (parse line)
506 (rxmatch-case line
507 [#/^([0-9A-Fa-f]+)(\.\.([0-9A-Fa-f]+))?\s+\;\s+(\w+)/ (#f s #f e p)
508 (list (string->symbol p)
509 (parse-code s)
510 (if e (parse-code e) (parse-code s)))]
511 [else #f]))
512 (define (flag start end proc)
513 (do-ec (: c start (+ end 1)) (set! (proc (dict-get table c)) #t)))
514 (define (handle line)
515 (match (parse line)
516 [('Other_Alphabetic s e) (flag s e ucd-entry-alphabetic)]
517 [('Other_Uppercase s e) (flag s e ucd-entry-uppercase)]
518 [('Other_Lowercase s e) (flag s e ucd-entry-lowercase)]
519 [_ #f]))
520
521 (dict-for-each table
522 (^(k v)
523 (case (ucd-entry-category v)
524 [(Lu) (set! (ucd-entry-alphabetic v) #t)
525 (set! (ucd-entry-uppercase v) #t)]
526 [(Ll) (set! (ucd-entry-alphabetic v) #t)
527 (set! (ucd-entry-lowercase v) #t)]
528 [(Lt Lm Lo Nl) (set! (ucd-entry-alphabetic v) #t)])))
529 (with-input-from-file file
530 (cut generator-for-each handle read-line)))
531
532 (define (east-asian-width file db)
533 (define table (unichar-db-width-table db))
534 (define (parse line)
535 (rxmatch-case line
536 [#/^([0-9A-Fa-f]+)(\.\.([0-9A-Fa-f]+))?\s*\;\s*(\w+)/ (#f s #f e p)
537 (list (string->symbol p)
538 (parse-code s)
539 (if e (parse-code e) (parse-code s)))]
540 [else #f]))
541 (define (handle line)
542 (match (parse line)
543 [(width start end)
544 (do-ec (: c start (+ end 1))
545 (dict-put! table c width))]
546 [_ #f]))
547 (with-input-from-file file
548 (cut generator-for-each handle read-line)))
549
550 (define (ensure-ucd-break-property db code)
551 (let1 table (unichar-db-break-table db)
552 (or (dict-get table code #f)
553 (rlet1 e (make-ucd-break-property)
554 (dict-put! table code e)))))
555
556 (define ((break-property accessor) symbols file db)
557 (define (check str)
558 (rlet1 sym (string->symbol str)
559 (unless (memq sym symbols)
560 (errorf "Unrecognized break property `~a'. This must be a newer \
561 version of UnicodeData. The code needs to be updated." sym))))
562 (define (handle line)
563 (rxmatch-case line
564 [#/^([0-9a-fA-F]{4,6})\.\.([0-9a-fA-F]{4,6})\s+\;\s+(\w+)/ (_ ss ee cc)
565 (do-ec (: c (parse-code ss) (+ (parse-code ee) 1))
566 (let1 e (ensure-ucd-break-property db c)
567 (set! (accessor e) (check cc))))]
568 [#/^([0-9a-fA-F]{4,6})\s+\;\s+(\w+)/ (_ ss cc)
569 (let1 e (ensure-ucd-break-property db (parse-code ss))
570 (set! (accessor e) (check cc)))]))
571 (with-input-from-file file
572 (cut generator-for-each handle read-line)))
573
574 (define grapheme-break-property (break-property ucd-break-property-grapheme))
575 (define word-break-property (break-property ucd-break-property-word))
576
577 ;; Grapheme break property needs to know emoji Extended_Pictographic
578 (define (augument-break-property file db)
579 (define (ensure c)
580 (let1 e (ensure-ucd-break-property db c)
581 (case (ucd-break-property-grapheme e)
582 [(Other) (set! (ucd-break-property-grapheme e) 'ExtPict)]
583 [else (warn "Overriding non-'Other' grapheme property of U+~4,'0x (~s)\n"
584 c (ucd-break-property-grapheme e))])
585 (case (ucd-break-property-word e)
586 [(Other) (set! (ucd-break-property-word e) 'ExtPict)]
587 [(ALetter) (set! (ucd-break-property-word e) 'ExtPict_ALetter)]
588 [else (warn "Overriding non-'Other' word property of U+~4,'0x (~s)\n"
589 c (ucd-break-property-word e))])))
590 (define (handle line)
591 (rxmatch-case line
592 [#/^([0-9a-fA-F]{4,6})\.\.([0-9a-fA-F]{4,6})\s+\;\s+Extended_Pictographic/
593 (_ ss ee)
594 (do-ec (: c (parse-code ss) (+ (parse-code ee) 1))
595 (ensure c))]
596 [#/^([0-9a-fA-F]{4,6})\s+\;\s+Extended_Pictographic/ (_ ss)
597 (ensure (parse-code ss))]))
598 (with-input-from-file file
599 (cut generator-for-each handle read-line)))
600
601 ;;;
602 ;;; Dumping and restoring database
603 ;;;
604
605 ;; Dump format should be regarded as internal - we may change it at any
606 ;; tiem as we like.
607
608 ;; For now, we dump it in three parts, correspoinding to the unichar-db
609 ;; slots. To suppress the size of the dump, we use simple run-length
610 ;; compressing at the S-expr level.
611 ;;
612 ;; <RLE-compressed-list> : <RLE-compressed-entry> ...
613 ;; <RLE-compressed-entry> : <entry>
614 ;; | (rep <count> <entry> ...)
615 ;;
616 ;; 1. The low codepoint table part
617 ;;
618 ;; RLE compressed list of <ucd-entry>
619 ;;
620 ;; <ucd-entry> : <basic-entry>
621 ;; <basic-entry> : #f ; unassigned
622 ;; | (<category> [<case-map>] [A] [L] [U] [<digit>])
623 ;; <category> : <symbol> ; general category symbol
624 ;; <case-map> | ( {upper|lower} <offset> ) ; for simple-case-map
625 ;; | (<code> <simple-map-info> <extended-map-info>)
626 ;; <simple-map-info> : #f
627 ;; | (<maybe-offset> <maybe-offset> <maybe-offset>)
628 ;; <extended-map-info> : #f
629 ;; | ((<code> ...) (<code> ...) (<code> ...))
630 ;;
631 ;; 2. The upper category ranges
632 ;;
633 ;; List of (<start-code> <category>).
634 ;;
635 ;; 3. Break properties
636 ;;
637 ;; RLE compressed list of (<grapheme-break-property> <word-break-property>).
638 ;;
639 ;; 4. East Asian Width properties
640 ;;
641 ;; RLE compressed list of <east-asian-width>
642
643
644 (define (ucd-save-db db)
645 (define (convert-case-map cm)
646 (cond [(ucd-simple-case-map? cm)
647 `(,(ucd-simple-case-map-case cm)
648 ,(ucd-simple-case-map-offset cm))]
649 [(ucd-extended-case-map? cm)
650 `(,(ucd-extended-case-map-code cm)
651 ,(ucd-extended-case-map-simple-map cm)
652 ,(ucd-extended-case-map-special-map cm))]
653 [else #f]))
654 (define (format-ucd-entry e)
655 (and e
656 `(,(ucd-entry-category e)
657 ,@(cond-list
658 [(convert-case-map (ucd-entry-case-map e))]
659 [(ucd-entry-alphabetic e) 'A]
660 [(ucd-entry-uppercase e) 'U]
661 [(ucd-entry-lowercase e) 'L]
662 [(ucd-entry-digit-value e)]))))
663 (define (format-break-property-entry e)
664 (and e
665 `(,(ucd-break-property-grapheme e)
666 ,(ucd-break-property-word e))))
667
668 (print ";; Generated by text.unicode.ucd from Unicode Character Database")
669 (print ";; unicode version")
670 (write (unichar-db-version db))
671 (print)
672
673 (print ";; unichar-db-table")
674 (print "(")
675 (let1 tab (unichar-db-table db)
676 ($ generator-for-each print
677 $ rle-compressing-generator
678 $ gmap (^c (format-ucd-entry (dict-get tab c #f)))
679 $ giota $ + 1 $ apply max $ dict-keys tab))
680 (print ")")
681
682 (print ";; unichar-db-high-ranges")
683 (print "(")
684 (let1 tab (unichar-db-ranges db)
685 (dolist [k (sort (dict-keys tab))]
686 (print (list k (dict-get tab k)))))
687 (print ")")
688
689 (print ";; unichar-db-break-table")
690 (print "(")
691 (let1 tab (unichar-db-break-table db)
692 ($ generator-for-each print
693 $ rle-compressing-generator
694 $ gmap (^c (format-break-property-entry (dict-get tab c #f)))
695 $ giota $ + 1 $ apply max $ dict-keys tab))
696 (print ")")
697
698 (print ";; unichar-db-width-table")
699 (print "(")
700 (let1 tab (unichar-db-width-table db)
701 ($ generator-for-each print
702 $ rle-compressing-generator
703 $ gmap (^c (dict-get tab c #f))
704 $ giota $ + 1 $ apply max $ dict-keys tab))
705 (print ")")
706 )
707
708 (define (ucd-load-db port)
709 (rlet1 db (make-unichar-db)
710
711 (define (recover-case-map e)
712 (match e
713 [#f #f]
714 [(case offset) (make-ucd-simple-case-map case offset)]
715 [(code simple ext) (make-ucd-extended-case-map code simple ext)]))
716 (define (add-entry! e code)
717 (and e
718 (receive (category case-map flags)
719 (match e
720 [(c) (values c #f '())]
721 [(c (? pair? m) . flags) (values c m flags)]
722 [(c . flags) (values c #f flags)])
723 (hash-table-put! (unichar-db-table db)
724 code
725 (%make-ucd-entry category #f #f
726 (recover-case-map case-map)
727 (boolean (memq 'A flags))
728 (boolean (memq 'U flags))
729 (boolean (memq 'L flags))
730 (find number? flags))))))
731
732 (define (add-break-property! e code)
733 (match e
734 [#f #f]
735 [(g w) (let1 r (ensure-ucd-break-property db code)
736 (ucd-break-property-grapheme-set! r g)
737 (ucd-break-property-word-set! r w))]))
738
739 (define (add-width-property! e code)
740 (match e
741 [#f #f]
742 [sym (hash-table-put! (unichar-db-width-table db) code sym)]))
743
744 ;; version
745 (set! (unichar-db-version db) (read port))
746 ;; ucd-entries
747 (let1 entries (read port)
748 (generator-for-each add-entry!
749 ($ rle-decompressiong-generator
750 $ list->generator entries)
751 (giota)))
752 ;; high ranges
753 (dolist [range (read port)]
754 (match-let1 (start val) range
755 (dict-put! (unichar-db-ranges db) start val)))
756 ;; break properties
757 (let1 break-properties (read port)
758 (generator-for-each add-break-property!
759 ($ rle-decompressiong-generator
760 $ list->generator break-properties)
761 (giota)))
762 ;; width table
763 (let1 width-properties (read port)
764 (generator-for-each add-width-property!
765 ($ rle-decompressiong-generator
766 $ list->generator width-properties)
767 (giota)))
768 ))
769
770 ;; Run-length compressor/decompressor
771 (define (rle-compressing-generator input-gen)
772 (define (compress in seed)
773 (match seed
774 [#f ; initial state
775 (values '() `(,in 0))]
776 [(i0 c) ; may be repeating 1 item
777 (cond [(equal? in i0) (values '() `(,i0 ,(+ c 1)))] ;repeat
778 [(zero? c) (values '() `(,i0 ,in 0))] ; may be repeating 2
779 [else (values `((rep ,(+ c 1) ,i0)) `(,in 0))])]
780 [(i0 i1 c)
781 (cond [(equal? in i0) (values '() `(,i0 ,i1 ,c 1))] ; may be repeating 2
782 [(equal? in i1)
783 (if (zero? c)
784 (values `(,i0) `(,i1 1))
785 (values `((rep ,(+ c 1) ,i0 ,i1)) `(,in 0)))]
786 [(zero? c) (values `(,i0) `(,i1 ,in 0))]
787 [else (values `((rep ,(+ c 1) ,i0 ,i1)) `(,in 0))])]
788 [(i0 i1 c 1) ; we saw previous input == i0
789 (cond [(equal? in i1) (values '() `(,i0 ,i1 ,(+ c 1)))] ; repeating
790 [(equal? in i0) ; we saw [i0 i1 i0 i0]
791 (if (zero? c)
792 (values `(,i0 ,i1) `(,i0 1))
793 (values `((rep ,(+ c 1) ,i0 ,i1)) `(,i0 1)))]
794 [(zero? c) (values `(,i0 ,i1) `(,i0 ,in 0))]
795 [else (values `((rep ,(+ c 1) ,i0 ,i1)) `(,i0 ,in 0))])]
796 ))
797 (define (finish seed)
798 (match seed
799 [#f '()]
800 [(i0 0) `(,i0)]
801 [(i0 c) `((rep ,(+ c 1) ,i0))]
802 [(i0 i1 0) `(,i0 ,i1)]
803 [(i0 i1 c) `((rep ,(+ c 1) ,i0 ,i1))]
804 [(i0 i1 c 1) `((rep ,(+ c 1) ,i0 ,i1) ,i0)]))
805 (gbuffer-filter compress #f input-gen finish))
806
807 (define (rle-decompressiong-generator input-gen)
808 ;; Compressed a -> Generator a
809 (define (entry->gen entry)
810 (match entry
811 [('rep c . items) (gflatten (gmap (^_ items) (giota c)))]
812 [item (list->generator (list item))]))
813 ($ gconcatenate $ gmap entry->gen input-gen))