| 1 |
;;; |
| 2 |
;;; text.unicode.ucd - Unicode character database library |
| 3 |
;;; |
| 4 |
;;; Originally written by Shiro Kawai, 2011 |
| 5 |
;;; Public Domain - use as you like. |
| 6 |
;;; |
| 7 |
|
| 8 |
;; Read Unicode data tables and generates Scheme data structures |
| 9 |
;; for ease of use. |
| 10 |
;; |
| 11 |
;; This data structures are not intended to be lightweight for |
| 12 |
;; in-application use; they are intermediate format to generate |
| 13 |
;; more efficient structure suitable for applications. |
| 14 |
;; The src/gen-unicode.scm script uses this module. |
| 15 |
;; |
| 16 |
;; The applications should never 'use' this module. The info is |
| 17 |
;; available from Gauche core API and gauche.unicode modules. |
| 18 |
|
| 19 |
(define-module text.unicode.ucd |
| 20 |
(use gauche.record) |
| 21 |
(use gauche.dictionary) |
| 22 |
(use gauche.collection) |
| 23 |
(use gauche.uvector) |
| 24 |
(use gauche.charconv) |
| 25 |
(use gauche.generator) |
| 26 |
(use srfi.13) |
| 27 |
(use srfi.42) |
| 28 |
(use util.match) |
| 29 |
(use file.util) |
| 30 |
(use text.csv) |
| 31 |
(export ucd-parse-files |
| 32 |
ucd-get-entry ucd-get-category-ranges |
| 33 |
ucd-get-break-property ucd-get-east-asian-width |
| 34 |
ucd-map-entries |
| 35 |
|
| 36 |
ucd-save-db ucd-load-db ucd-version |
| 37 |
|
| 38 |
ucd-entry ucd-entry-category |
| 39 |
ucd-entry-case-info ucd-entry-special-case-info |
| 40 |
ucd-entry-case-map ucd-entry-alphabetic |
| 41 |
ucd-entry-uppercase ucd-entry-lowercase |
| 42 |
ucd-entry-digit-value |
| 43 |
|
| 44 |
ucd-simple-case-map? |
| 45 |
ucd-simple-case-map-case |
| 46 |
ucd-simple-case-map-offset |
| 47 |
ucd-extended-case-map? |
| 48 |
ucd-extended-case-map-simple-map |
| 49 |
ucd-extended-case-map-special-map |
| 50 |
ucd-extended-case-map-code |
| 51 |
|
| 52 |
ucd-break-property |
| 53 |
ucd-break-property-grapheme |
| 54 |
ucd-break-property-word |
| 55 |
|
| 56 |
ucd-general-categories |
| 57 |
ucd-grapheme-break-properties |
| 58 |
ucd-word-break-properties |
| 59 |
|
| 60 |
ucd-east-asian-widths |
| 61 |
|
| 62 |
eucjp->ucd-entry sjis->ucd-entry |
| 63 |
) |
| 64 |
) |
| 65 |
(select-module text.unicode.ucd) |
| 66 |
|
| 67 |
;;; |
| 68 |
;;; The database record |
| 69 |
;;; |
| 70 |
|
| 71 |
;; We can't simply have codepoint -> entry table for all the codepoints, |
| 72 |
;; since it'll be too huge. We use the fact that the region above |
| 73 |
;; U+20000 has many consecutive characters that shares some of the |
| 74 |
;; properties. |
| 75 |
;; |
| 76 |
;; The slots of unichar-db should be considered private. They may be |
| 77 |
;; changed in future versions. |
| 78 |
|
| 79 |
;; UnicodeData.txt format |
| 80 |
;; code |
| 81 |
;; name |
| 82 |
;; category |
| 83 |
;; combining-class |
| 84 |
;; bidi-class |
| 85 |
;; decomposition |
| 86 |
;; numeric-value-1 (for decimal only) |
| 87 |
;; numeric-value-2 (for decimal & digits) |
| 88 |
;; numeric-value-3 (for decimal, digits & numeric) |
| 89 |
;; bidi-mirrored |
| 90 |
;; old-name |
| 91 |
;; notes |
| 92 |
;; simple-uppercase |
| 93 |
;; simple-lowercase |
| 94 |
;; simple-titlecase |
| 95 |
|
| 96 |
(define-record-type unichar-db %make-unichar-db #f |
| 97 |
;; Unicode version, in string, e.g. "6.0.3" |
| 98 |
(version) |
| 99 |
;; codepoint -> ucd-entry |
| 100 |
;; The 'table' maps characters below U+20000 to the ucd-entry record. |
| 101 |
;; It is a hashtable keyed by codepoint. |
| 102 |
table |
| 103 |
;; start-codepoint -> Maybe category |
| 104 |
;; This is only used for chars over U+20000. |
| 105 |
ranges |
| 106 |
;; codepoint -> ucd-break-property |
| 107 |
break-table |
| 108 |
;; codepoint -> east-asian-width |
| 109 |
width-table |
| 110 |
) |
| 111 |
|
| 112 |
(define (make-unichar-db) |
| 113 |
(%make-unichar-db #f |
| 114 |
(make-hash-table 'eqv?) |
| 115 |
(make-hash-table 'eqv?) |
| 116 |
(make-hash-table 'eqv?) |
| 117 |
(make-hash-table 'eqv?))) |
| 118 |
|
| 119 |
;; equal? method is mainly for testing |
| 120 |
(define-method object-equal? ((a unichar-db) (b unichar-db)) |
| 121 |
(define (dict-equal? ta tb) |
| 122 |
(and (= (size-of ta) (size-of tb)) |
| 123 |
(every (^k (equal? (dict-get ta k) (dict-get tb k))) |
| 124 |
(dict-keys ta)))) |
| 125 |
(and (dict-equal? (unichar-db-table a) (unichar-db-table b)) |
| 126 |
(dict-equal? (unichar-db-ranges a) (unichar-db-ranges b)) |
| 127 |
(dict-equal? (unichar-db-break-table a) (unichar-db-break-table b)))) |
| 128 |
|
| 129 |
(define-record-type ucd-entry %make-ucd-entry #f |
| 130 |
(category) |
| 131 |
(case-info) ; (upper lower title) codepoints |
| 132 |
(special-case-info) ; ((upper ...) (lower ...) (title ...)) codepoints |
| 133 |
(case-map) ; either ucd-simple-case-map or ucd-extended-case-map |
| 134 |
; set by assign-case-mapping |
| 135 |
(alphabetic) ; #t if Alphabetic |
| 136 |
(uppercase) ; #t if Uppercase |
| 137 |
(lowercase) ; #t if Lowercase |
| 138 |
(digit-value) ; 0..9 for Nd chars, #f otherwise |
| 139 |
) |
| 140 |
|
| 141 |
(define-method object-equal? ((a ucd-entry) (b ucd-entry)) |
| 142 |
(and (eq? (ucd-entry-category a) (ucd-entry-category b)) |
| 143 |
(equal? (ucd-entry-case-map a) (ucd-entry-case-map b)) |
| 144 |
(eq? (ucd-entry-alphabetic a) (ucd-entry-alphabetic b)) |
| 145 |
(eq? (ucd-entry-uppercase a) (ucd-entry-uppercase b)) |
| 146 |
(eq? (ucd-entry-lowercase a) (ucd-entry-lowercase b)) |
| 147 |
(eq? (ucd-entry-digit-value a) (ucd-entry-digit-value b)))) |
| 148 |
|
| 149 |
(define-record-type ucd-break-property %make-ucd-break-property #f |
| 150 |
(grapheme) ; Grapheme_Break category |
| 151 |
(word) ; Word_Break category |
| 152 |
) |
| 153 |
|
| 154 |
(define-method object-equal? ((a ucd-break-property) (b ucd-break-property)) |
| 155 |
(and (eq? (ucd-break-property-grapheme a) (ucd-break-property-grapheme b)) |
| 156 |
(eq? (ucd-break-property-word a) (ucd-break-property-word b)))) |
| 157 |
|
| 158 |
(define (make-ucd-entry category case-info digit-value) |
| 159 |
(%make-ucd-entry category case-info #f #f #f #f #f digit-value)) |
| 160 |
|
| 161 |
(define (make-ucd-break-property) |
| 162 |
(%make-ucd-break-property 'Other 'Other)) |
| 163 |
|
| 164 |
;; API |
| 165 |
(define (ucd-get-entry db ucs) |
| 166 |
(dict-get (unichar-db-table db) ucs #f)) |
| 167 |
|
| 168 |
;; API |
| 169 |
;; Call (proc code ucd-entry) for all entries in unichar-db-table |
| 170 |
;; NB: The db table only contains entries up to U+20000. Higher entries |
| 171 |
;; has to be dealt with separately. |
| 172 |
(define (ucd-map-entries db proc) |
| 173 |
(dict-map (unichar-db-table db) proc)) |
| 174 |
|
| 175 |
;; API |
| 176 |
;; ((<start-codepoint> . <category-symbol>) ...) |
| 177 |
(define (ucd-get-category-ranges db) |
| 178 |
(sort (coerce-to <list> (unichar-db-ranges db)) < car)) |
| 179 |
|
| 180 |
;; API |
| 181 |
;; return break property |
| 182 |
(define (ucd-get-break-property db code) |
| 183 |
(dict-get (unichar-db-break-table db) code #f)) |
| 184 |
|
| 185 |
;; API |
| 186 |
;; return East Asian Width |
| 187 |
(define (ucd-get-east-asian-width db code) |
| 188 |
(dict-get (unichar-db-width-table db) code 'N)) |
| 189 |
|
| 190 |
(define-record-type ucd-simple-case-map #t #t |
| 191 |
case ; upper or lower, for this character itself (title==upper) |
| 192 |
offset) ; offset to convert to the opposite case |
| 193 |
|
| 194 |
(define-method object-equal? ((a ucd-simple-case-map) |
| 195 |
(b ucd-simple-case-map)) |
| 196 |
(and (eq? (ucd-simple-case-map-case a) (ucd-simple-case-map-case b)) |
| 197 |
(eqv? (ucd-simple-case-map-offset a) (ucd-simple-case-map-offset b)))) |
| 198 |
|
| 199 |
(define-record-type ucd-extended-case-map #t #t |
| 200 |
code ; original code |
| 201 |
simple-map ; (upper-off lower-off title-off) or #f |
| 202 |
special-map) ; ((upper ... ) (lower ...) (title ...)) or #f |
| 203 |
|
| 204 |
(define-method object-equal? ((a ucd-extended-case-map) |
| 205 |
(b ucd-extended-case-map)) |
| 206 |
(and (eqv? (ucd-extended-case-map-code a) |
| 207 |
(ucd-extended-case-map-code b)) |
| 208 |
(equal? (ucd-extended-case-map-simple-map a) |
| 209 |
(ucd-extended-case-map-simple-map b)) |
| 210 |
(equal? (ucd-extended-case-map-special-map a) |
| 211 |
(ucd-extended-case-map-special-map b)))) |
| 212 |
|
| 213 |
;; These order must match with the enum in src/gauche/char_attr.h |
| 214 |
(define (ucd-general-categories) |
| 215 |
'(Lu Ll Lt Lm Lo Mn Mc Me Nd Nl No Pc Pd Ps Pe Pi Pf Po |
| 216 |
Sm Sc Sk So Zs Zl Zp Cc Cf Cs Co Cn)) |
| 217 |
(define (ucd-east-asian-widths) |
| 218 |
'(A F H N Na W)) |
| 219 |
|
| 220 |
;; Break properties. The actual property values are in |
| 221 |
;; src/gauche/priv/unicode_attr.h, which is generated by src/gen-unicode.scm. |
| 222 |
;; NB: We use 'Other instead of 'Any in UAX#29. |
| 223 |
;; NB: 'ExtPict is not defined in Grapheme Break Properties, but in |
| 224 |
;; Emoji Properties. It is required to implement GB11 and WB3c rule |
| 225 |
;; (see ext/gauche/unicode.scm) |
| 226 |
(define (ucd-grapheme-break-properties) |
| 227 |
'(CR LF Control Extend ZWJ Regional_Indicator Prepend SpacingMark |
| 228 |
L V T LV LVT ExtPict Other)) |
| 229 |
|
| 230 |
(define (ucd-word-break-properties) |
| 231 |
'(CR LF Newline Extend ZWJ Regional_Indicator |
| 232 |
Format Katakana Hebrew_Letter ALetter Single_Quote Double_Quote |
| 233 |
MidLetter MidNum MidNumLet Numeric ExtendNumLet WSegSpace |
| 234 |
ExtPict ExtPict_ALetter Other)) |
| 235 |
|
| 236 |
;; |
| 237 |
;; CES encoding conversions. |
| 238 |
;; |
| 239 |
|
| 240 |
;; We don't have utf8->ucs4 yet in 0.9.1, so we need to roll our own. |
| 241 |
(define (utf8vec->ucs4 vec) |
| 242 |
(let1 b0 (~ vec 0) |
| 243 |
(cond [(< b0 #x80) b0] |
| 244 |
[(< b0 #xe0) (+ (ash (logand b0 #x1f) 6) |
| 245 |
(logand (~ vec 1) #x3f))] |
| 246 |
[(< b0 #xf0) (+ (ash (logand b0 #x0f) 12) |
| 247 |
(ash (logand (~ vec 1) #x3f) 6) |
| 248 |
(logand (~ vec 2) #x3f))] |
| 249 |
[else (+ (ash (logand b0 #x07) 18) |
| 250 |
(ash (logand (~ vec 1) #x3f) 12) |
| 251 |
(ash (logand (~ vec 2) #x3f) 6) |
| 252 |
(logand (~ vec 3) #x3f))]))) |
| 253 |
|
| 254 |
(define (get-char/ces vec ces) |
| 255 |
(guard (e [else #f]) |
| 256 |
(utf8vec->ucs4 |
| 257 |
(string->u8vector |
| 258 |
(ces-convert (u8vector->string vec) ces 'utf-8))))) |
| 259 |
|
| 260 |
(define (2b->u8vector code) |
| 261 |
(u8vector (ash code -8) (logand code #xff))) |
| 262 |
(define (3b->u8vector code) |
| 263 |
(u8vector (ash code -16) (logand (ash code -8) #xff) (logand code #xff))) |
| 264 |
|
| 265 |
;; can return #f |
| 266 |
(define (eucjp->ucs code) |
| 267 |
;; NB: The codes in *jisx0213-extras* should be filtered before |
| 268 |
;; calling this. |
| 269 |
(and-let* ([v (cond [(< code #x80) (u8vector code)] |
| 270 |
[(<= #x8ea1 code #x8ef2) (2b->u8vector code)] |
| 271 |
[(<= #xa1a1 code #xfefe) (2b->u8vector code)] |
| 272 |
[(<= #x8fa1a1 code #x8ffefe) (3b->u8vector code)] |
| 273 |
[else #f])]) |
| 274 |
(get-char/ces v 'euc-jp))) |
| 275 |
|
| 276 |
;; can return #f |
| 277 |
(define (sjis->ucs code) |
| 278 |
;; NB: The codes in *jisx0213-extras* should be filtered before |
| 279 |
;; calling this. |
| 280 |
(and-let* ([v (cond [(< code #x80) (u8vector code)] |
| 281 |
[(<= #xa0 code #xdf) (u8vector code)] |
| 282 |
[(<= #x8040 code #x8ffc) (2b->u8vector code)] |
| 283 |
[(<= #x9040 code #x9ffc) (2b->u8vector code)] |
| 284 |
[(<= #xe040 code #xeffc) (2b->u8vector code)] |
| 285 |
[(<= #xf040 code #xfffc) (2b->u8vector code)] |
| 286 |
[else #f])]) |
| 287 |
(get-char/ces v 'sjis))) |
| 288 |
|
| 289 |
;; NB: JISX0213:2004 has 25 letters that aren't included in Unicode 6.0. |
| 290 |
;; We define general categories of them by our own. |
| 291 |
(define *jisx0213-extras* |
| 292 |
'(;; sjis euc-jp category |
| 293 |
(#x82F5 #xA4F7 Lo) ; U+304B U+309A Kana+semi voice mark |
| 294 |
(#x82F6 #xA4F8 Lo) ; U+304D U+309A |
| 295 |
(#x82F7 #xA4F9 Lo) ; U+304F U+309A |
| 296 |
(#x82F8 #xA4FA Lo) ; U+3051 U+309A |
| 297 |
(#x82F9 #xA4FB Lo) ; U+3053 U+309A |
| 298 |
(#x8397 #xA5F7 Lo) ; U+30AB U+309A |
| 299 |
(#x8398 #xA5F8 Lo) ; U+30AD U+309A |
| 300 |
(#x8399 #xA5F9 Lo) ; U+30AF U+309A |
| 301 |
(#x839A #xA5FA Lo) ; U+30B1 U+309A |
| 302 |
(#x839B #xA5FB Lo) ; U+30B3 U+309A |
| 303 |
(#x839C #xA5FC Lo) ; U+30BB U+309A |
| 304 |
(#x839D #xA5FD Lo) ; U+30C4 U+309A |
| 305 |
(#x839E #xA5FE Lo) ; U+30C8 U+309A |
| 306 |
(#x83F6 #xA6F8 Lo) ; U+31F7 U+309A |
| 307 |
(#x8663 #xABC4 Ll) ; U+00E6 U+0300 IPA+accent |
| 308 |
(#x8667 #xABC8 Ll) ; U+0254 U+0300 |
| 309 |
(#x8668 #xABC9 Ll) ; U+0254 U+0301 |
| 310 |
(#x8669 #xABCA Ll) ; U+028C U+0300 |
| 311 |
(#x866A #xABCB Ll) ; U+028C U+0301 |
| 312 |
(#x866B #xABCC Ll) ; U+0259 U+0300 |
| 313 |
(#x866C #xABCD Ll) ; U+0259 U+0301 |
| 314 |
(#x866D #xABCE Ll) ; U+025A U+0300 |
| 315 |
(#x866E #xABCF Ll) ; U+025A U+0301 |
| 316 |
(#x8685 #xABE5 Sk) ; U+02E9 U+02E5 Modifier |
| 317 |
(#x8686 #xABE6 Sk) ; U+02E5 U+02E9 Modifier |
| 318 |
)) |
| 319 |
|
| 320 |
(define (xxx->entry db code get-extra-code ->ucs) |
| 321 |
(cond [(find (^l (eqv? code (get-extra-code l))) *jisx0213-extras*) |
| 322 |
=> (^l (let1 cat (caddr l) |
| 323 |
(rlet1 e (make-ucd-entry cat '(#f #f #f) #f) |
| 324 |
(when (eq? cat 'Ll) (set! (ucd-entry-lowercase e) #t)) |
| 325 |
(when (eq? cat 'Lo) (set! (ucd-entry-alphabetic e) #t)))))] |
| 326 |
[(->ucs code) |
| 327 |
=> (^c (dict-get (unichar-db-table db) c #f))] |
| 328 |
[else #f])) |
| 329 |
|
| 330 |
;; The following APIs are needed since we have to handle jisx0213 extra |
| 331 |
;; characters. |
| 332 |
|
| 333 |
;; API |
| 334 |
;; can return #f if CODE is 'hole' (unassigned) |
| 335 |
(define (eucjp->ucd-entry db code) |
| 336 |
(xxx->entry db code cadr eucjp->ucs)) |
| 337 |
|
| 338 |
;; API |
| 339 |
;; can return #f if CODE is 'hole' (unassigned) |
| 340 |
(define (sjis->ucd-entry db code) |
| 341 |
(xxx->entry db code car sjis->ucs)) |
| 342 |
|
| 343 |
;;; |
| 344 |
;;; Parsing UnicodeData and other tables. |
| 345 |
;;; |
| 346 |
|
| 347 |
(define (ucd-parse-files datadir) |
| 348 |
(check-data-files datadir) |
| 349 |
(let1 db (unicode-data (build-path datadir "UnicodeData.txt")) |
| 350 |
(check-assumptions db) |
| 351 |
(set-unicode-version! db (build-path datadir "SpecialCasing.txt")) |
| 352 |
(special-case-mapping (build-path datadir "SpecialCasing.txt") db) |
| 353 |
(additional-case-entries (build-path datadir "PropList.txt") db) |
| 354 |
(assign-case-mapping db) |
| 355 |
(grapheme-break-property |
| 356 |
(ucd-grapheme-break-properties) |
| 357 |
(build-path datadir "auxiliary/GraphemeBreakProperty.txt") db) |
| 358 |
(word-break-property |
| 359 |
(ucd-word-break-properties) |
| 360 |
(build-path datadir "auxiliary/WordBreakProperty.txt") db) |
| 361 |
(augument-break-property |
| 362 |
(build-path datadir "emoji/emoji-data.txt") db) |
| 363 |
(east-asian-width (build-path datadir "EastAsianWidth.txt") db) |
| 364 |
db)) |
| 365 |
|
| 366 |
(define-constant +required-files+ |
| 367 |
'("UnicodeData.txt" "SpecialCasing.txt" "PropList.txt" "EastAsianWidth.txt" |
| 368 |
"auxiliary/GraphemeBreakProperty.txt" "auxiliary/WordBreakProperty.txt")) |
| 369 |
|
| 370 |
(define (check-data-files dir) |
| 371 |
(dolist [f +required-files+ #t] |
| 372 |
(unless (file-exists? (build-path dir f)) |
| 373 |
(errorf "Couldn't find datafile `~a' in ~a; the directory should \ |
| 374 |
contain these files: ~a" f dir +required-files+)))) |
| 375 |
|
| 376 |
;; Process UnicodeData |
| 377 |
(define (unicode-data file) |
| 378 |
(define db (make-unichar-db)) |
| 379 |
;; state := (prev-code prev-cat range?) |
| 380 |
(call-with-input-file file |
| 381 |
(^p (port-fold (unicode-data-1 db) '(#f #f #f) |
| 382 |
(cut (make-csv-reader #\;) p)))) |
| 383 |
db) |
| 384 |
|
| 385 |
(define (dict-fill! dic start end ent) |
| 386 |
(dotimes [i (- end start)] |
| 387 |
(dict-put! dic (+ start i) ent))) |
| 388 |
|
| 389 |
(define (unicode-data-1 db) |
| 390 |
(define table (unichar-db-table db)) |
| 391 |
(define ranges (unichar-db-ranges db)) |
| 392 |
(^[entry state] |
| 393 |
(match-let ([(scode name scat _ _ _ snum _ _ _ _ _ sup slo sti) entry] |
| 394 |
[(prev-code prev-cat range?) state]) |
| 395 |
(let* ([code (parse-code scode)] |
| 396 |
[range-start? (#/First>/ name)] |
| 397 |
[cat (string->symbol scat)] |
| 398 |
[case-info `(,(parse-code sup) ,(parse-code slo) ,(parse-code sti))] |
| 399 |
[entry (make-ucd-entry cat case-info (string->number snum))]) |
| 400 |
;; register table entry |
| 401 |
(cond |
| 402 |
[(not prev-code) ; initial loop |
| 403 |
(dict-put! table code entry)] |
| 404 |
[(and range? (not (= code (+ prev-code 1)))) ;gap |
| 405 |
(dict-fill! table (+ prev-code 1) (+ code 1) entry)] |
| 406 |
[else |
| 407 |
(dict-put! table code entry)]) |
| 408 |
;; insert category range over U+20000 |
| 409 |
(cond [(= code #x20000) (dict-put! ranges code cat)] ; start |
| 410 |
[(= code #x10fffd) (dict-put! ranges #x10fffe #f)] ;end |
| 411 |
[(> code #x20000) |
| 412 |
(cond [(not (= code (+ prev-code 1))) |
| 413 |
(unless range? |
| 414 |
(dict-put! ranges (+ prev-code 1) #f) |
| 415 |
(dict-put! ranges code cat))] |
| 416 |
[(not (eq? cat prev-cat)) |
| 417 |
(dict-put! ranges code cat)])]) |
| 418 |
`(,code ,cat ,range-start?))))) |
| 419 |
|
| 420 |
(define (check-assumptions db) |
| 421 |
(define table (unichar-db-table db)) |
| 422 |
;; Check if U+2xxxx range only has Lo and no case mappings. |
| 423 |
(dotimes [i #x10000] |
| 424 |
(let1 e (dict-get table (+ i #x20000) #f) |
| 425 |
(when (and e (or (not (eq? (ucd-entry-category e) 'Lo)) |
| 426 |
(not (equal? (ucd-entry-case-info e) '(#f #f #f))))) |
| 427 |
(errorf "Unicode assumption failed: U+~x ~s ~s ~s" |
| 428 |
(+ i #x20000) |
| 429 |
(ucd-entry-category e) |
| 430 |
(ucd-entry-case-info e) |
| 431 |
(ucd-entry-special-case-info e))))) |
| 432 |
) |
| 433 |
|
| 434 |
(define (parse-code str) (string->number str 16)) |
| 435 |
|
| 436 |
;; Get Unicode version number from a data file |
| 437 |
;; UCD data files (except UnicodeData.txt) begins with a comment line |
| 438 |
;; like this: |
| 439 |
;; # FileName-X.X.X.txt |
| 440 |
;; where the actual file name is FileName.txt, and X.X.X is the version |
| 441 |
;; number. We use it to determine unicode version heuristically. |
| 442 |
(define (get-unicode-version file) |
| 443 |
(rxmatch-let (#/^(\w+)\.txt$/ (sys-basename file)) [_ base] |
| 444 |
(let* ([rx (string->regexp #"~|base|-(\\d+\\.\\d+\\.\\d+)\\.txt")] |
| 445 |
[line (with-input-from-file file read-line)] |
| 446 |
[m (rx line)]) |
| 447 |
(and m (m 1))))) |
| 448 |
|
| 449 |
;; API |
| 450 |
(define (ucd-version db) (unichar-db-version db)) |
| 451 |
|
| 452 |
(define (set-unicode-version! db . files) |
| 453 |
(set! (unichar-db-version db) (any get-unicode-version files))) |
| 454 |
|
| 455 |
|
| 456 |
;; Augument the db with SpecialCasing.txt. |
| 457 |
;; We ignore entries with conditional mappings. The only language-insensitive |
| 458 |
;; conditional mapping is the final form of sigma: |
| 459 |
;; 03A3; 03C2; 03A3; 03A3; Final_Sigma; # GREEK CAPITAL LETTER SIGMA |
| 460 |
;; We handle this one explicitly. |
| 461 |
(define (special-case-mapping file db) |
| 462 |
(define table (unichar-db-table db)) |
| 463 |
(define (handle line) |
| 464 |
(if-let1 sp (rxmatch->string #/^([0-9a-fA-F][^#]*)/ line 1) |
| 465 |
(match-let1 (scode slo sti sup sconditions . _) (string-split sp #/\;\s*/) |
| 466 |
(when (string-null? sconditions) |
| 467 |
(if-let1 code (parse-code scode) |
| 468 |
(let ([lo (map parse-code (string-split slo #[\s]))] |
| 469 |
[ti (map parse-code (string-split sti #[\s]))] |
| 470 |
[up (map parse-code (string-split sup #[\s]))] |
| 471 |
[e (dict-get table code)]) |
| 472 |
(set! (ucd-entry-special-case-info e) (list up lo ti)))))))) |
| 473 |
(with-input-from-file file |
| 474 |
(cut generator-for-each handle read-line))) |
| 475 |
|
| 476 |
;; Assign case-map slot by either simple-case-map or extended-case-map |
| 477 |
(define (assign-case-mapping db) |
| 478 |
(define (handle code entry) |
| 479 |
(set! (ucd-entry-case-map entry) |
| 480 |
(if (ucd-entry-special-case-info entry) |
| 481 |
(make-ucd-extended-case-map code |
| 482 |
(map (^s (and s (- s code))) |
| 483 |
(ucd-entry-case-info entry)) |
| 484 |
(ucd-entry-special-case-info entry)) |
| 485 |
(match (ucd-entry-case-info entry) |
| 486 |
[#f #f] |
| 487 |
[(#f #f #f) #f] |
| 488 |
[(#f (? number? lower) #f) (=> fail) |
| 489 |
(if (<= -8192 (- lower code) 8191) |
| 490 |
(make-ucd-simple-case-map 'upper (- lower code)) |
| 491 |
(fail))] |
| 492 |
[((? number? upper) #f title) (=> fail) |
| 493 |
(if (and (eqv? upper title) |
| 494 |
(<= -8192 (- upper code) 8191)) |
| 495 |
(make-ucd-simple-case-map 'lower (- upper code)) |
| 496 |
(fail))] |
| 497 |
[_ ($ make-ucd-extended-case-map code |
| 498 |
(map (^s (and s (- s code))) |
| 499 |
(ucd-entry-case-info entry)) |
| 500 |
(ucd-entry-special-case-info entry))])))) |
| 501 |
(hash-table-for-each (unichar-db-table db) handle)) |
| 502 |
|
| 503 |
(define (additional-case-entries file db) |
| 504 |
(define table (unichar-db-table db)) |
| 505 |
(define (parse line) |
| 506 |
(rxmatch-case line |
| 507 |
[#/^([0-9A-Fa-f]+)(\.\.([0-9A-Fa-f]+))?\s+\;\s+(\w+)/ (#f s #f e p) |
| 508 |
(list (string->symbol p) |
| 509 |
(parse-code s) |
| 510 |
(if e (parse-code e) (parse-code s)))] |
| 511 |
[else #f])) |
| 512 |
(define (flag start end proc) |
| 513 |
(do-ec (: c start (+ end 1)) (set! (proc (dict-get table c)) #t))) |
| 514 |
(define (handle line) |
| 515 |
(match (parse line) |
| 516 |
[('Other_Alphabetic s e) (flag s e ucd-entry-alphabetic)] |
| 517 |
[('Other_Uppercase s e) (flag s e ucd-entry-uppercase)] |
| 518 |
[('Other_Lowercase s e) (flag s e ucd-entry-lowercase)] |
| 519 |
[_ #f])) |
| 520 |
|
| 521 |
(dict-for-each table |
| 522 |
(^(k v) |
| 523 |
(case (ucd-entry-category v) |
| 524 |
[(Lu) (set! (ucd-entry-alphabetic v) #t) |
| 525 |
(set! (ucd-entry-uppercase v) #t)] |
| 526 |
[(Ll) (set! (ucd-entry-alphabetic v) #t) |
| 527 |
(set! (ucd-entry-lowercase v) #t)] |
| 528 |
[(Lt Lm Lo Nl) (set! (ucd-entry-alphabetic v) #t)]))) |
| 529 |
(with-input-from-file file |
| 530 |
(cut generator-for-each handle read-line))) |
| 531 |
|
| 532 |
(define (east-asian-width file db) |
| 533 |
(define table (unichar-db-width-table db)) |
| 534 |
(define (parse line) |
| 535 |
(rxmatch-case line |
| 536 |
[#/^([0-9A-Fa-f]+)(\.\.([0-9A-Fa-f]+))?\s*\;\s*(\w+)/ (#f s #f e p) |
| 537 |
(list (string->symbol p) |
| 538 |
(parse-code s) |
| 539 |
(if e (parse-code e) (parse-code s)))] |
| 540 |
[else #f])) |
| 541 |
(define (handle line) |
| 542 |
(match (parse line) |
| 543 |
[(width start end) |
| 544 |
(do-ec (: c start (+ end 1)) |
| 545 |
(dict-put! table c width))] |
| 546 |
[_ #f])) |
| 547 |
(with-input-from-file file |
| 548 |
(cut generator-for-each handle read-line))) |
| 549 |
|
| 550 |
(define (ensure-ucd-break-property db code) |
| 551 |
(let1 table (unichar-db-break-table db) |
| 552 |
(or (dict-get table code #f) |
| 553 |
(rlet1 e (make-ucd-break-property) |
| 554 |
(dict-put! table code e))))) |
| 555 |
|
| 556 |
(define ((break-property accessor) symbols file db) |
| 557 |
(define (check str) |
| 558 |
(rlet1 sym (string->symbol str) |
| 559 |
(unless (memq sym symbols) |
| 560 |
(errorf "Unrecognized break property `~a'. This must be a newer \ |
| 561 |
version of UnicodeData. The code needs to be updated." sym)))) |
| 562 |
(define (handle line) |
| 563 |
(rxmatch-case line |
| 564 |
[#/^([0-9a-fA-F]{4,6})\.\.([0-9a-fA-F]{4,6})\s+\;\s+(\w+)/ (_ ss ee cc) |
| 565 |
(do-ec (: c (parse-code ss) (+ (parse-code ee) 1)) |
| 566 |
(let1 e (ensure-ucd-break-property db c) |
| 567 |
(set! (accessor e) (check cc))))] |
| 568 |
[#/^([0-9a-fA-F]{4,6})\s+\;\s+(\w+)/ (_ ss cc) |
| 569 |
(let1 e (ensure-ucd-break-property db (parse-code ss)) |
| 570 |
(set! (accessor e) (check cc)))])) |
| 571 |
(with-input-from-file file |
| 572 |
(cut generator-for-each handle read-line))) |
| 573 |
|
| 574 |
(define grapheme-break-property (break-property ucd-break-property-grapheme)) |
| 575 |
(define word-break-property (break-property ucd-break-property-word)) |
| 576 |
|
| 577 |
;; Grapheme break property needs to know emoji Extended_Pictographic |
| 578 |
(define (augument-break-property file db) |
| 579 |
(define (ensure c) |
| 580 |
(let1 e (ensure-ucd-break-property db c) |
| 581 |
(case (ucd-break-property-grapheme e) |
| 582 |
[(Other) (set! (ucd-break-property-grapheme e) 'ExtPict)] |
| 583 |
[else (warn "Overriding non-'Other' grapheme property of U+~4,'0x (~s)\n" |
| 584 |
c (ucd-break-property-grapheme e))]) |
| 585 |
(case (ucd-break-property-word e) |
| 586 |
[(Other) (set! (ucd-break-property-word e) 'ExtPict)] |
| 587 |
[(ALetter) (set! (ucd-break-property-word e) 'ExtPict_ALetter)] |
| 588 |
[else (warn "Overriding non-'Other' word property of U+~4,'0x (~s)\n" |
| 589 |
c (ucd-break-property-word e))]))) |
| 590 |
(define (handle line) |
| 591 |
(rxmatch-case line |
| 592 |
[#/^([0-9a-fA-F]{4,6})\.\.([0-9a-fA-F]{4,6})\s+\;\s+Extended_Pictographic/ |
| 593 |
(_ ss ee) |
| 594 |
(do-ec (: c (parse-code ss) (+ (parse-code ee) 1)) |
| 595 |
(ensure c))] |
| 596 |
[#/^([0-9a-fA-F]{4,6})\s+\;\s+Extended_Pictographic/ (_ ss) |
| 597 |
(ensure (parse-code ss))])) |
| 598 |
(with-input-from-file file |
| 599 |
(cut generator-for-each handle read-line))) |
| 600 |
|
| 601 |
;;; |
| 602 |
;;; Dumping and restoring database |
| 603 |
;;; |
| 604 |
|
| 605 |
;; Dump format should be regarded as internal - we may change it at any |
| 606 |
;; tiem as we like. |
| 607 |
|
| 608 |
;; For now, we dump it in three parts, correspoinding to the unichar-db |
| 609 |
;; slots. To suppress the size of the dump, we use simple run-length |
| 610 |
;; compressing at the S-expr level. |
| 611 |
;; |
| 612 |
;; <RLE-compressed-list> : <RLE-compressed-entry> ... |
| 613 |
;; <RLE-compressed-entry> : <entry> |
| 614 |
;; | (rep <count> <entry> ...) |
| 615 |
;; |
| 616 |
;; 1. The low codepoint table part |
| 617 |
;; |
| 618 |
;; RLE compressed list of <ucd-entry> |
| 619 |
;; |
| 620 |
;; <ucd-entry> : <basic-entry> |
| 621 |
;; <basic-entry> : #f ; unassigned |
| 622 |
;; | (<category> [<case-map>] [A] [L] [U] [<digit>]) |
| 623 |
;; <category> : <symbol> ; general category symbol |
| 624 |
;; <case-map> | ( {upper|lower} <offset> ) ; for simple-case-map |
| 625 |
;; | (<code> <simple-map-info> <extended-map-info>) |
| 626 |
;; <simple-map-info> : #f |
| 627 |
;; | (<maybe-offset> <maybe-offset> <maybe-offset>) |
| 628 |
;; <extended-map-info> : #f |
| 629 |
;; | ((<code> ...) (<code> ...) (<code> ...)) |
| 630 |
;; |
| 631 |
;; 2. The upper category ranges |
| 632 |
;; |
| 633 |
;; List of (<start-code> <category>). |
| 634 |
;; |
| 635 |
;; 3. Break properties |
| 636 |
;; |
| 637 |
;; RLE compressed list of (<grapheme-break-property> <word-break-property>). |
| 638 |
;; |
| 639 |
;; 4. East Asian Width properties |
| 640 |
;; |
| 641 |
;; RLE compressed list of <east-asian-width> |
| 642 |
|
| 643 |
|
| 644 |
(define (ucd-save-db db) |
| 645 |
(define (convert-case-map cm) |
| 646 |
(cond [(ucd-simple-case-map? cm) |
| 647 |
`(,(ucd-simple-case-map-case cm) |
| 648 |
,(ucd-simple-case-map-offset cm))] |
| 649 |
[(ucd-extended-case-map? cm) |
| 650 |
`(,(ucd-extended-case-map-code cm) |
| 651 |
,(ucd-extended-case-map-simple-map cm) |
| 652 |
,(ucd-extended-case-map-special-map cm))] |
| 653 |
[else #f])) |
| 654 |
(define (format-ucd-entry e) |
| 655 |
(and e |
| 656 |
`(,(ucd-entry-category e) |
| 657 |
,@(cond-list |
| 658 |
[(convert-case-map (ucd-entry-case-map e))] |
| 659 |
[(ucd-entry-alphabetic e) 'A] |
| 660 |
[(ucd-entry-uppercase e) 'U] |
| 661 |
[(ucd-entry-lowercase e) 'L] |
| 662 |
[(ucd-entry-digit-value e)])))) |
| 663 |
(define (format-break-property-entry e) |
| 664 |
(and e |
| 665 |
`(,(ucd-break-property-grapheme e) |
| 666 |
,(ucd-break-property-word e)))) |
| 667 |
|
| 668 |
(print ";; Generated by text.unicode.ucd from Unicode Character Database") |
| 669 |
(print ";; unicode version") |
| 670 |
(write (unichar-db-version db)) |
| 671 |
(print) |
| 672 |
|
| 673 |
(print ";; unichar-db-table") |
| 674 |
(print "(") |
| 675 |
(let1 tab (unichar-db-table db) |
| 676 |
($ generator-for-each print |
| 677 |
$ rle-compressing-generator |
| 678 |
$ gmap (^c (format-ucd-entry (dict-get tab c #f))) |
| 679 |
$ giota $ + 1 $ apply max $ dict-keys tab)) |
| 680 |
(print ")") |
| 681 |
|
| 682 |
(print ";; unichar-db-high-ranges") |
| 683 |
(print "(") |
| 684 |
(let1 tab (unichar-db-ranges db) |
| 685 |
(dolist [k (sort (dict-keys tab))] |
| 686 |
(print (list k (dict-get tab k))))) |
| 687 |
(print ")") |
| 688 |
|
| 689 |
(print ";; unichar-db-break-table") |
| 690 |
(print "(") |
| 691 |
(let1 tab (unichar-db-break-table db) |
| 692 |
($ generator-for-each print |
| 693 |
$ rle-compressing-generator |
| 694 |
$ gmap (^c (format-break-property-entry (dict-get tab c #f))) |
| 695 |
$ giota $ + 1 $ apply max $ dict-keys tab)) |
| 696 |
(print ")") |
| 697 |
|
| 698 |
(print ";; unichar-db-width-table") |
| 699 |
(print "(") |
| 700 |
(let1 tab (unichar-db-width-table db) |
| 701 |
($ generator-for-each print |
| 702 |
$ rle-compressing-generator |
| 703 |
$ gmap (^c (dict-get tab c #f)) |
| 704 |
$ giota $ + 1 $ apply max $ dict-keys tab)) |
| 705 |
(print ")") |
| 706 |
) |
| 707 |
|
| 708 |
(define (ucd-load-db port) |
| 709 |
(rlet1 db (make-unichar-db) |
| 710 |
|
| 711 |
(define (recover-case-map e) |
| 712 |
(match e |
| 713 |
[#f #f] |
| 714 |
[(case offset) (make-ucd-simple-case-map case offset)] |
| 715 |
[(code simple ext) (make-ucd-extended-case-map code simple ext)])) |
| 716 |
(define (add-entry! e code) |
| 717 |
(and e |
| 718 |
(receive (category case-map flags) |
| 719 |
(match e |
| 720 |
[(c) (values c #f '())] |
| 721 |
[(c (? pair? m) . flags) (values c m flags)] |
| 722 |
[(c . flags) (values c #f flags)]) |
| 723 |
(hash-table-put! (unichar-db-table db) |
| 724 |
code |
| 725 |
(%make-ucd-entry category #f #f |
| 726 |
(recover-case-map case-map) |
| 727 |
(boolean (memq 'A flags)) |
| 728 |
(boolean (memq 'U flags)) |
| 729 |
(boolean (memq 'L flags)) |
| 730 |
(find number? flags)))))) |
| 731 |
|
| 732 |
(define (add-break-property! e code) |
| 733 |
(match e |
| 734 |
[#f #f] |
| 735 |
[(g w) (let1 r (ensure-ucd-break-property db code) |
| 736 |
(ucd-break-property-grapheme-set! r g) |
| 737 |
(ucd-break-property-word-set! r w))])) |
| 738 |
|
| 739 |
(define (add-width-property! e code) |
| 740 |
(match e |
| 741 |
[#f #f] |
| 742 |
[sym (hash-table-put! (unichar-db-width-table db) code sym)])) |
| 743 |
|
| 744 |
;; version |
| 745 |
(set! (unichar-db-version db) (read port)) |
| 746 |
;; ucd-entries |
| 747 |
(let1 entries (read port) |
| 748 |
(generator-for-each add-entry! |
| 749 |
($ rle-decompressiong-generator |
| 750 |
$ list->generator entries) |
| 751 |
(giota))) |
| 752 |
;; high ranges |
| 753 |
(dolist [range (read port)] |
| 754 |
(match-let1 (start val) range |
| 755 |
(dict-put! (unichar-db-ranges db) start val))) |
| 756 |
;; break properties |
| 757 |
(let1 break-properties (read port) |
| 758 |
(generator-for-each add-break-property! |
| 759 |
($ rle-decompressiong-generator |
| 760 |
$ list->generator break-properties) |
| 761 |
(giota))) |
| 762 |
;; width table |
| 763 |
(let1 width-properties (read port) |
| 764 |
(generator-for-each add-width-property! |
| 765 |
($ rle-decompressiong-generator |
| 766 |
$ list->generator width-properties) |
| 767 |
(giota))) |
| 768 |
)) |
| 769 |
|
| 770 |
;; Run-length compressor/decompressor |
| 771 |
(define (rle-compressing-generator input-gen) |
| 772 |
(define (compress in seed) |
| 773 |
(match seed |
| 774 |
[#f ; initial state |
| 775 |
(values '() `(,in 0))] |
| 776 |
[(i0 c) ; may be repeating 1 item |
| 777 |
(cond [(equal? in i0) (values '() `(,i0 ,(+ c 1)))] ;repeat |
| 778 |
[(zero? c) (values '() `(,i0 ,in 0))] ; may be repeating 2 |
| 779 |
[else (values `((rep ,(+ c 1) ,i0)) `(,in 0))])] |
| 780 |
[(i0 i1 c) |
| 781 |
(cond [(equal? in i0) (values '() `(,i0 ,i1 ,c 1))] ; may be repeating 2 |
| 782 |
[(equal? in i1) |
| 783 |
(if (zero? c) |
| 784 |
(values `(,i0) `(,i1 1)) |
| 785 |
(values `((rep ,(+ c 1) ,i0 ,i1)) `(,in 0)))] |
| 786 |
[(zero? c) (values `(,i0) `(,i1 ,in 0))] |
| 787 |
[else (values `((rep ,(+ c 1) ,i0 ,i1)) `(,in 0))])] |
| 788 |
[(i0 i1 c 1) ; we saw previous input == i0 |
| 789 |
(cond [(equal? in i1) (values '() `(,i0 ,i1 ,(+ c 1)))] ; repeating |
| 790 |
[(equal? in i0) ; we saw [i0 i1 i0 i0] |
| 791 |
(if (zero? c) |
| 792 |
(values `(,i0 ,i1) `(,i0 1)) |
| 793 |
(values `((rep ,(+ c 1) ,i0 ,i1)) `(,i0 1)))] |
| 794 |
[(zero? c) (values `(,i0 ,i1) `(,i0 ,in 0))] |
| 795 |
[else (values `((rep ,(+ c 1) ,i0 ,i1)) `(,i0 ,in 0))])] |
| 796 |
)) |
| 797 |
(define (finish seed) |
| 798 |
(match seed |
| 799 |
[#f '()] |
| 800 |
[(i0 0) `(,i0)] |
| 801 |
[(i0 c) `((rep ,(+ c 1) ,i0))] |
| 802 |
[(i0 i1 0) `(,i0 ,i1)] |
| 803 |
[(i0 i1 c) `((rep ,(+ c 1) ,i0 ,i1))] |
| 804 |
[(i0 i1 c 1) `((rep ,(+ c 1) ,i0 ,i1) ,i0)])) |
| 805 |
(gbuffer-filter compress #f input-gen finish)) |
| 806 |
|
| 807 |
(define (rle-decompressiong-generator input-gen) |
| 808 |
;; Compressed a -> Generator a |
| 809 |
(define (entry->gen entry) |
| 810 |
(match entry |
| 811 |
[('rep c . items) (gflatten (gmap (^_ items) (giota c)))] |
| 812 |
[item (list->generator (list item))])) |
| 813 |
($ gconcatenate $ gmap entry->gen input-gen)) |