Debian Codegraph @

internal/export/unicode/gen.go

raw
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ignore
6
7 // Unicode table generator.
8 // Data read from the web.
9
10 package main
11
12 import (
13 "flag"
14 "fmt"
15 "log"
16 "maps"
17 "os"
18 "regexp"
19 "slices"
20 "sort"
21 "strings"
22 "unicode"
23
24 "golang.org/x/text/internal/gen"
25 "golang.org/x/text/internal/ucd"
26 "golang.org/x/text/unicode/rangetable"
27 )
28
29 func main() {
30 flag.Parse()
31 setupOutput()
32 loadChars() // always needed
33 loadCasefold()
34 printCategories()
35 printScriptOrProperty(false)
36 printScriptOrProperty(true)
37 printCases()
38 printLatinProperties()
39 printCasefold()
40 printSizes()
41 flushOutput()
42 }
43
44 func defaultVersion() string {
45 if v := os.Getenv("UNICODE_VERSION"); v != "" {
46 return v
47 }
48 return unicode.Version
49 }
50
51 var tablelist = flag.String("tables",
52 "all",
53 "comma-separated list of which tables to generate; can be letter")
54 var scriptlist = flag.String("scripts",
55 "all",
56 "comma-separated list of which script tables to generate")
57 var proplist = flag.String("props",
58 "all",
59 "comma-separated list of which property tables to generate")
60 var cases = flag.Bool("cases",
61 true,
62 "generate case tables")
63 var test = flag.Bool("test",
64 false,
65 "test existing tables; can be used to compare web data with package data")
66
67 var scriptRe = regexp.MustCompile(`^([0-9A-F]+)(\.\.[0-9A-F]+)? *; ([A-Za-z_]+)$`)
68 var logger = log.New(os.Stderr, "", log.Lshortfile)
69
70 var output *gen.CodeWriter
71
72 func setupOutput() {
73 output = gen.NewCodeWriter()
74 }
75
76 func flushOutput() {
77 output.WriteGoFile("tables.go", "unicode")
78 }
79
80 func printf(format string, args ...interface{}) {
81 fmt.Fprintf(output, format, args...)
82 }
83
84 func print(args ...interface{}) {
85 fmt.Fprint(output, args...)
86 }
87
88 func println(args ...interface{}) {
89 fmt.Fprintln(output, args...)
90 }
91
92 var category = map[string]bool{
93 // Nd Lu etc.
94 // We use one-character names to identify merged categories
95 "L": true, // Lu Ll Lt Lm Lo
96 "LC": true, // Lu Ll Lt
97 "P": true, // Pc Pd Ps Pe Pu Pf Po
98 "M": true, // Mn Mc Me
99 "N": true, // Nd Nl No
100 "S": true, // Sm Sc Sk So
101 "Z": true, // Zs Zl Zp
102 "C": true, // Cc Cf Cs Co Cn
103 "Cn": true, // unassigned
104 }
105
106 // This contains only the properties we're interested in.
107 type Char struct {
108 codePoint rune // if zero, this index is not a valid code point.
109 category string
110 upperCase rune
111 lowerCase rune
112 titleCase rune
113 foldCase rune // simple case folding
114 caseOrbit rune // next in simple case folding orbit
115 }
116
117 const MaxChar = 0x10FFFF
118
119 var chars = make([]Char, MaxChar+1)
120 var scripts = make(map[string][]rune)
121 var props = make(map[string][]rune) // a property looks like a script; can share the format
122
123 func allCategories() []string {
124 a := make([]string, 0, len(category))
125 for k := range category {
126 a = append(a, k)
127 }
128 sort.Strings(a)
129 return a
130 }
131
132 func all(scripts map[string][]rune) []string {
133 a := make([]string, 0, len(scripts))
134 for k := range scripts {
135 a = append(a, k)
136 }
137 sort.Strings(a)
138 return a
139 }
140
141 func allCatFold(m map[string]map[rune]bool) []string {
142 a := make([]string, 0, len(m))
143 for k := range m {
144 a = append(a, k)
145 }
146 sort.Strings(a)
147 return a
148 }
149
150 func categoryOp(code rune, class uint8) bool {
151 category := chars[code].category
152 return len(category) > 0 && category[0] == class
153 }
154
155 func loadChars() {
156 for code := range chars {
157 chars[code].category = "Cn" // unassigned
158 }
159 ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) {
160 c := Char{codePoint: p.Rune(0)}
161
162 getRune := func(field int) rune {
163 if p.String(field) == "" {
164 return 0
165 }
166 return p.Rune(field)
167 }
168
169 c.category = p.String(ucd.GeneralCategory)
170 category[c.category] = true
171 switch c.category {
172 case "Nd":
173 // Decimal digit
174 p.Int(ucd.NumericValue)
175 case "Lu":
176 c.upperCase = getRune(ucd.CodePoint)
177 c.lowerCase = getRune(ucd.SimpleLowercaseMapping)
178 c.titleCase = getRune(ucd.SimpleTitlecaseMapping)
179 case "Ll":
180 c.upperCase = getRune(ucd.SimpleUppercaseMapping)
181 c.lowerCase = getRune(ucd.CodePoint)
182 c.titleCase = getRune(ucd.SimpleTitlecaseMapping)
183 case "Lt":
184 c.upperCase = getRune(ucd.SimpleUppercaseMapping)
185 c.lowerCase = getRune(ucd.SimpleLowercaseMapping)
186 c.titleCase = getRune(ucd.CodePoint)
187 default:
188 c.upperCase = getRune(ucd.SimpleUppercaseMapping)
189 c.lowerCase = getRune(ucd.SimpleLowercaseMapping)
190 c.titleCase = getRune(ucd.SimpleTitlecaseMapping)
191 }
192
193 chars[c.codePoint] = c
194 })
195 }
196
197 func loadCasefold() {
198 ucd.Parse(gen.OpenUCDFile("CaseFolding.txt"), func(p *ucd.Parser) {
199 kind := p.String(1)
200 if kind != "C" && kind != "S" {
201 // Only care about 'common' and 'simple' foldings.
202 return
203 }
204 p1 := p.Rune(0)
205 p2 := p.Rune(2)
206 chars[p1].foldCase = rune(p2)
207 })
208 }
209
210 var categoryMapping = map[string]string{
211 "LC": "Letter, cased: Ll | Lt | Lu",
212 "Lu": "Letter, uppercase",
213 "Ll": "Letter, lowercase",
214 "Lt": "Letter, titlecase",
215 "Lm": "Letter, modifier",
216 "Lo": "Letter, other",
217 "Mn": "Mark, nonspacing",
218 "Mc": "Mark, spacing combining",
219 "Me": "Mark, enclosing",
220 "Nd": "Number, decimal digit",
221 "Nl": "Number, letter",
222 "No": "Number, other",
223 "Pc": "Punctuation, connector",
224 "Pd": "Punctuation, dash",
225 "Ps": "Punctuation, open",
226 "Pe": "Punctuation, close",
227 "Pi": "Punctuation, initial quote",
228 "Pf": "Punctuation, final quote",
229 "Po": "Punctuation, other",
230 "Sm": "Symbol, math",
231 "Sc": "Symbol, currency",
232 "Sk": "Symbol, modifier",
233 "So": "Symbol, other",
234 "Zs": "Separator, space",
235 "Zl": "Separator, line",
236 "Zp": "Separator, paragraph",
237 "Cc": "Other, control",
238 "Cf": "Other, format",
239 "Cs": "Other, surrogate",
240 "Co": "Other, private use",
241 "Cn": "Other, not assigned",
242 }
243
244 func printCategories() {
245 if *tablelist == "" {
246 return
247 }
248 // Find out which categories to dump
249 list := strings.Split(*tablelist, ",")
250 if *tablelist == "all" {
251 list = allCategories()
252 }
253 if *test {
254 fullCategoryTest(list)
255 return
256 }
257
258 println("// Version is the Unicode edition from which the tables are derived.")
259 printf("const Version = %q\n\n", gen.UnicodeVersion())
260
261 if *tablelist == "all" {
262 println("// Categories is the set of Unicode category tables.")
263 println("var Categories = map[string] *RangeTable {")
264 for _, k := range allCategories() {
265 printf("\t%q: %s,\n", k, k)
266 }
267 print("}\n\n")
268 printCategoryAliases()
269 }
270
271 decl := make(sort.StringSlice, len(list))
272 ndecl := 0
273 for _, name := range list {
274 if _, ok := category[name]; !ok {
275 logger.Fatal("unknown category", name)
276 }
277 // We generate an UpperCase name to serve as concise documentation and an _UnderScored
278 // name to store the data. This stops godoc dumping all the tables but keeps them
279 // available to clients.
280 // Cases deserving special comments
281 varDecl := ""
282 switch name {
283 case "C":
284 varDecl = "\tOther = _C; // Other/C is the set of Unicode control, special, and unassigned code points, category C.\n"
285 varDecl += "\tC = _C\n"
286 case "L":
287 varDecl = "\tLetter = _L; // Letter/L is the set of Unicode letters, category L.\n"
288 varDecl += "\tL = _L\n"
289 case "M":
290 varDecl = "\tMark = _M; // Mark/M is the set of Unicode mark characters, category M.\n"
291 varDecl += "\tM = _M\n"
292 case "N":
293 varDecl = "\tNumber = _N; // Number/N is the set of Unicode number characters, category N.\n"
294 varDecl += "\tN = _N\n"
295 case "P":
296 varDecl = "\tPunct = _P; // Punct/P is the set of Unicode punctuation characters, category P.\n"
297 varDecl += "\tP = _P\n"
298 case "S":
299 varDecl = "\tSymbol = _S; // Symbol/S is the set of Unicode symbol characters, category S.\n"
300 varDecl += "\tS = _S\n"
301 case "Z":
302 varDecl = "\tSpace = _Z; // Space/Z is the set of Unicode space characters, category Z.\n"
303 varDecl += "\tZ = _Z\n"
304 case "Nd":
305 varDecl = "\tDigit = _Nd; // Digit is the set of Unicode characters with the \"decimal digit\" property.\n"
306 case "Lu":
307 varDecl = "\tUpper = _Lu; // Upper is the set of Unicode upper case letters.\n"
308 case "Ll":
309 varDecl = "\tLower = _Ll; // Lower is the set of Unicode lower case letters.\n"
310 case "Lt":
311 varDecl = "\tTitle = _Lt; // Title is the set of Unicode title case letters.\n"
312 }
313 if len(name) > 1 {
314 desc, ok := categoryMapping[name]
315 if ok {
316 varDecl += fmt.Sprintf(
317 "\t%s = _%s; // %s is the set of Unicode characters in category %s (%s).\n",
318 name, name, name, name, desc)
319 } else {
320 varDecl += fmt.Sprintf(
321 "\t%s = _%s; // %s is the set of Unicode characters in category %s.\n",
322 name, name, name, name)
323 }
324 }
325 decl[ndecl] = varDecl
326 ndecl++
327 match := func(cat string) bool { return cat == name }
328 if len(name) == 1 { // unified categories
329 match = func(cat string) bool { return strings.HasPrefix(cat, name) }
330 }
331 if name == "LC" { // special unified category
332 match = func(cat string) bool { return cat == "Ll" || cat == "Lt" || cat == "Lu" }
333 }
334 dumpRange("_"+name, func(code rune) bool { return match(chars[code].category) })
335 }
336 decl.Sort()
337 println("// These variables have type *RangeTable.")
338 println("var (")
339 for _, d := range decl {
340 print(d)
341 }
342 print(")\n\n")
343 }
344
345 func printCategoryAliases() {
346 known := make(map[string]bool)
347 for _, name := range allCategories() {
348 known[name] = true
349 }
350
351 table := make(map[string]string)
352 ucd.Parse(gen.OpenUCDFile("PropertyValueAliases.txt"), func(p *ucd.Parser) {
353 if p.String(0) != "gc" {
354 return
355 }
356 name := p.String(1)
357 if !known[name] {
358 logger.Print("unknown category: ", name)
359 }
360 table[p.String(2)] = name
361 if a := p.String(3); a != "" {
362 table[a] = name
363 }
364 })
365
366 println("// CategoryAliases maps category aliases to standard category names.")
367 println("var CategoryAliases = map[string]string{")
368 for _, name := range slices.Sorted(maps.Keys(table)) {
369 printf("\t%q: %q,\n", name, table[name])
370 }
371 print("}\n\n")
372 }
373
374 type Op func(code rune) bool
375
376 func dumpRange(name string, inCategory Op) {
377 runes := []rune{}
378 for i := range chars {
379 r := rune(i)
380 if inCategory(r) {
381 runes = append(runes, r)
382 }
383 }
384 printRangeTable(name, runes)
385 }
386
387 func printRangeTable(name string, runes []rune) {
388 rt := rangetable.New(runes...)
389 printf("var %s = &RangeTable{\n", name)
390 println("\tR16: []Range16{")
391 for _, r := range rt.R16 {
392 printf("\t\t{%#04x, %#04x, %d},\n", r.Lo, r.Hi, r.Stride)
393 range16Count++
394 }
395 println("\t},")
396 if len(rt.R32) > 0 {
397 println("\tR32: []Range32{")
398 for _, r := range rt.R32 {
399 printf("\t\t{%#x, %#x, %d},\n", r.Lo, r.Hi, r.Stride)
400 range32Count++
401 }
402 println("\t},")
403 }
404 if rt.LatinOffset > 0 {
405 printf("\tLatinOffset: %d,\n", rt.LatinOffset)
406 }
407 printf("}\n\n")
408 }
409
410 func fullCategoryTest(list []string) {
411 for _, name := range list {
412 if _, ok := category[name]; !ok {
413 logger.Fatal("unknown category", name)
414 }
415 r, ok := unicode.Categories[name]
416 if !ok && len(name) > 1 {
417 logger.Fatalf("unknown table %q", name)
418 }
419 if len(name) == 1 {
420 verifyRange(name, func(code rune) bool { return categoryOp(code, name[0]) }, r)
421 } else {
422 verifyRange(
423 name,
424 func(code rune) bool { return chars[code].category == name },
425 r)
426 }
427 }
428 }
429
430 func verifyRange(name string, inCategory Op, table *unicode.RangeTable) {
431 count := 0
432 for j := range chars {
433 i := rune(j)
434 web := inCategory(i)
435 pkg := unicode.Is(table, i)
436 if web != pkg {
437 fmt.Fprintf(os.Stderr, "%s: %U: web=%t pkg=%t\n", name, i, web, pkg)
438 count++
439 if count > 10 {
440 break
441 }
442 }
443 }
444 }
445
446 func fullScriptTest(list []string, installed map[string]*unicode.RangeTable, scripts map[string][]rune) {
447 for _, name := range list {
448 if _, ok := scripts[name]; !ok {
449 logger.Fatal("unknown script", name)
450 }
451 _, ok := installed[name]
452 if !ok {
453 logger.Fatal("unknown table", name)
454 }
455 for _, r := range scripts[name] {
456 if !unicode.Is(installed[name], rune(r)) {
457 fmt.Fprintf(os.Stderr, "%U: not in script %s\n", r, name)
458 }
459 }
460 }
461 }
462
463 var deprecatedAliases = map[string]string{
464 "Sentence_Terminal": "STerm",
465 }
466
467 // PropList.txt has the same format as Scripts.txt so we can share its parser.
468 func printScriptOrProperty(doProps bool) {
469 flaglist := *scriptlist
470 file := "Scripts.txt"
471 table := scripts
472 installed := unicode.Scripts
473 if doProps {
474 flaglist = *proplist
475 file = "PropList.txt"
476 table = props
477 installed = unicode.Properties
478 }
479 if flaglist == "" {
480 return
481 }
482 ucd.Parse(gen.OpenUCDFile(file), func(p *ucd.Parser) {
483 name := p.String(1)
484 table[name] = append(table[name], p.Rune(0))
485 })
486 // Find out which scripts to dump
487 list := strings.Split(flaglist, ",")
488 if flaglist == "all" {
489 list = all(table)
490 }
491 if *test {
492 fullScriptTest(list, installed, table)
493 return
494 }
495
496 if flaglist == "all" {
497 if doProps {
498 println("// Properties is the set of Unicode property tables.")
499 println("var Properties = map[string] *RangeTable{")
500 } else {
501 println("// Scripts is the set of Unicode script tables.")
502 println("var Scripts = map[string] *RangeTable{")
503 }
504 for _, k := range all(table) {
505 printf("\t%q: %s,\n", k, k)
506 if alias, ok := deprecatedAliases[k]; ok {
507 printf("\t%q: %s,\n", alias, k)
508 }
509 }
510 print("}\n\n")
511 }
512
513 decl := make(sort.StringSlice, len(list)+len(deprecatedAliases))
514 ndecl := 0
515 for _, name := range list {
516 if doProps {
517 decl[ndecl] = fmt.Sprintf(
518 "\t%s = _%s;\t// %s is the set of Unicode characters with property %s.\n",
519 name, name, name, name)
520 } else {
521 decl[ndecl] = fmt.Sprintf(
522 "\t%s = _%s;\t// %s is the set of Unicode characters in script %s.\n",
523 name, name, name, name)
524 }
525 ndecl++
526 if alias, ok := deprecatedAliases[name]; ok {
527 decl[ndecl] = fmt.Sprintf(
528 "\t%[1]s = _%[2]s;\t// %[1]s is an alias for %[2]s.\n",
529 alias, name)
530 ndecl++
531 }
532 printRangeTable("_"+name, table[name])
533 }
534 decl.Sort()
535 println("// These variables have type *RangeTable.")
536 println("var (")
537 for _, d := range decl {
538 print(d)
539 }
540 print(")\n\n")
541 }
542
543 const (
544 CaseUpper = 1 << iota
545 CaseLower
546 CaseTitle
547 CaseNone = 0 // must be zero
548 CaseMissing = -1 // character not present; not a valid case state
549 )
550
551 type caseState struct {
552 point rune
553 _case int
554 deltaToUpper rune
555 deltaToLower rune
556 deltaToTitle rune
557 }
558
559 // Is d a continuation of the state of c?
560 func (c *caseState) adjacent(d *caseState) bool {
561 if d.point < c.point {
562 c, d = d, c
563 }
564 switch {
565 case d.point != c.point+1: // code points not adjacent (shouldn't happen)
566 return false
567 case d._case != c._case: // different cases
568 return c.upperLowerAdjacent(d)
569 case c._case == CaseNone:
570 return false
571 case c._case == CaseMissing:
572 return false
573 case d.deltaToUpper != c.deltaToUpper:
574 return false
575 case d.deltaToLower != c.deltaToLower:
576 return false
577 case d.deltaToTitle != c.deltaToTitle:
578 return false
579 }
580 return true
581 }
582
583 // Is d the same as c, but opposite in upper/lower case? this would make it
584 // an element of an UpperLower sequence.
585 func (c *caseState) upperLowerAdjacent(d *caseState) bool {
586 // check they're a matched case pair. we know they have adjacent values
587 switch {
588 case c._case == CaseUpper && d._case != CaseLower:
589 return false
590 case c._case == CaseLower && d._case != CaseUpper:
591 return false
592 }
593 // matched pair (at least in upper/lower). make the order Upper Lower
594 if c._case == CaseLower {
595 c, d = d, c
596 }
597 // for an Upper Lower sequence the deltas have to be in order
598 // c: 0 1 0
599 // d: -1 0 -1
600 switch {
601 case c.deltaToUpper != 0:
602 return false
603 case c.deltaToLower != 1:
604 return false
605 case c.deltaToTitle != 0:
606 return false
607 case d.deltaToUpper != -1:
608 return false
609 case d.deltaToLower != 0:
610 return false
611 case d.deltaToTitle != -1:
612 return false
613 }
614 return true
615 }
616
617 // Does this character start an UpperLower sequence?
618 func (c *caseState) isUpperLower() bool {
619 // for an Upper Lower sequence the deltas have to be in order
620 // c: 0 1 0
621 switch {
622 case c.deltaToUpper != 0:
623 return false
624 case c.deltaToLower != 1:
625 return false
626 case c.deltaToTitle != 0:
627 return false
628 }
629 return true
630 }
631
632 // Does this character start a LowerUpper sequence?
633 func (c *caseState) isLowerUpper() bool {
634 // for an Upper Lower sequence the deltas have to be in order
635 // c: -1 0 -1
636 switch {
637 case c.deltaToUpper != -1:
638 return false
639 case c.deltaToLower != 0:
640 return false
641 case c.deltaToTitle != -1:
642 return false
643 }
644 return true
645 }
646
647 func getCaseState(i rune) (c *caseState) {
648 c = &caseState{point: i, _case: CaseNone}
649 ch := &chars[i]
650 switch ch.codePoint {
651 case 0:
652 c._case = CaseMissing // Will get NUL wrong but that doesn't matter
653 return
654 case ch.upperCase:
655 c._case = CaseUpper
656 case ch.lowerCase:
657 c._case = CaseLower
658 case ch.titleCase:
659 c._case = CaseTitle
660 }
661 // Some things such as roman numeral U+2161 don't describe themselves
662 // as upper case, but have a lower case. Second-guess them.
663 if c._case == CaseNone && ch.lowerCase != 0 {
664 c._case = CaseUpper
665 }
666 // Same in the other direction.
667 if c._case == CaseNone && ch.upperCase != 0 {
668 c._case = CaseLower
669 }
670
671 if ch.upperCase != 0 {
672 c.deltaToUpper = ch.upperCase - i
673 }
674 if ch.lowerCase != 0 {
675 c.deltaToLower = ch.lowerCase - i
676 }
677 if ch.titleCase != 0 {
678 c.deltaToTitle = ch.titleCase - i
679 }
680 return
681 }
682
683 func printCases() {
684 if *test {
685 fullCaseTest()
686 return
687 }
688 printf(
689 "// CaseRanges is the table describing case mappings for all letters with\n" +
690 "// non-self mappings.\n" +
691 "var CaseRanges = _CaseRanges\n" +
692 "var _CaseRanges = []CaseRange {\n")
693
694 var startState *caseState // the start of a run; nil for not active
695 var prevState = &caseState{} // the state of the previous character
696 for i := range chars {
697 state := getCaseState(rune(i))
698 if state.adjacent(prevState) {
699 prevState = state
700 continue
701 }
702 // end of run (possibly)
703 printCaseRange(startState, prevState)
704 startState = nil
705 if state._case != CaseMissing && state._case != CaseNone {
706 startState = state
707 }
708 prevState = state
709 }
710 print("}\n")
711 }
712
713 func printCaseRange(lo, hi *caseState) {
714 if lo == nil {
715 return
716 }
717 if lo.deltaToUpper == 0 && lo.deltaToLower == 0 && lo.deltaToTitle == 0 {
718 // character represents itself in all cases - no need to mention it
719 return
720 }
721 switch {
722 case hi.point > lo.point && lo.isUpperLower():
723 printf("\t{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n",
724 lo.point, hi.point)
725 case hi.point > lo.point && lo.isLowerUpper():
726 logger.Fatalf("LowerUpper sequence: should not happen: %U. If it's real, need to fix To()", lo.point)
727 printf("\t{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n",
728 lo.point, hi.point)
729 default:
730 printf("\t{0x%04X, 0x%04X, d{%d, %d, %d}},\n",
731 lo.point, hi.point,
732 lo.deltaToUpper, lo.deltaToLower, lo.deltaToTitle)
733 }
734 }
735
736 // If the cased value in the Char is 0, it means use the rune itself.
737 func caseIt(r, cased rune) rune {
738 if cased == 0 {
739 return r
740 }
741 return cased
742 }
743
744 func fullCaseTest() {
745 for j, c := range chars {
746 i := rune(j)
747 lower := unicode.ToLower(i)
748 want := caseIt(i, c.lowerCase)
749 if lower != want {
750 fmt.Fprintf(os.Stderr, "lower %U should be %U is %U\n", i, want, lower)
751 }
752 upper := unicode.ToUpper(i)
753 want = caseIt(i, c.upperCase)
754 if upper != want {
755 fmt.Fprintf(os.Stderr, "upper %U should be %U is %U\n", i, want, upper)
756 }
757 title := unicode.ToTitle(i)
758 want = caseIt(i, c.titleCase)
759 if title != want {
760 fmt.Fprintf(os.Stderr, "title %U should be %U is %U\n", i, want, title)
761 }
762 }
763 }
764
765 func printLatinProperties() {
766 if *test {
767 return
768 }
769 println("var properties = [MaxLatin1+1]uint8{")
770 for code := 0; code <= unicode.MaxLatin1; code++ {
771 var property string
772 switch chars[code].category {
773 case "Cc", "": // NUL has no category.
774 property = "pC"
775 case "Cf": // soft hyphen, unique category, not printable.
776 property = "0"
777 case "Ll":
778 property = "pLl | pp"
779 case "Lo":
780 property = "pLo | pp"
781 case "Lu":
782 property = "pLu | pp"
783 case "Nd", "No":
784 property = "pN | pp"
785 case "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps":
786 property = "pP | pp"
787 case "Sc", "Sk", "Sm", "So":
788 property = "pS | pp"
789 case "Zs":
790 property = "pZ"
791 default:
792 logger.Fatalf("%U has unknown category %q", code, chars[code].category)
793 }
794 // Special case
795 if code == ' ' {
796 property = "pZ | pp"
797 }
798 printf("\t0x%02X: %s, // %q\n", code, property, code)
799 }
800 printf("}\n\n")
801 }
802
803 func printCasefold() {
804 // Build list of case-folding groups attached to each canonical folded char (typically lower case).
805 var caseOrbit = make([][]rune, MaxChar+1)
806 for j := range chars {
807 i := rune(j)
808 c := &chars[i]
809 if c.foldCase == 0 {
810 continue
811 }
812 orb := caseOrbit[c.foldCase]
813 if orb == nil {
814 orb = append(orb, c.foldCase)
815 }
816 caseOrbit[c.foldCase] = append(orb, i)
817 }
818
819 // Insert explicit 1-element groups when assuming [lower, upper] would be wrong.
820 for j := range chars {
821 i := rune(j)
822 c := &chars[i]
823 f := c.foldCase
824 if f == 0 {
825 f = i
826 }
827 orb := caseOrbit[f]
828 if orb == nil && (c.upperCase != 0 && c.upperCase != i || c.lowerCase != 0 && c.lowerCase != i) {
829 // Default assumption of [upper, lower] is wrong.
830 caseOrbit[i] = []rune{i}
831 }
832 }
833
834 // Delete the groups for which assuming [lower, upper] or [upper, lower] is right.
835 for i, orb := range caseOrbit {
836 if len(orb) == 2 && chars[orb[0]].upperCase == orb[1] && chars[orb[1]].lowerCase == orb[0] {
837 caseOrbit[i] = nil
838 }
839 if len(orb) == 2 && chars[orb[1]].upperCase == orb[0] && chars[orb[0]].lowerCase == orb[1] {
840 caseOrbit[i] = nil
841 }
842 }
843
844 // Record orbit information in chars.
845 for _, orb := range caseOrbit {
846 if orb == nil {
847 continue
848 }
849 sort.Slice(orb, func(i, j int) bool {
850 return orb[i] < orb[j]
851 })
852 c := orb[len(orb)-1]
853 for _, d := range orb {
854 chars[c].caseOrbit = d
855 c = d
856 }
857 }
858
859 printAsciiFold()
860 printCaseOrbit()
861
862 // Tables of category and script folding exceptions: code points
863 // that must be added when interpreting a particular category/script
864 // in a case-folding context.
865 cat := make(map[string]map[rune]bool)
866 for name := range category {
867 if x := foldExceptions(inCategory(name)); len(x) > 0 {
868 cat[name] = x
869 }
870 }
871
872 scr := make(map[string]map[rune]bool)
873 for name := range scripts {
874 if x := foldExceptions(scripts[name]); len(x) > 0 {
875 scr[name] = x
876 }
877 }
878
879 printCatFold("FoldCategory", cat)
880 printCatFold("FoldScript", scr)
881 }
882
883 // inCategory returns a list of all the runes in the category.
884 func inCategory(name string) []rune {
885 var x []rune
886 for j := range chars {
887 i := rune(j)
888 c := &chars[i]
889 if c.category == name || len(name) == 1 && len(c.category) > 1 && c.category[0] == name[0] {
890 x = append(x, i)
891 }
892 }
893 return x
894 }
895
896 // foldExceptions returns a list of all the runes fold-equivalent
897 // to runes in class but not in class themselves.
898 func foldExceptions(class []rune) map[rune]bool {
899 // Create map containing class and all fold-equivalent chars.
900 m := make(map[rune]bool)
901 for _, r := range class {
902 c := &chars[r]
903 if c.caseOrbit == 0 {
904 // Just upper and lower.
905 if u := c.upperCase; u != 0 {
906 m[u] = true
907 }
908 if l := c.lowerCase; l != 0 {
909 m[l] = true
910 }
911 m[r] = true
912 continue
913 }
914 // Otherwise walk orbit.
915 r0 := r
916 for {
917 m[r] = true
918 r = chars[r].caseOrbit
919 if r == r0 {
920 break
921 }
922 }
923 }
924
925 // Remove class itself.
926 for _, r := range class {
927 delete(m, r)
928 }
929
930 // What's left is the exceptions.
931 return m
932 }
933
934 var comment = map[string]string{
935 "FoldCategory": "// FoldCategory maps a category name to a table of\n" +
936 "// code points outside the category that are equivalent under\n" +
937 "// simple case folding to code points inside the category.\n" +
938 "// If there is no entry for a category name, there are no such points.\n",
939
940 "FoldScript": "// FoldScript maps a script name to a table of\n" +
941 "// code points outside the script that are equivalent under\n" +
942 "// simple case folding to code points inside the script.\n" +
943 "// If there is no entry for a script name, there are no such points.\n",
944 }
945
946 func printAsciiFold() {
947 printf("var asciiFold = [MaxASCII + 1]uint16{\n")
948 for i := rune(0); i <= unicode.MaxASCII; i++ {
949 c := chars[i]
950 f := c.caseOrbit
951 if f == 0 {
952 if c.lowerCase != i && c.lowerCase != 0 {
953 f = c.lowerCase
954 } else if c.upperCase != i && c.upperCase != 0 {
955 f = c.upperCase
956 } else {
957 f = i
958 }
959 }
960 printf("\t0x%04X,\n", f)
961 }
962 printf("}\n\n")
963 }
964
965 func printCaseOrbit() {
966 if *test {
967 for j := range chars {
968 i := rune(j)
969 c := &chars[i]
970 f := c.caseOrbit
971 if f == 0 {
972 if c.lowerCase != i && c.lowerCase != 0 {
973 f = c.lowerCase
974 } else if c.upperCase != i && c.upperCase != 0 {
975 f = c.upperCase
976 } else {
977 f = i
978 }
979 }
980 if g := unicode.SimpleFold(i); g != f {
981 fmt.Fprintf(os.Stderr, "unicode.SimpleFold(%#U) = %#U, want %#U\n", i, g, f)
982 }
983 }
984 return
985 }
986
987 printf("var caseOrbit = []foldPair{\n")
988 for i := range chars {
989 c := &chars[i]
990 if c.caseOrbit != 0 {
991 printf("\t{0x%04X, 0x%04X},\n", i, c.caseOrbit)
992 foldPairCount++
993 }
994 }
995 printf("}\n\n")
996 }
997
998 func printCatFold(name string, m map[string]map[rune]bool) {
999 if *test {
1000 var pkgMap map[string]*unicode.RangeTable
1001 if name == "FoldCategory" {
1002 pkgMap = unicode.FoldCategory
1003 } else {
1004 pkgMap = unicode.FoldScript
1005 }
1006 if len(pkgMap) != len(m) {
1007 fmt.Fprintf(os.Stderr, "unicode.%s has %d elements, want %d\n", name, len(pkgMap), len(m))
1008 return
1009 }
1010 for k, v := range m {
1011 t, ok := pkgMap[k]
1012 if !ok {
1013 fmt.Fprintf(os.Stderr, "unicode.%s[%q] missing\n", name, k)
1014 continue
1015 }
1016 n := 0
1017 for _, r := range t.R16 {
1018 for c := rune(r.Lo); c <= rune(r.Hi); c += rune(r.Stride) {
1019 if !v[c] {
1020 fmt.Fprintf(os.Stderr, "unicode.%s[%q] contains %#U, should not\n", name, k, c)
1021 }
1022 n++
1023 }
1024 }
1025 for _, r := range t.R32 {
1026 for c := rune(r.Lo); c <= rune(r.Hi); c += rune(r.Stride) {
1027 if !v[c] {
1028 fmt.Fprintf(os.Stderr, "unicode.%s[%q] contains %#U, should not\n", name, k, c)
1029 }
1030 n++
1031 }
1032 }
1033 if n != len(v) {
1034 fmt.Fprintf(os.Stderr, "unicode.%s[%q] has %d code points, want %d\n", name, k, n, len(v))
1035 }
1036 }
1037 return
1038 }
1039
1040 print(comment[name])
1041 printf("var %s = map[string]*RangeTable{\n", name)
1042 for _, name := range allCatFold(m) {
1043 printf("\t%q: fold%s,\n", name, name)
1044 }
1045 printf("}\n\n")
1046 for _, name := range allCatFold(m) {
1047 class := m[name]
1048 dumpRange("fold"+name, func(code rune) bool { return class[code] })
1049 }
1050 }
1051
1052 var range16Count = 0 // Number of entries in the 16-bit range tables.
1053 var range32Count = 0 // Number of entries in the 32-bit range tables.
1054 var foldPairCount = 0 // Number of fold pairs in the exception tables.
1055
1056 func printSizes() {
1057 if *test {
1058 return
1059 }
1060 println()
1061 printf("// Range entries: %d 16-bit, %d 32-bit, %d total.\n", range16Count, range32Count, range16Count+range32Count)
1062 range16Bytes := range16Count * 3 * 2
1063 range32Bytes := range32Count * 3 * 4
1064 printf("// Range bytes: %d 16-bit, %d 32-bit, %d total.\n", range16Bytes, range32Bytes, range16Bytes+range32Bytes)
1065 println()
1066 printf("// Fold orbit bytes: %d pairs, %d bytes\n", foldPairCount, foldPairCount*2*2)
1067 }