Debian Codegraph @

unicode/norm/maketables.go

raw
1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ignore
6
7 // Normalization table generator.
8 // Data read from the web.
9 // See forminfo.go for a description of the trie values associated with each rune.
10
11 package main
12
13 import (
14 "bytes"
15 "encoding/binary"
16 "flag"
17 "fmt"
18 "io"
19 "log"
20 "sort"
21 "strconv"
22 "strings"
23
24 "golang.org/x/text/internal/gen"
25 "golang.org/x/text/internal/triegen"
26 "golang.org/x/text/internal/ucd"
27 )
28
29 func main() {
30 gen.Init()
31 loadUnicodeData()
32 compactCCC()
33 loadCompositionExclusions()
34 completeCharFields(FCanonical)
35 completeCharFields(FCompatibility)
36 computeNonStarterCounts()
37 verifyComputed()
38 printChars()
39 testDerived()
40 printTestdata()
41 makeTables()
42 }
43
44 var (
45 tablelist = flag.String("tables",
46 "all",
47 "comma-separated list of which tables to generate; "+
48 "can be 'decomp', 'recomp', 'info' and 'all'")
49 test = flag.Bool("test",
50 false,
51 "test existing tables against DerivedNormalizationProps and generate test data for regression testing")
52 verbose = flag.Bool("verbose",
53 false,
54 "write data to stdout as it is parsed")
55 )
56
57 const MaxChar = 0x10FFFF // anything above this shouldn't exist
58
59 // Quick Check properties of runes allow us to quickly
60 // determine whether a rune may occur in a normal form.
61 // For a given normal form, a rune may be guaranteed to occur
62 // verbatim (QC=Yes), may or may not combine with another
63 // rune (QC=Maybe), or may not occur (QC=No).
64 type QCResult int
65
66 const (
67 QCUnknown QCResult = iota
68 QCYes
69 QCNo
70 QCMaybe
71 )
72
73 func (r QCResult) String() string {
74 switch r {
75 case QCYes:
76 return "Yes"
77 case QCNo:
78 return "No"
79 case QCMaybe:
80 return "Maybe"
81 }
82 return "***UNKNOWN***"
83 }
84
85 const (
86 FCanonical = iota // NFC or NFD
87 FCompatibility // NFKC or NFKD
88 FNumberOfFormTypes
89 )
90
91 const (
92 MComposed = iota // NFC or NFKC
93 MDecomposed // NFD or NFKD
94 MNumberOfModes
95 )
96
97 // This contains only the properties we're interested in.
98 type Char struct {
99 name string
100 codePoint rune // if zero, this index is not a valid code point.
101 ccc uint8 // canonical combining class
102 origCCC uint8
103 excludeInComp bool // from CompositionExclusions.txt
104 compatDecomp bool // it has a compatibility expansion
105
106 nTrailingNonStarters uint8
107 nLeadingNonStarters uint8 // must be equal to trailing if non-zero
108
109 forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility
110
111 state State
112 }
113
114 var chars = make([]Char, MaxChar+1)
115 var cccMap = make(map[uint8]uint8)
116
117 func (c Char) String() string {
118 buf := new(bytes.Buffer)
119
120 fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name)
121 fmt.Fprintf(buf, " ccc: %v\n", c.ccc)
122 fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp)
123 fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp)
124 fmt.Fprintf(buf, " state: %v\n", c.state)
125 fmt.Fprintf(buf, " NFC:\n")
126 fmt.Fprint(buf, c.forms[FCanonical])
127 fmt.Fprintf(buf, " NFKC:\n")
128 fmt.Fprint(buf, c.forms[FCompatibility])
129
130 return buf.String()
131 }
132
133 // In UnicodeData.txt, some ranges are marked like this:
134 //
135 // 3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
136 // 4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
137 //
138 // parseCharacter keeps a state variable indicating the weirdness.
139 type State int
140
141 const (
142 SNormal State = iota // known to be zero for the type
143 SFirst
144 SLast
145 SMissing
146 )
147
148 var lastChar = rune('\u0000')
149
150 func (c Char) isValid() bool {
151 return c.codePoint != 0 && c.state != SMissing
152 }
153
154 type FormInfo struct {
155 quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed
156 verified [MNumberOfModes]bool // index: MComposed or MDecomposed
157
158 combinesForward bool // May combine with rune on the right
159 combinesBackward bool // May combine with rune on the left
160 isOneWay bool // Never appears in result
161 inDecomp bool // Some decompositions result in this char.
162 suffixDecomp bool // Appears after first rune of a decomposition
163 decomp Decomposition
164 expandedDecomp Decomposition
165 }
166
167 func (f FormInfo) String() string {
168 buf := bytes.NewBuffer(make([]byte, 0))
169
170 fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed])
171 fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed])
172 fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward)
173 fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward)
174 fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay)
175 fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp)
176 fmt.Fprintf(buf, " decomposition: %X\n", f.decomp)
177 fmt.Fprintf(buf, " expandedDecomp: %X\n", f.expandedDecomp)
178
179 return buf.String()
180 }
181
182 type Decomposition []rune
183
184 func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
185 decomp := strings.Split(s, " ")
186 if len(decomp) > 0 && skipfirst {
187 decomp = decomp[1:]
188 }
189 for _, d := range decomp {
190 point, err := strconv.ParseUint(d, 16, 64)
191 if err != nil {
192 return a, err
193 }
194 a = append(a, rune(point))
195 }
196 return a, nil
197 }
198
199 func loadUnicodeData() {
200 f := gen.OpenUCDFile("UnicodeData.txt")
201 defer f.Close()
202 p := ucd.New(f)
203 for p.Next() {
204 r := p.Rune(ucd.CodePoint)
205 char := &chars[r]
206
207 char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass))
208 decmap := p.String(ucd.DecompMapping)
209
210 exp, err := parseDecomposition(decmap, false)
211 isCompat := false
212 if err != nil {
213 if len(decmap) > 0 {
214 exp, err = parseDecomposition(decmap, true)
215 if err != nil {
216 log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
217 }
218 isCompat = true
219 }
220 }
221
222 char.name = p.String(ucd.Name)
223 char.codePoint = r
224 char.forms[FCompatibility].decomp = exp
225 if !isCompat {
226 char.forms[FCanonical].decomp = exp
227 } else {
228 char.compatDecomp = true
229 }
230 if len(decmap) > 0 {
231 char.forms[FCompatibility].decomp = exp
232 }
233 }
234 if err := p.Err(); err != nil {
235 log.Fatal(err)
236 }
237 }
238
239 // compactCCC converts the sparse set of CCC values to a contiguous one,
240 // reducing the number of bits needed from 8 to 6.
241 func compactCCC() {
242 m := make(map[uint8]uint8)
243 for i := range chars {
244 c := &chars[i]
245 m[c.ccc] = 0
246 }
247 cccs := []int{}
248 for v, _ := range m {
249 cccs = append(cccs, int(v))
250 }
251 sort.Ints(cccs)
252 for i, c := range cccs {
253 cccMap[uint8(i)] = uint8(c)
254 m[uint8(c)] = uint8(i)
255 }
256 for i := range chars {
257 c := &chars[i]
258 c.origCCC = c.ccc
259 c.ccc = m[c.ccc]
260 }
261 if len(m) >= 1<<6 {
262 log.Fatalf("too many difference CCC values: %d >= 64", len(m))
263 }
264 }
265
266 // CompositionExclusions.txt has form:
267 // 0958 # ...
268 // See https://unicode.org/reports/tr44/ for full explanation
269 func loadCompositionExclusions() {
270 f := gen.OpenUCDFile("CompositionExclusions.txt")
271 defer f.Close()
272 p := ucd.New(f)
273 for p.Next() {
274 c := &chars[p.Rune(0)]
275 if c.excludeInComp {
276 log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
277 }
278 c.excludeInComp = true
279 }
280 if e := p.Err(); e != nil {
281 log.Fatal(e)
282 }
283 }
284
285 // hasCompatDecomp returns true if any of the recursive
286 // decompositions contains a compatibility expansion.
287 // In this case, the character may not occur in NFK*.
288 func hasCompatDecomp(r rune) bool {
289 c := &chars[r]
290 if c.compatDecomp {
291 return true
292 }
293 for _, d := range c.forms[FCompatibility].decomp {
294 if hasCompatDecomp(d) {
295 return true
296 }
297 }
298 return false
299 }
300
301 // Hangul related constants.
302 const (
303 HangulBase = 0xAC00
304 HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28)
305
306 JamoLBase = 0x1100
307 JamoLEnd = 0x1113
308 JamoVBase = 0x1161
309 JamoVEnd = 0x1176
310 JamoTBase = 0x11A8
311 JamoTEnd = 0x11C3
312
313 JamoLVTCount = 19 * 21 * 28
314 JamoTCount = 28
315 )
316
317 func isHangul(r rune) bool {
318 return HangulBase <= r && r < HangulEnd
319 }
320
321 func isHangulWithoutJamoT(r rune) bool {
322 if !isHangul(r) {
323 return false
324 }
325 r -= HangulBase
326 return r < JamoLVTCount && r%JamoTCount == 0
327 }
328
329 func ccc(r rune) uint8 {
330 return chars[r].ccc
331 }
332
333 // Insert a rune in a buffer, ordered by Canonical Combining Class.
334 func insertOrdered(b Decomposition, r rune) Decomposition {
335 n := len(b)
336 b = append(b, 0)
337 cc := ccc(r)
338 if cc > 0 {
339 // Use bubble sort.
340 for ; n > 0; n-- {
341 if ccc(b[n-1]) <= cc {
342 break
343 }
344 b[n] = b[n-1]
345 }
346 }
347 b[n] = r
348 return b
349 }
350
351 // Recursively decompose.
352 func decomposeRecursive(form int, r rune, d Decomposition) Decomposition {
353 dcomp := chars[r].forms[form].decomp
354 if len(dcomp) == 0 {
355 return insertOrdered(d, r)
356 }
357 for _, c := range dcomp {
358 d = decomposeRecursive(form, c, d)
359 }
360 return d
361 }
362
363 func completeCharFields(form int) {
364 // Phase 0: pre-expand decomposition.
365 for i := range chars {
366 f := &chars[i].forms[form]
367 if len(f.decomp) == 0 {
368 continue
369 }
370 exp := make(Decomposition, 0)
371 for _, c := range f.decomp {
372 exp = decomposeRecursive(form, c, exp)
373 }
374 f.expandedDecomp = exp
375 }
376
377 // Phase 1: composition exclusion, mark decomposition.
378 for i := range chars {
379 c := &chars[i]
380 f := &c.forms[form]
381
382 // Marks script-specific exclusions and version restricted.
383 f.isOneWay = c.excludeInComp
384
385 // Singletons
386 f.isOneWay = f.isOneWay || len(f.decomp) == 1
387
388 // Non-starter decompositions
389 if len(f.decomp) > 1 {
390 chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0
391 f.isOneWay = f.isOneWay || chk
392 }
393
394 // Runes that decompose into more than two runes.
395 f.isOneWay = f.isOneWay || len(f.decomp) > 2
396
397 if form == FCompatibility {
398 f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint)
399 }
400
401 for i, r := range f.decomp {
402 chars[r].forms[form].inDecomp = true
403 if i > 0 {
404 chars[r].forms[form].suffixDecomp = true
405 }
406 }
407 }
408
409 // Phase 2: forward and backward combining.
410 for i := range chars {
411 c := &chars[i]
412 f := &c.forms[form]
413
414 if !f.isOneWay && len(f.decomp) == 2 {
415 f0 := &chars[f.decomp[0]].forms[form]
416 f1 := &chars[f.decomp[1]].forms[form]
417 if !f0.isOneWay {
418 f0.combinesForward = true
419 }
420 if !f1.isOneWay {
421 f1.combinesBackward = true
422 }
423 }
424 if isHangulWithoutJamoT(rune(i)) {
425 f.combinesForward = true
426 }
427 if (i & 0xffff00) == JamoLBase {
428 if JamoLBase <= i && i < JamoLEnd {
429 f.combinesForward = true
430 }
431 if JamoVBase <= i && i < JamoVEnd {
432 f.combinesBackward = true
433 f.combinesForward = true
434 }
435 if JamoTBase <= i && i < JamoTEnd {
436 f.combinesBackward = true
437 }
438 }
439 }
440
441 // Phase 2½: backward combining propagation.
442 for i := range chars {
443 c := &chars[i]
444 f := &c.forms[form]
445
446 // If the first rune of f's decomposition combines backward,
447 // then f itself must be considered to combine backward.
448 // This handles the "MaybeNo" runes introduced in Unicode 16.
449 // https://www.unicode.org/reports/tr15/tr15-56.html#Contexts_Care
450 if !f.isOneWay && len(f.decomp) > 0 {
451 f0 := &chars[f.decomp[0]].forms[form]
452 if f0.combinesBackward {
453 f.combinesBackward = true
454 }
455 }
456 }
457
458 // Phase 3: quick check values.
459 for i := range chars {
460 c := &chars[i]
461 f := &c.forms[form]
462
463 switch {
464 case len(f.decomp) > 0:
465 f.quickCheck[MDecomposed] = QCNo
466 case isHangul(rune(i)):
467 f.quickCheck[MDecomposed] = QCNo
468 default:
469 f.quickCheck[MDecomposed] = QCYes
470 }
471 switch {
472 case f.isOneWay:
473 f.quickCheck[MComposed] = QCNo
474 case !f.combinesBackward:
475 f.quickCheck[MComposed] = QCYes
476 default:
477 f.quickCheck[MComposed] = QCMaybe
478 }
479 }
480 }
481
482 func computeNonStarterCounts() {
483 // Phase 4: leading and trailing non-starter count
484 for i := range chars {
485 c := &chars[i]
486
487 runes := []rune{rune(i)}
488 // We always use FCompatibility so that the CGJ insertion points do not
489 // change for repeated normalizations with different forms.
490 if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 {
491 runes = exp
492 }
493 // We consider runes that combine backwards to be non-starters for the
494 // purpose of Stream-Safe Text Processing.
495 for _, r := range runes {
496 if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
497 break
498 }
499 c.nLeadingNonStarters++
500 }
501 for i := len(runes) - 1; i >= 0; i-- {
502 if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
503 break
504 }
505 c.nTrailingNonStarters++
506 }
507 if c.nTrailingNonStarters > 3 {
508 log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes)
509 }
510
511 if isHangul(rune(i)) {
512 c.nTrailingNonStarters = 2
513 if isHangulWithoutJamoT(rune(i)) {
514 c.nTrailingNonStarters = 1
515 }
516 }
517
518 if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t {
519 log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t)
520 }
521 if t := c.nTrailingNonStarters; t > 3 {
522 log.Fatalf("%U: number of trailing non-starters is %d > 3", t)
523 }
524 }
525 }
526
527 func printBytes(w io.Writer, b []byte, name string) {
528 fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b))
529 fmt.Fprintf(w, "var %s = [...]byte {", name)
530 for i, c := range b {
531 switch {
532 case i%64 == 0:
533 fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63)
534 case i%8 == 0:
535 fmt.Fprintf(w, "\n")
536 }
537 fmt.Fprintf(w, "0x%.2X, ", c)
538 }
539 fmt.Fprint(w, "\n}\n\n")
540 }
541
542 // See forminfo.go for format.
543 func makeEntry(f *FormInfo, c *Char) uint16 {
544 e := uint16(0)
545 if r := c.codePoint; HangulBase <= r && r < HangulEnd {
546 e |= 0x40
547 }
548 if f.combinesForward {
549 e |= 0x20
550 }
551 if f.quickCheck[MDecomposed] == QCNo {
552 e |= 0x4
553 }
554 switch f.quickCheck[MComposed] {
555 case QCYes:
556 case QCNo:
557 e |= 0x10
558 case QCMaybe:
559 e |= 0x18
560 default:
561 log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed])
562 }
563 e |= uint16(c.nTrailingNonStarters)
564 return e
565 }
566
567 // decompSet keeps track of unique decompositions, grouped by whether
568 // the decomposition is followed by a trailing and/or leading CCC.
569 type decompSet [7]map[string]bool
570
571 const (
572 normalDecomp = iota
573 firstMulti
574 firstCCC
575 endMulti
576 firstLeadingCCC
577 firstCCCZeroExcept
578 firstStarterWithNLead
579 lastDecomp
580 )
581
582 var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"}
583
584 func makeDecompSet() decompSet {
585 m := decompSet{}
586 for i := range m {
587 m[i] = make(map[string]bool)
588 }
589 return m
590 }
591 func (m *decompSet) insert(key int, s string) {
592 m[key][s] = true
593 }
594
595 func printCharInfoTables(w io.Writer) int {
596 mkstr := func(r rune, f *FormInfo, c *Char) (int, string) {
597 d := f.expandedDecomp
598 s := string([]rune(d))
599 slen := len(s)
600 if slen == 31 || slen == 32 || slen > 33 {
601 log.Fatalf("%U: too many bytes in decomposition: %d", slen)
602 }
603 if slen == 33 {
604 slen = 31
605 }
606 head := uint8(slen) | uint8(makeEntry(f, c)>>3<<5)
607 s = string([]byte{head}) + s
608
609 lccc := ccc(d[0])
610 tccc := ccc(d[len(d)-1])
611 cc := ccc(r)
612 if cc != 0 && lccc == 0 && tccc == 0 {
613 log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
614 }
615 if tccc < lccc && lccc != 0 {
616 const msg = "%U: lccc (%d) must be <= tcc (%d)"
617 log.Fatalf(msg, r, lccc, tccc)
618 }
619 index := normalDecomp
620 nTrail := chars[r].nTrailingNonStarters
621 nLead := chars[r].nLeadingNonStarters
622 if tccc > 0 || lccc > 0 || nTrail > 0 {
623 tccc <<= 2
624 tccc |= nTrail
625 s += string([]byte{tccc})
626 index = endMulti
627 for _, r := range d[1:] {
628 if ccc(r) == 0 && !chars[r].forms[FCanonical].combinesBackward {
629 index = firstCCC
630 }
631 }
632 if lccc > 0 || nLead > 0 {
633 s += string([]byte{lccc})
634 if index == firstCCC {
635 log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
636 }
637 index = firstLeadingCCC
638 }
639 if cc != lccc {
640 if cc != 0 {
641 log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
642 }
643 index = firstCCCZeroExcept
644 }
645 } else if len(d) > 1 {
646 index = firstMulti
647 }
648 return index, s
649 }
650
651 decompSet := makeDecompSet()
652 const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail.
653 decompSet.insert(firstStarterWithNLead, nLeadStr)
654
655 // Store the uniqued decompositions in a byte buffer,
656 // preceded by their byte length.
657 for _, c := range chars {
658 for _, f := range c.forms {
659 if len(f.expandedDecomp) == 0 {
660 continue
661 }
662 index, s := mkstr(c.codePoint, &f, &c)
663 decompSet.insert(index, s)
664 }
665 }
666
667 decompositions := bytes.NewBuffer(make([]byte, 0, 10000))
668 size := 0
669 positionMap := make(map[string]uint16)
670 decompositions.WriteString("\000")
671 fmt.Fprintln(w, "const (")
672 for i, m := range decompSet {
673 sa := []string{}
674 for s := range m {
675 sa = append(sa, s)
676 }
677 sort.Strings(sa)
678 for _, s := range sa {
679 p := decompositions.Len()
680 decompositions.WriteString(s)
681 positionMap[s] = uint16(p)
682 }
683 if cname[i] != "" {
684 fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len())
685 }
686 }
687 fmt.Fprintln(w, "maxDecomp = 0x8000")
688 fmt.Fprintln(w, ")")
689 b := decompositions.Bytes()
690 printBytes(w, b, "decomps")
691 size += len(b)
692
693 varnames := []string{"nfc", "nfkc"}
694 for i := 0; i < FNumberOfFormTypes; i++ {
695 trie := triegen.NewTrie(varnames[i])
696
697 for r, c := range chars {
698 f := c.forms[i]
699 d := f.expandedDecomp
700 if len(d) != 0 {
701 _, key := mkstr(c.codePoint, &f, &c)
702 trie.Insert(rune(r), uint64(positionMap[key]))
703 if c.ccc != ccc(d[0]) {
704 // We assume the lead ccc of a decomposition !=0 in this case.
705 if ccc(d[0]) == 0 {
706 log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
707 }
708 }
709 } else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
710 // Handle cases where it can't be detected that the nLead should be equal
711 // to nTrail.
712 trie.Insert(c.codePoint, uint64(positionMap[nLeadStr]))
713 } else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 {
714 trie.Insert(c.codePoint, uint64(0x8000|v))
715 }
716 }
717 sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]}))
718 if err != nil {
719 log.Fatal(err)
720 }
721 size += sz
722 }
723 return size
724 }
725
726 func contains(sa []string, s string) bool {
727 for _, a := range sa {
728 if a == s {
729 return true
730 }
731 }
732 return false
733 }
734
735 func makeTables() {
736 w := &bytes.Buffer{}
737
738 size := 0
739 if *tablelist == "" {
740 return
741 }
742 list := strings.Split(*tablelist, ",")
743 if *tablelist == "all" {
744 list = []string{"recomp", "info"}
745 }
746
747 // Compute maximum decomposition size.
748 max := 0
749 for _, c := range chars {
750 if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max {
751 max = n
752 }
753 }
754 fmt.Fprintln(w, `import "sync"`)
755 fmt.Fprintln(w)
756
757 fmt.Fprintln(w, "const (")
758 fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.")
759 fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion())
760 fmt.Fprintln(w)
761 fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
762 fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at")
763 fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that")
764 fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.")
765 fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
766 fmt.Fprintln(w, ")\n")
767
768 // Print the CCC remap table.
769 size += len(cccMap)
770 fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap))
771 for i := 0; i < len(cccMap); i++ {
772 if i%8 == 0 {
773 fmt.Fprintln(w)
774 }
775 fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)])
776 }
777 fmt.Fprintln(w, "\n}\n")
778
779 if contains(list, "info") {
780 size += printCharInfoTables(w)
781 }
782
783 if contains(list, "recomp") {
784 // Note that we use 32 bit keys, instead of 64 bit.
785 // This clips the bits of three entries, but we know
786 // this won't cause a collision. The compiler will catch
787 // any changes made to UnicodeData.txt that introduces
788 // a collision.
789 // Note that the recomposition map for NFC and NFKC
790 // are identical.
791
792 // Recomposition map
793 nrentries := 0
794 for _, c := range chars {
795 f := c.forms[FCanonical]
796 if !f.isOneWay && len(f.decomp) > 0 {
797 nrentries++
798 }
799 }
800 sz := nrentries * 8
801 size += sz
802 fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz)
803 fmt.Fprintln(w, "var recompMap map[uint32]rune")
804 fmt.Fprintln(w, "var recompMapOnce sync.Once\n")
805 fmt.Fprintln(w, `const recompMapPacked = "" +`)
806 var buf [8]byte
807 for i, c := range chars {
808 f := c.forms[FCanonical]
809 d := f.decomp
810 if !f.isOneWay && len(d) > 0 {
811 key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1]))
812 binary.BigEndian.PutUint32(buf[:4], key)
813 binary.BigEndian.PutUint32(buf[4:], uint32(i))
814 fmt.Fprintf(w, "\t\t%q + // 0x%.8X: 0x%.8X\n", string(buf[:]), key, uint32(i))
815 }
816 }
817 // hack so we don't have to special case the trailing plus sign
818 fmt.Fprintf(w, ` ""`)
819 fmt.Fprintln(w)
820 }
821
822 fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
823 gen.WriteVersionedGoFile("tables.go", "norm", w.Bytes())
824 }
825
826 func printChars() {
827 if *verbose {
828 for _, c := range chars {
829 if !c.isValid() || c.state == SMissing {
830 continue
831 }
832 fmt.Println(c)
833 }
834 }
835 }
836
837 // verifyComputed does various consistency tests.
838 func verifyComputed() {
839 for i, c := range chars {
840 for _, f := range c.forms {
841 isNo := (f.quickCheck[MDecomposed] == QCNo)
842 if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) {
843 log.Fatalf("%U: NF*D QC must be No if rune decomposes", i)
844 }
845
846 isMaybe := f.quickCheck[MComposed] == QCMaybe
847 if f.combinesBackward != isMaybe {
848 log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i)
849 }
850
851 if len(f.expandedDecomp) != 0 {
852 continue
853 }
854 if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b {
855 // We accept these runes to be treated differently (it only affects
856 // segment breaking in iteration, most likely on improper use), but
857 // reconsider if more characters are added.
858 // U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;;
859 // U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;;
860 // U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
861 // U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;;
862 // U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;;
863 // U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;;
864 if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) {
865 log.Fatalf("%U: nLead was %v; want %v", i, a, b)
866 }
867 }
868 }
869 nfc := c.forms[FCanonical]
870 nfkc := c.forms[FCompatibility]
871 if nfc.combinesBackward != nfkc.combinesBackward {
872 log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
873 }
874 }
875 }
876
877 // Use values in DerivedNormalizationProps.txt to compare against the
878 // values we computed.
879 // DerivedNormalizationProps.txt has form:
880 // 00C0..00C5 ; NFD_QC; N # ...
881 // 0374 ; NFD_QC; N # ...
882 // See https://unicode.org/reports/tr44/ for full explanation
883 func testDerived() {
884 f := gen.OpenUCDFile("DerivedNormalizationProps.txt")
885 defer f.Close()
886 p := ucd.New(f)
887 for p.Next() {
888 r := p.Rune(0)
889 c := &chars[r]
890
891 var ftype, mode int
892 qt := p.String(1)
893 switch qt {
894 case "NFC_QC":
895 ftype, mode = FCanonical, MComposed
896 case "NFD_QC":
897 ftype, mode = FCanonical, MDecomposed
898 case "NFKC_QC":
899 ftype, mode = FCompatibility, MComposed
900 case "NFKD_QC":
901 ftype, mode = FCompatibility, MDecomposed
902 default:
903 continue
904 }
905 var qr QCResult
906 switch p.String(2) {
907 case "Y":
908 qr = QCYes
909 case "N":
910 qr = QCNo
911 case "M":
912 qr = QCMaybe
913 default:
914 log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
915 }
916 if got := c.forms[ftype].quickCheck[mode]; got != qr {
917 log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
918 }
919 c.forms[ftype].verified[mode] = true
920 }
921 if err := p.Err(); err != nil {
922 log.Fatal(err)
923 }
924 // Any unspecified value must be QCYes. Verify this.
925 for i, c := range chars {
926 for j, fd := range c.forms {
927 for k, qr := range fd.quickCheck {
928 if !fd.verified[k] && qr != QCYes {
929 m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n"
930 log.Printf(m, i, j, k, qr, c.name)
931 }
932 }
933 }
934 }
935 }
936
937 var testHeader = `const (
938 Yes = iota
939 No
940 Maybe
941 )
942
943 type formData struct {
944 qc uint8
945 combinesForward bool
946 decomposition string
947 }
948
949 type runeData struct {
950 r rune
951 ccc uint8
952 nLead uint8
953 nTrail uint8
954 f [2]formData // 0: canonical; 1: compatibility
955 }
956
957 func f(qc uint8, cf bool, dec string) [2]formData {
958 return [2]formData{{qc, cf, dec}, {qc, cf, dec}}
959 }
960
961 func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData {
962 return [2]formData{{qc, cf, d}, {qck, cfk, dk}}
963 }
964
965 var testData = []runeData{
966 `
967
968 func printTestdata() {
969 type lastInfo struct {
970 ccc uint8
971 nLead uint8
972 nTrail uint8
973 f string
974 }
975
976 last := lastInfo{}
977 w := &bytes.Buffer{}
978 fmt.Fprintf(w, testHeader)
979 for r, c := range chars {
980 f := c.forms[FCanonical]
981 qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
982 f = c.forms[FCompatibility]
983 qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
984 s := ""
985 if d == dk && qc == qck && cf == cfk {
986 s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d)
987 } else {
988 s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk)
989 }
990 current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s}
991 if last != current {
992 fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
993 last = current
994 }
995 }
996 fmt.Fprintln(w, "}")
997 gen.WriteVersionedGoFile("data_test.go", "norm", w.Bytes())
998 }