| 1 |
// Copyright 2009 The Go Authors. All rights reserved. |
| 2 |
// Use of this source code is governed by a BSD-style |
| 3 |
// license that can be found in the LICENSE file. |
| 4 |
|
| 5 |
//go:build ignore |
| 6 |
|
| 7 |
// Unicode table generator. |
| 8 |
// Data read from the web. |
| 9 |
|
| 10 |
package main |
| 11 |
|
| 12 |
import ( |
| 13 |
"flag" |
| 14 |
"fmt" |
| 15 |
"log" |
| 16 |
"maps" |
| 17 |
"os" |
| 18 |
"regexp" |
| 19 |
"slices" |
| 20 |
"sort" |
| 21 |
"strings" |
| 22 |
"unicode" |
| 23 |
|
| 24 |
"golang.org/x/text/internal/gen" |
| 25 |
"golang.org/x/text/internal/ucd" |
| 26 |
"golang.org/x/text/unicode/rangetable" |
| 27 |
) |
| 28 |
|
| 29 |
func main() { |
| 30 |
flag.Parse() |
| 31 |
setupOutput() |
| 32 |
loadChars() // always needed |
| 33 |
loadCasefold() |
| 34 |
printCategories() |
| 35 |
printScriptOrProperty(false) |
| 36 |
printScriptOrProperty(true) |
| 37 |
printCases() |
| 38 |
printLatinProperties() |
| 39 |
printCasefold() |
| 40 |
printSizes() |
| 41 |
flushOutput() |
| 42 |
} |
| 43 |
|
| 44 |
func defaultVersion() string { |
| 45 |
if v := os.Getenv("UNICODE_VERSION"); v != "" { |
| 46 |
return v |
| 47 |
} |
| 48 |
return unicode.Version |
| 49 |
} |
| 50 |
|
| 51 |
var tablelist = flag.String("tables", |
| 52 |
"all", |
| 53 |
"comma-separated list of which tables to generate; can be letter") |
| 54 |
var scriptlist = flag.String("scripts", |
| 55 |
"all", |
| 56 |
"comma-separated list of which script tables to generate") |
| 57 |
var proplist = flag.String("props", |
| 58 |
"all", |
| 59 |
"comma-separated list of which property tables to generate") |
| 60 |
var cases = flag.Bool("cases", |
| 61 |
true, |
| 62 |
"generate case tables") |
| 63 |
var test = flag.Bool("test", |
| 64 |
false, |
| 65 |
"test existing tables; can be used to compare web data with package data") |
| 66 |
|
| 67 |
var scriptRe = regexp.MustCompile(`^([0-9A-F]+)(\.\.[0-9A-F]+)? *; ([A-Za-z_]+)$`) |
| 68 |
var logger = log.New(os.Stderr, "", log.Lshortfile) |
| 69 |
|
| 70 |
var output *gen.CodeWriter |
| 71 |
|
| 72 |
func setupOutput() { |
| 73 |
output = gen.NewCodeWriter() |
| 74 |
} |
| 75 |
|
| 76 |
func flushOutput() { |
| 77 |
output.WriteGoFile("tables.go", "unicode") |
| 78 |
} |
| 79 |
|
| 80 |
func printf(format string, args ...interface{}) { |
| 81 |
fmt.Fprintf(output, format, args...) |
| 82 |
} |
| 83 |
|
| 84 |
func print(args ...interface{}) { |
| 85 |
fmt.Fprint(output, args...) |
| 86 |
} |
| 87 |
|
| 88 |
func println(args ...interface{}) { |
| 89 |
fmt.Fprintln(output, args...) |
| 90 |
} |
| 91 |
|
| 92 |
var category = map[string]bool{ |
| 93 |
// Nd Lu etc. |
| 94 |
// We use one-character names to identify merged categories |
| 95 |
"L": true, // Lu Ll Lt Lm Lo |
| 96 |
"LC": true, // Lu Ll Lt |
| 97 |
"P": true, // Pc Pd Ps Pe Pu Pf Po |
| 98 |
"M": true, // Mn Mc Me |
| 99 |
"N": true, // Nd Nl No |
| 100 |
"S": true, // Sm Sc Sk So |
| 101 |
"Z": true, // Zs Zl Zp |
| 102 |
"C": true, // Cc Cf Cs Co Cn |
| 103 |
"Cn": true, // unassigned |
| 104 |
} |
| 105 |
|
| 106 |
// This contains only the properties we're interested in. |
| 107 |
type Char struct { |
| 108 |
codePoint rune // if zero, this index is not a valid code point. |
| 109 |
category string |
| 110 |
upperCase rune |
| 111 |
lowerCase rune |
| 112 |
titleCase rune |
| 113 |
foldCase rune // simple case folding |
| 114 |
caseOrbit rune // next in simple case folding orbit |
| 115 |
} |
| 116 |
|
| 117 |
const MaxChar = 0x10FFFF |
| 118 |
|
| 119 |
var chars = make([]Char, MaxChar+1) |
| 120 |
var scripts = make(map[string][]rune) |
| 121 |
var props = make(map[string][]rune) // a property looks like a script; can share the format |
| 122 |
|
| 123 |
func allCategories() []string { |
| 124 |
a := make([]string, 0, len(category)) |
| 125 |
for k := range category { |
| 126 |
a = append(a, k) |
| 127 |
} |
| 128 |
sort.Strings(a) |
| 129 |
return a |
| 130 |
} |
| 131 |
|
| 132 |
func all(scripts map[string][]rune) []string { |
| 133 |
a := make([]string, 0, len(scripts)) |
| 134 |
for k := range scripts { |
| 135 |
a = append(a, k) |
| 136 |
} |
| 137 |
sort.Strings(a) |
| 138 |
return a |
| 139 |
} |
| 140 |
|
| 141 |
func allCatFold(m map[string]map[rune]bool) []string { |
| 142 |
a := make([]string, 0, len(m)) |
| 143 |
for k := range m { |
| 144 |
a = append(a, k) |
| 145 |
} |
| 146 |
sort.Strings(a) |
| 147 |
return a |
| 148 |
} |
| 149 |
|
| 150 |
func categoryOp(code rune, class uint8) bool { |
| 151 |
category := chars[code].category |
| 152 |
return len(category) > 0 && category[0] == class |
| 153 |
} |
| 154 |
|
| 155 |
func loadChars() { |
| 156 |
for code := range chars { |
| 157 |
chars[code].category = "Cn" // unassigned |
| 158 |
} |
| 159 |
ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) { |
| 160 |
c := Char{codePoint: p.Rune(0)} |
| 161 |
|
| 162 |
getRune := func(field int) rune { |
| 163 |
if p.String(field) == "" { |
| 164 |
return 0 |
| 165 |
} |
| 166 |
return p.Rune(field) |
| 167 |
} |
| 168 |
|
| 169 |
c.category = p.String(ucd.GeneralCategory) |
| 170 |
category[c.category] = true |
| 171 |
switch c.category { |
| 172 |
case "Nd": |
| 173 |
// Decimal digit |
| 174 |
p.Int(ucd.NumericValue) |
| 175 |
case "Lu": |
| 176 |
c.upperCase = getRune(ucd.CodePoint) |
| 177 |
c.lowerCase = getRune(ucd.SimpleLowercaseMapping) |
| 178 |
c.titleCase = getRune(ucd.SimpleTitlecaseMapping) |
| 179 |
case "Ll": |
| 180 |
c.upperCase = getRune(ucd.SimpleUppercaseMapping) |
| 181 |
c.lowerCase = getRune(ucd.CodePoint) |
| 182 |
c.titleCase = getRune(ucd.SimpleTitlecaseMapping) |
| 183 |
case "Lt": |
| 184 |
c.upperCase = getRune(ucd.SimpleUppercaseMapping) |
| 185 |
c.lowerCase = getRune(ucd.SimpleLowercaseMapping) |
| 186 |
c.titleCase = getRune(ucd.CodePoint) |
| 187 |
default: |
| 188 |
c.upperCase = getRune(ucd.SimpleUppercaseMapping) |
| 189 |
c.lowerCase = getRune(ucd.SimpleLowercaseMapping) |
| 190 |
c.titleCase = getRune(ucd.SimpleTitlecaseMapping) |
| 191 |
} |
| 192 |
|
| 193 |
chars[c.codePoint] = c |
| 194 |
}) |
| 195 |
} |
| 196 |
|
| 197 |
func loadCasefold() { |
| 198 |
ucd.Parse(gen.OpenUCDFile("CaseFolding.txt"), func(p *ucd.Parser) { |
| 199 |
kind := p.String(1) |
| 200 |
if kind != "C" && kind != "S" { |
| 201 |
// Only care about 'common' and 'simple' foldings. |
| 202 |
return |
| 203 |
} |
| 204 |
p1 := p.Rune(0) |
| 205 |
p2 := p.Rune(2) |
| 206 |
chars[p1].foldCase = rune(p2) |
| 207 |
}) |
| 208 |
} |
| 209 |
|
| 210 |
var categoryMapping = map[string]string{ |
| 211 |
"LC": "Letter, cased: Ll | Lt | Lu", |
| 212 |
"Lu": "Letter, uppercase", |
| 213 |
"Ll": "Letter, lowercase", |
| 214 |
"Lt": "Letter, titlecase", |
| 215 |
"Lm": "Letter, modifier", |
| 216 |
"Lo": "Letter, other", |
| 217 |
"Mn": "Mark, nonspacing", |
| 218 |
"Mc": "Mark, spacing combining", |
| 219 |
"Me": "Mark, enclosing", |
| 220 |
"Nd": "Number, decimal digit", |
| 221 |
"Nl": "Number, letter", |
| 222 |
"No": "Number, other", |
| 223 |
"Pc": "Punctuation, connector", |
| 224 |
"Pd": "Punctuation, dash", |
| 225 |
"Ps": "Punctuation, open", |
| 226 |
"Pe": "Punctuation, close", |
| 227 |
"Pi": "Punctuation, initial quote", |
| 228 |
"Pf": "Punctuation, final quote", |
| 229 |
"Po": "Punctuation, other", |
| 230 |
"Sm": "Symbol, math", |
| 231 |
"Sc": "Symbol, currency", |
| 232 |
"Sk": "Symbol, modifier", |
| 233 |
"So": "Symbol, other", |
| 234 |
"Zs": "Separator, space", |
| 235 |
"Zl": "Separator, line", |
| 236 |
"Zp": "Separator, paragraph", |
| 237 |
"Cc": "Other, control", |
| 238 |
"Cf": "Other, format", |
| 239 |
"Cs": "Other, surrogate", |
| 240 |
"Co": "Other, private use", |
| 241 |
"Cn": "Other, not assigned", |
| 242 |
} |
| 243 |
|
| 244 |
func printCategories() { |
| 245 |
if *tablelist == "" { |
| 246 |
return |
| 247 |
} |
| 248 |
// Find out which categories to dump |
| 249 |
list := strings.Split(*tablelist, ",") |
| 250 |
if *tablelist == "all" { |
| 251 |
list = allCategories() |
| 252 |
} |
| 253 |
if *test { |
| 254 |
fullCategoryTest(list) |
| 255 |
return |
| 256 |
} |
| 257 |
|
| 258 |
println("// Version is the Unicode edition from which the tables are derived.") |
| 259 |
printf("const Version = %q\n\n", gen.UnicodeVersion()) |
| 260 |
|
| 261 |
if *tablelist == "all" { |
| 262 |
println("// Categories is the set of Unicode category tables.") |
| 263 |
println("var Categories = map[string] *RangeTable {") |
| 264 |
for _, k := range allCategories() { |
| 265 |
printf("\t%q: %s,\n", k, k) |
| 266 |
} |
| 267 |
print("}\n\n") |
| 268 |
printCategoryAliases() |
| 269 |
} |
| 270 |
|
| 271 |
decl := make(sort.StringSlice, len(list)) |
| 272 |
ndecl := 0 |
| 273 |
for _, name := range list { |
| 274 |
if _, ok := category[name]; !ok { |
| 275 |
logger.Fatal("unknown category", name) |
| 276 |
} |
| 277 |
// We generate an UpperCase name to serve as concise documentation and an _UnderScored |
| 278 |
// name to store the data. This stops godoc dumping all the tables but keeps them |
| 279 |
// available to clients. |
| 280 |
// Cases deserving special comments |
| 281 |
varDecl := "" |
| 282 |
switch name { |
| 283 |
case "C": |
| 284 |
varDecl = "\tOther = _C; // Other/C is the set of Unicode control, special, and unassigned code points, category C.\n" |
| 285 |
varDecl += "\tC = _C\n" |
| 286 |
case "L": |
| 287 |
varDecl = "\tLetter = _L; // Letter/L is the set of Unicode letters, category L.\n" |
| 288 |
varDecl += "\tL = _L\n" |
| 289 |
case "M": |
| 290 |
varDecl = "\tMark = _M; // Mark/M is the set of Unicode mark characters, category M.\n" |
| 291 |
varDecl += "\tM = _M\n" |
| 292 |
case "N": |
| 293 |
varDecl = "\tNumber = _N; // Number/N is the set of Unicode number characters, category N.\n" |
| 294 |
varDecl += "\tN = _N\n" |
| 295 |
case "P": |
| 296 |
varDecl = "\tPunct = _P; // Punct/P is the set of Unicode punctuation characters, category P.\n" |
| 297 |
varDecl += "\tP = _P\n" |
| 298 |
case "S": |
| 299 |
varDecl = "\tSymbol = _S; // Symbol/S is the set of Unicode symbol characters, category S.\n" |
| 300 |
varDecl += "\tS = _S\n" |
| 301 |
case "Z": |
| 302 |
varDecl = "\tSpace = _Z; // Space/Z is the set of Unicode space characters, category Z.\n" |
| 303 |
varDecl += "\tZ = _Z\n" |
| 304 |
case "Nd": |
| 305 |
varDecl = "\tDigit = _Nd; // Digit is the set of Unicode characters with the \"decimal digit\" property.\n" |
| 306 |
case "Lu": |
| 307 |
varDecl = "\tUpper = _Lu; // Upper is the set of Unicode upper case letters.\n" |
| 308 |
case "Ll": |
| 309 |
varDecl = "\tLower = _Ll; // Lower is the set of Unicode lower case letters.\n" |
| 310 |
case "Lt": |
| 311 |
varDecl = "\tTitle = _Lt; // Title is the set of Unicode title case letters.\n" |
| 312 |
} |
| 313 |
if len(name) > 1 { |
| 314 |
desc, ok := categoryMapping[name] |
| 315 |
if ok { |
| 316 |
varDecl += fmt.Sprintf( |
| 317 |
"\t%s = _%s; // %s is the set of Unicode characters in category %s (%s).\n", |
| 318 |
name, name, name, name, desc) |
| 319 |
} else { |
| 320 |
varDecl += fmt.Sprintf( |
| 321 |
"\t%s = _%s; // %s is the set of Unicode characters in category %s.\n", |
| 322 |
name, name, name, name) |
| 323 |
} |
| 324 |
} |
| 325 |
decl[ndecl] = varDecl |
| 326 |
ndecl++ |
| 327 |
match := func(cat string) bool { return cat == name } |
| 328 |
if len(name) == 1 { // unified categories |
| 329 |
match = func(cat string) bool { return strings.HasPrefix(cat, name) } |
| 330 |
} |
| 331 |
if name == "LC" { // special unified category |
| 332 |
match = func(cat string) bool { return cat == "Ll" || cat == "Lt" || cat == "Lu" } |
| 333 |
} |
| 334 |
dumpRange("_"+name, func(code rune) bool { return match(chars[code].category) }) |
| 335 |
} |
| 336 |
decl.Sort() |
| 337 |
println("// These variables have type *RangeTable.") |
| 338 |
println("var (") |
| 339 |
for _, d := range decl { |
| 340 |
print(d) |
| 341 |
} |
| 342 |
print(")\n\n") |
| 343 |
} |
| 344 |
|
| 345 |
func printCategoryAliases() { |
| 346 |
known := make(map[string]bool) |
| 347 |
for _, name := range allCategories() { |
| 348 |
known[name] = true |
| 349 |
} |
| 350 |
|
| 351 |
table := make(map[string]string) |
| 352 |
ucd.Parse(gen.OpenUCDFile("PropertyValueAliases.txt"), func(p *ucd.Parser) { |
| 353 |
if p.String(0) != "gc" { |
| 354 |
return |
| 355 |
} |
| 356 |
name := p.String(1) |
| 357 |
if !known[name] { |
| 358 |
logger.Print("unknown category: ", name) |
| 359 |
} |
| 360 |
table[p.String(2)] = name |
| 361 |
if a := p.String(3); a != "" { |
| 362 |
table[a] = name |
| 363 |
} |
| 364 |
}) |
| 365 |
|
| 366 |
println("// CategoryAliases maps category aliases to standard category names.") |
| 367 |
println("var CategoryAliases = map[string]string{") |
| 368 |
for _, name := range slices.Sorted(maps.Keys(table)) { |
| 369 |
printf("\t%q: %q,\n", name, table[name]) |
| 370 |
} |
| 371 |
print("}\n\n") |
| 372 |
} |
| 373 |
|
| 374 |
type Op func(code rune) bool |
| 375 |
|
| 376 |
func dumpRange(name string, inCategory Op) { |
| 377 |
runes := []rune{} |
| 378 |
for i := range chars { |
| 379 |
r := rune(i) |
| 380 |
if inCategory(r) { |
| 381 |
runes = append(runes, r) |
| 382 |
} |
| 383 |
} |
| 384 |
printRangeTable(name, runes) |
| 385 |
} |
| 386 |
|
| 387 |
func printRangeTable(name string, runes []rune) { |
| 388 |
rt := rangetable.New(runes...) |
| 389 |
printf("var %s = &RangeTable{\n", name) |
| 390 |
println("\tR16: []Range16{") |
| 391 |
for _, r := range rt.R16 { |
| 392 |
printf("\t\t{%#04x, %#04x, %d},\n", r.Lo, r.Hi, r.Stride) |
| 393 |
range16Count++ |
| 394 |
} |
| 395 |
println("\t},") |
| 396 |
if len(rt.R32) > 0 { |
| 397 |
println("\tR32: []Range32{") |
| 398 |
for _, r := range rt.R32 { |
| 399 |
printf("\t\t{%#x, %#x, %d},\n", r.Lo, r.Hi, r.Stride) |
| 400 |
range32Count++ |
| 401 |
} |
| 402 |
println("\t},") |
| 403 |
} |
| 404 |
if rt.LatinOffset > 0 { |
| 405 |
printf("\tLatinOffset: %d,\n", rt.LatinOffset) |
| 406 |
} |
| 407 |
printf("}\n\n") |
| 408 |
} |
| 409 |
|
| 410 |
func fullCategoryTest(list []string) { |
| 411 |
for _, name := range list { |
| 412 |
if _, ok := category[name]; !ok { |
| 413 |
logger.Fatal("unknown category", name) |
| 414 |
} |
| 415 |
r, ok := unicode.Categories[name] |
| 416 |
if !ok && len(name) > 1 { |
| 417 |
logger.Fatalf("unknown table %q", name) |
| 418 |
} |
| 419 |
if len(name) == 1 { |
| 420 |
verifyRange(name, func(code rune) bool { return categoryOp(code, name[0]) }, r) |
| 421 |
} else { |
| 422 |
verifyRange( |
| 423 |
name, |
| 424 |
func(code rune) bool { return chars[code].category == name }, |
| 425 |
r) |
| 426 |
} |
| 427 |
} |
| 428 |
} |
| 429 |
|
| 430 |
func verifyRange(name string, inCategory Op, table *unicode.RangeTable) { |
| 431 |
count := 0 |
| 432 |
for j := range chars { |
| 433 |
i := rune(j) |
| 434 |
web := inCategory(i) |
| 435 |
pkg := unicode.Is(table, i) |
| 436 |
if web != pkg { |
| 437 |
fmt.Fprintf(os.Stderr, "%s: %U: web=%t pkg=%t\n", name, i, web, pkg) |
| 438 |
count++ |
| 439 |
if count > 10 { |
| 440 |
break |
| 441 |
} |
| 442 |
} |
| 443 |
} |
| 444 |
} |
| 445 |
|
| 446 |
func fullScriptTest(list []string, installed map[string]*unicode.RangeTable, scripts map[string][]rune) { |
| 447 |
for _, name := range list { |
| 448 |
if _, ok := scripts[name]; !ok { |
| 449 |
logger.Fatal("unknown script", name) |
| 450 |
} |
| 451 |
_, ok := installed[name] |
| 452 |
if !ok { |
| 453 |
logger.Fatal("unknown table", name) |
| 454 |
} |
| 455 |
for _, r := range scripts[name] { |
| 456 |
if !unicode.Is(installed[name], rune(r)) { |
| 457 |
fmt.Fprintf(os.Stderr, "%U: not in script %s\n", r, name) |
| 458 |
} |
| 459 |
} |
| 460 |
} |
| 461 |
} |
| 462 |
|
| 463 |
var deprecatedAliases = map[string]string{ |
| 464 |
"Sentence_Terminal": "STerm", |
| 465 |
} |
| 466 |
|
| 467 |
// PropList.txt has the same format as Scripts.txt so we can share its parser. |
| 468 |
func printScriptOrProperty(doProps bool) { |
| 469 |
flaglist := *scriptlist |
| 470 |
file := "Scripts.txt" |
| 471 |
table := scripts |
| 472 |
installed := unicode.Scripts |
| 473 |
if doProps { |
| 474 |
flaglist = *proplist |
| 475 |
file = "PropList.txt" |
| 476 |
table = props |
| 477 |
installed = unicode.Properties |
| 478 |
} |
| 479 |
if flaglist == "" { |
| 480 |
return |
| 481 |
} |
| 482 |
ucd.Parse(gen.OpenUCDFile(file), func(p *ucd.Parser) { |
| 483 |
name := p.String(1) |
| 484 |
table[name] = append(table[name], p.Rune(0)) |
| 485 |
}) |
| 486 |
// Find out which scripts to dump |
| 487 |
list := strings.Split(flaglist, ",") |
| 488 |
if flaglist == "all" { |
| 489 |
list = all(table) |
| 490 |
} |
| 491 |
if *test { |
| 492 |
fullScriptTest(list, installed, table) |
| 493 |
return |
| 494 |
} |
| 495 |
|
| 496 |
if flaglist == "all" { |
| 497 |
if doProps { |
| 498 |
println("// Properties is the set of Unicode property tables.") |
| 499 |
println("var Properties = map[string] *RangeTable{") |
| 500 |
} else { |
| 501 |
println("// Scripts is the set of Unicode script tables.") |
| 502 |
println("var Scripts = map[string] *RangeTable{") |
| 503 |
} |
| 504 |
for _, k := range all(table) { |
| 505 |
printf("\t%q: %s,\n", k, k) |
| 506 |
if alias, ok := deprecatedAliases[k]; ok { |
| 507 |
printf("\t%q: %s,\n", alias, k) |
| 508 |
} |
| 509 |
} |
| 510 |
print("}\n\n") |
| 511 |
} |
| 512 |
|
| 513 |
decl := make(sort.StringSlice, len(list)+len(deprecatedAliases)) |
| 514 |
ndecl := 0 |
| 515 |
for _, name := range list { |
| 516 |
if doProps { |
| 517 |
decl[ndecl] = fmt.Sprintf( |
| 518 |
"\t%s = _%s;\t// %s is the set of Unicode characters with property %s.\n", |
| 519 |
name, name, name, name) |
| 520 |
} else { |
| 521 |
decl[ndecl] = fmt.Sprintf( |
| 522 |
"\t%s = _%s;\t// %s is the set of Unicode characters in script %s.\n", |
| 523 |
name, name, name, name) |
| 524 |
} |
| 525 |
ndecl++ |
| 526 |
if alias, ok := deprecatedAliases[name]; ok { |
| 527 |
decl[ndecl] = fmt.Sprintf( |
| 528 |
"\t%[1]s = _%[2]s;\t// %[1]s is an alias for %[2]s.\n", |
| 529 |
alias, name) |
| 530 |
ndecl++ |
| 531 |
} |
| 532 |
printRangeTable("_"+name, table[name]) |
| 533 |
} |
| 534 |
decl.Sort() |
| 535 |
println("// These variables have type *RangeTable.") |
| 536 |
println("var (") |
| 537 |
for _, d := range decl { |
| 538 |
print(d) |
| 539 |
} |
| 540 |
print(")\n\n") |
| 541 |
} |
| 542 |
|
| 543 |
const ( |
| 544 |
CaseUpper = 1 << iota |
| 545 |
CaseLower |
| 546 |
CaseTitle |
| 547 |
CaseNone = 0 // must be zero |
| 548 |
CaseMissing = -1 // character not present; not a valid case state |
| 549 |
) |
| 550 |
|
| 551 |
type caseState struct { |
| 552 |
point rune |
| 553 |
_case int |
| 554 |
deltaToUpper rune |
| 555 |
deltaToLower rune |
| 556 |
deltaToTitle rune |
| 557 |
} |
| 558 |
|
| 559 |
// Is d a continuation of the state of c? |
| 560 |
func (c *caseState) adjacent(d *caseState) bool { |
| 561 |
if d.point < c.point { |
| 562 |
c, d = d, c |
| 563 |
} |
| 564 |
switch { |
| 565 |
case d.point != c.point+1: // code points not adjacent (shouldn't happen) |
| 566 |
return false |
| 567 |
case d._case != c._case: // different cases |
| 568 |
return c.upperLowerAdjacent(d) |
| 569 |
case c._case == CaseNone: |
| 570 |
return false |
| 571 |
case c._case == CaseMissing: |
| 572 |
return false |
| 573 |
case d.deltaToUpper != c.deltaToUpper: |
| 574 |
return false |
| 575 |
case d.deltaToLower != c.deltaToLower: |
| 576 |
return false |
| 577 |
case d.deltaToTitle != c.deltaToTitle: |
| 578 |
return false |
| 579 |
} |
| 580 |
return true |
| 581 |
} |
| 582 |
|
| 583 |
// Is d the same as c, but opposite in upper/lower case? this would make it |
| 584 |
// an element of an UpperLower sequence. |
| 585 |
func (c *caseState) upperLowerAdjacent(d *caseState) bool { |
| 586 |
// check they're a matched case pair. we know they have adjacent values |
| 587 |
switch { |
| 588 |
case c._case == CaseUpper && d._case != CaseLower: |
| 589 |
return false |
| 590 |
case c._case == CaseLower && d._case != CaseUpper: |
| 591 |
return false |
| 592 |
} |
| 593 |
// matched pair (at least in upper/lower). make the order Upper Lower |
| 594 |
if c._case == CaseLower { |
| 595 |
c, d = d, c |
| 596 |
} |
| 597 |
// for an Upper Lower sequence the deltas have to be in order |
| 598 |
// c: 0 1 0 |
| 599 |
// d: -1 0 -1 |
| 600 |
switch { |
| 601 |
case c.deltaToUpper != 0: |
| 602 |
return false |
| 603 |
case c.deltaToLower != 1: |
| 604 |
return false |
| 605 |
case c.deltaToTitle != 0: |
| 606 |
return false |
| 607 |
case d.deltaToUpper != -1: |
| 608 |
return false |
| 609 |
case d.deltaToLower != 0: |
| 610 |
return false |
| 611 |
case d.deltaToTitle != -1: |
| 612 |
return false |
| 613 |
} |
| 614 |
return true |
| 615 |
} |
| 616 |
|
| 617 |
// Does this character start an UpperLower sequence? |
| 618 |
func (c *caseState) isUpperLower() bool { |
| 619 |
// for an Upper Lower sequence the deltas have to be in order |
| 620 |
// c: 0 1 0 |
| 621 |
switch { |
| 622 |
case c.deltaToUpper != 0: |
| 623 |
return false |
| 624 |
case c.deltaToLower != 1: |
| 625 |
return false |
| 626 |
case c.deltaToTitle != 0: |
| 627 |
return false |
| 628 |
} |
| 629 |
return true |
| 630 |
} |
| 631 |
|
| 632 |
// Does this character start a LowerUpper sequence? |
| 633 |
func (c *caseState) isLowerUpper() bool { |
| 634 |
// for an Upper Lower sequence the deltas have to be in order |
| 635 |
// c: -1 0 -1 |
| 636 |
switch { |
| 637 |
case c.deltaToUpper != -1: |
| 638 |
return false |
| 639 |
case c.deltaToLower != 0: |
| 640 |
return false |
| 641 |
case c.deltaToTitle != -1: |
| 642 |
return false |
| 643 |
} |
| 644 |
return true |
| 645 |
} |
| 646 |
|
| 647 |
func getCaseState(i rune) (c *caseState) { |
| 648 |
c = &caseState{point: i, _case: CaseNone} |
| 649 |
ch := &chars[i] |
| 650 |
switch ch.codePoint { |
| 651 |
case 0: |
| 652 |
c._case = CaseMissing // Will get NUL wrong but that doesn't matter |
| 653 |
return |
| 654 |
case ch.upperCase: |
| 655 |
c._case = CaseUpper |
| 656 |
case ch.lowerCase: |
| 657 |
c._case = CaseLower |
| 658 |
case ch.titleCase: |
| 659 |
c._case = CaseTitle |
| 660 |
} |
| 661 |
// Some things such as roman numeral U+2161 don't describe themselves |
| 662 |
// as upper case, but have a lower case. Second-guess them. |
| 663 |
if c._case == CaseNone && ch.lowerCase != 0 { |
| 664 |
c._case = CaseUpper |
| 665 |
} |
| 666 |
// Same in the other direction. |
| 667 |
if c._case == CaseNone && ch.upperCase != 0 { |
| 668 |
c._case = CaseLower |
| 669 |
} |
| 670 |
|
| 671 |
if ch.upperCase != 0 { |
| 672 |
c.deltaToUpper = ch.upperCase - i |
| 673 |
} |
| 674 |
if ch.lowerCase != 0 { |
| 675 |
c.deltaToLower = ch.lowerCase - i |
| 676 |
} |
| 677 |
if ch.titleCase != 0 { |
| 678 |
c.deltaToTitle = ch.titleCase - i |
| 679 |
} |
| 680 |
return |
| 681 |
} |
| 682 |
|
| 683 |
func printCases() { |
| 684 |
if *test { |
| 685 |
fullCaseTest() |
| 686 |
return |
| 687 |
} |
| 688 |
printf( |
| 689 |
"// CaseRanges is the table describing case mappings for all letters with\n" + |
| 690 |
"// non-self mappings.\n" + |
| 691 |
"var CaseRanges = _CaseRanges\n" + |
| 692 |
"var _CaseRanges = []CaseRange {\n") |
| 693 |
|
| 694 |
var startState *caseState // the start of a run; nil for not active |
| 695 |
var prevState = &caseState{} // the state of the previous character |
| 696 |
for i := range chars { |
| 697 |
state := getCaseState(rune(i)) |
| 698 |
if state.adjacent(prevState) { |
| 699 |
prevState = state |
| 700 |
continue |
| 701 |
} |
| 702 |
// end of run (possibly) |
| 703 |
printCaseRange(startState, prevState) |
| 704 |
startState = nil |
| 705 |
if state._case != CaseMissing && state._case != CaseNone { |
| 706 |
startState = state |
| 707 |
} |
| 708 |
prevState = state |
| 709 |
} |
| 710 |
print("}\n") |
| 711 |
} |
| 712 |
|
| 713 |
func printCaseRange(lo, hi *caseState) { |
| 714 |
if lo == nil { |
| 715 |
return |
| 716 |
} |
| 717 |
if lo.deltaToUpper == 0 && lo.deltaToLower == 0 && lo.deltaToTitle == 0 { |
| 718 |
// character represents itself in all cases - no need to mention it |
| 719 |
return |
| 720 |
} |
| 721 |
switch { |
| 722 |
case hi.point > lo.point && lo.isUpperLower(): |
| 723 |
printf("\t{0x%04X, 0x%04X, d{UpperLower, UpperLower, UpperLower}},\n", |
| 724 |
lo.point, hi.point) |
| 725 |
case hi.point > lo.point && lo.isLowerUpper(): |
| 726 |
logger.Fatalf("LowerUpper sequence: should not happen: %U. If it's real, need to fix To()", lo.point) |
| 727 |
printf("\t{0x%04X, 0x%04X, d{LowerUpper, LowerUpper, LowerUpper}},\n", |
| 728 |
lo.point, hi.point) |
| 729 |
default: |
| 730 |
printf("\t{0x%04X, 0x%04X, d{%d, %d, %d}},\n", |
| 731 |
lo.point, hi.point, |
| 732 |
lo.deltaToUpper, lo.deltaToLower, lo.deltaToTitle) |
| 733 |
} |
| 734 |
} |
| 735 |
|
| 736 |
// If the cased value in the Char is 0, it means use the rune itself. |
| 737 |
func caseIt(r, cased rune) rune { |
| 738 |
if cased == 0 { |
| 739 |
return r |
| 740 |
} |
| 741 |
return cased |
| 742 |
} |
| 743 |
|
| 744 |
func fullCaseTest() { |
| 745 |
for j, c := range chars { |
| 746 |
i := rune(j) |
| 747 |
lower := unicode.ToLower(i) |
| 748 |
want := caseIt(i, c.lowerCase) |
| 749 |
if lower != want { |
| 750 |
fmt.Fprintf(os.Stderr, "lower %U should be %U is %U\n", i, want, lower) |
| 751 |
} |
| 752 |
upper := unicode.ToUpper(i) |
| 753 |
want = caseIt(i, c.upperCase) |
| 754 |
if upper != want { |
| 755 |
fmt.Fprintf(os.Stderr, "upper %U should be %U is %U\n", i, want, upper) |
| 756 |
} |
| 757 |
title := unicode.ToTitle(i) |
| 758 |
want = caseIt(i, c.titleCase) |
| 759 |
if title != want { |
| 760 |
fmt.Fprintf(os.Stderr, "title %U should be %U is %U\n", i, want, title) |
| 761 |
} |
| 762 |
} |
| 763 |
} |
| 764 |
|
| 765 |
func printLatinProperties() { |
| 766 |
if *test { |
| 767 |
return |
| 768 |
} |
| 769 |
println("var properties = [MaxLatin1+1]uint8{") |
| 770 |
for code := 0; code <= unicode.MaxLatin1; code++ { |
| 771 |
var property string |
| 772 |
switch chars[code].category { |
| 773 |
case "Cc", "": // NUL has no category. |
| 774 |
property = "pC" |
| 775 |
case "Cf": // soft hyphen, unique category, not printable. |
| 776 |
property = "0" |
| 777 |
case "Ll": |
| 778 |
property = "pLl | pp" |
| 779 |
case "Lo": |
| 780 |
property = "pLo | pp" |
| 781 |
case "Lu": |
| 782 |
property = "pLu | pp" |
| 783 |
case "Nd", "No": |
| 784 |
property = "pN | pp" |
| 785 |
case "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps": |
| 786 |
property = "pP | pp" |
| 787 |
case "Sc", "Sk", "Sm", "So": |
| 788 |
property = "pS | pp" |
| 789 |
case "Zs": |
| 790 |
property = "pZ" |
| 791 |
default: |
| 792 |
logger.Fatalf("%U has unknown category %q", code, chars[code].category) |
| 793 |
} |
| 794 |
// Special case |
| 795 |
if code == ' ' { |
| 796 |
property = "pZ | pp" |
| 797 |
} |
| 798 |
printf("\t0x%02X: %s, // %q\n", code, property, code) |
| 799 |
} |
| 800 |
printf("}\n\n") |
| 801 |
} |
| 802 |
|
| 803 |
func printCasefold() { |
| 804 |
// Build list of case-folding groups attached to each canonical folded char (typically lower case). |
| 805 |
var caseOrbit = make([][]rune, MaxChar+1) |
| 806 |
for j := range chars { |
| 807 |
i := rune(j) |
| 808 |
c := &chars[i] |
| 809 |
if c.foldCase == 0 { |
| 810 |
continue |
| 811 |
} |
| 812 |
orb := caseOrbit[c.foldCase] |
| 813 |
if orb == nil { |
| 814 |
orb = append(orb, c.foldCase) |
| 815 |
} |
| 816 |
caseOrbit[c.foldCase] = append(orb, i) |
| 817 |
} |
| 818 |
|
| 819 |
// Insert explicit 1-element groups when assuming [lower, upper] would be wrong. |
| 820 |
for j := range chars { |
| 821 |
i := rune(j) |
| 822 |
c := &chars[i] |
| 823 |
f := c.foldCase |
| 824 |
if f == 0 { |
| 825 |
f = i |
| 826 |
} |
| 827 |
orb := caseOrbit[f] |
| 828 |
if orb == nil && (c.upperCase != 0 && c.upperCase != i || c.lowerCase != 0 && c.lowerCase != i) { |
| 829 |
// Default assumption of [upper, lower] is wrong. |
| 830 |
caseOrbit[i] = []rune{i} |
| 831 |
} |
| 832 |
} |
| 833 |
|
| 834 |
// Delete the groups for which assuming [lower, upper] or [upper, lower] is right. |
| 835 |
for i, orb := range caseOrbit { |
| 836 |
if len(orb) == 2 && chars[orb[0]].upperCase == orb[1] && chars[orb[1]].lowerCase == orb[0] { |
| 837 |
caseOrbit[i] = nil |
| 838 |
} |
| 839 |
if len(orb) == 2 && chars[orb[1]].upperCase == orb[0] && chars[orb[0]].lowerCase == orb[1] { |
| 840 |
caseOrbit[i] = nil |
| 841 |
} |
| 842 |
} |
| 843 |
|
| 844 |
// Record orbit information in chars. |
| 845 |
for _, orb := range caseOrbit { |
| 846 |
if orb == nil { |
| 847 |
continue |
| 848 |
} |
| 849 |
sort.Slice(orb, func(i, j int) bool { |
| 850 |
return orb[i] < orb[j] |
| 851 |
}) |
| 852 |
c := orb[len(orb)-1] |
| 853 |
for _, d := range orb { |
| 854 |
chars[c].caseOrbit = d |
| 855 |
c = d |
| 856 |
} |
| 857 |
} |
| 858 |
|
| 859 |
printAsciiFold() |
| 860 |
printCaseOrbit() |
| 861 |
|
| 862 |
// Tables of category and script folding exceptions: code points |
| 863 |
// that must be added when interpreting a particular category/script |
| 864 |
// in a case-folding context. |
| 865 |
cat := make(map[string]map[rune]bool) |
| 866 |
for name := range category { |
| 867 |
if x := foldExceptions(inCategory(name)); len(x) > 0 { |
| 868 |
cat[name] = x |
| 869 |
} |
| 870 |
} |
| 871 |
|
| 872 |
scr := make(map[string]map[rune]bool) |
| 873 |
for name := range scripts { |
| 874 |
if x := foldExceptions(scripts[name]); len(x) > 0 { |
| 875 |
scr[name] = x |
| 876 |
} |
| 877 |
} |
| 878 |
|
| 879 |
printCatFold("FoldCategory", cat) |
| 880 |
printCatFold("FoldScript", scr) |
| 881 |
} |
| 882 |
|
| 883 |
// inCategory returns a list of all the runes in the category. |
| 884 |
func inCategory(name string) []rune { |
| 885 |
var x []rune |
| 886 |
for j := range chars { |
| 887 |
i := rune(j) |
| 888 |
c := &chars[i] |
| 889 |
if c.category == name || len(name) == 1 && len(c.category) > 1 && c.category[0] == name[0] { |
| 890 |
x = append(x, i) |
| 891 |
} |
| 892 |
} |
| 893 |
return x |
| 894 |
} |
| 895 |
|
| 896 |
// foldExceptions returns a list of all the runes fold-equivalent |
| 897 |
// to runes in class but not in class themselves. |
| 898 |
func foldExceptions(class []rune) map[rune]bool { |
| 899 |
// Create map containing class and all fold-equivalent chars. |
| 900 |
m := make(map[rune]bool) |
| 901 |
for _, r := range class { |
| 902 |
c := &chars[r] |
| 903 |
if c.caseOrbit == 0 { |
| 904 |
// Just upper and lower. |
| 905 |
if u := c.upperCase; u != 0 { |
| 906 |
m[u] = true |
| 907 |
} |
| 908 |
if l := c.lowerCase; l != 0 { |
| 909 |
m[l] = true |
| 910 |
} |
| 911 |
m[r] = true |
| 912 |
continue |
| 913 |
} |
| 914 |
// Otherwise walk orbit. |
| 915 |
r0 := r |
| 916 |
for { |
| 917 |
m[r] = true |
| 918 |
r = chars[r].caseOrbit |
| 919 |
if r == r0 { |
| 920 |
break |
| 921 |
} |
| 922 |
} |
| 923 |
} |
| 924 |
|
| 925 |
// Remove class itself. |
| 926 |
for _, r := range class { |
| 927 |
delete(m, r) |
| 928 |
} |
| 929 |
|
| 930 |
// What's left is the exceptions. |
| 931 |
return m |
| 932 |
} |
| 933 |
|
| 934 |
var comment = map[string]string{ |
| 935 |
"FoldCategory": "// FoldCategory maps a category name to a table of\n" + |
| 936 |
"// code points outside the category that are equivalent under\n" + |
| 937 |
"// simple case folding to code points inside the category.\n" + |
| 938 |
"// If there is no entry for a category name, there are no such points.\n", |
| 939 |
|
| 940 |
"FoldScript": "// FoldScript maps a script name to a table of\n" + |
| 941 |
"// code points outside the script that are equivalent under\n" + |
| 942 |
"// simple case folding to code points inside the script.\n" + |
| 943 |
"// If there is no entry for a script name, there are no such points.\n", |
| 944 |
} |
| 945 |
|
| 946 |
func printAsciiFold() { |
| 947 |
printf("var asciiFold = [MaxASCII + 1]uint16{\n") |
| 948 |
for i := rune(0); i <= unicode.MaxASCII; i++ { |
| 949 |
c := chars[i] |
| 950 |
f := c.caseOrbit |
| 951 |
if f == 0 { |
| 952 |
if c.lowerCase != i && c.lowerCase != 0 { |
| 953 |
f = c.lowerCase |
| 954 |
} else if c.upperCase != i && c.upperCase != 0 { |
| 955 |
f = c.upperCase |
| 956 |
} else { |
| 957 |
f = i |
| 958 |
} |
| 959 |
} |
| 960 |
printf("\t0x%04X,\n", f) |
| 961 |
} |
| 962 |
printf("}\n\n") |
| 963 |
} |
| 964 |
|
| 965 |
func printCaseOrbit() { |
| 966 |
if *test { |
| 967 |
for j := range chars { |
| 968 |
i := rune(j) |
| 969 |
c := &chars[i] |
| 970 |
f := c.caseOrbit |
| 971 |
if f == 0 { |
| 972 |
if c.lowerCase != i && c.lowerCase != 0 { |
| 973 |
f = c.lowerCase |
| 974 |
} else if c.upperCase != i && c.upperCase != 0 { |
| 975 |
f = c.upperCase |
| 976 |
} else { |
| 977 |
f = i |
| 978 |
} |
| 979 |
} |
| 980 |
if g := unicode.SimpleFold(i); g != f { |
| 981 |
fmt.Fprintf(os.Stderr, "unicode.SimpleFold(%#U) = %#U, want %#U\n", i, g, f) |
| 982 |
} |
| 983 |
} |
| 984 |
return |
| 985 |
} |
| 986 |
|
| 987 |
printf("var caseOrbit = []foldPair{\n") |
| 988 |
for i := range chars { |
| 989 |
c := &chars[i] |
| 990 |
if c.caseOrbit != 0 { |
| 991 |
printf("\t{0x%04X, 0x%04X},\n", i, c.caseOrbit) |
| 992 |
foldPairCount++ |
| 993 |
} |
| 994 |
} |
| 995 |
printf("}\n\n") |
| 996 |
} |
| 997 |
|
| 998 |
func printCatFold(name string, m map[string]map[rune]bool) { |
| 999 |
if *test { |
| 1000 |
var pkgMap map[string]*unicode.RangeTable |
| 1001 |
if name == "FoldCategory" { |
| 1002 |
pkgMap = unicode.FoldCategory |
| 1003 |
} else { |
| 1004 |
pkgMap = unicode.FoldScript |
| 1005 |
} |
| 1006 |
if len(pkgMap) != len(m) { |
| 1007 |
fmt.Fprintf(os.Stderr, "unicode.%s has %d elements, want %d\n", name, len(pkgMap), len(m)) |
| 1008 |
return |
| 1009 |
} |
| 1010 |
for k, v := range m { |
| 1011 |
t, ok := pkgMap[k] |
| 1012 |
if !ok { |
| 1013 |
fmt.Fprintf(os.Stderr, "unicode.%s[%q] missing\n", name, k) |
| 1014 |
continue |
| 1015 |
} |
| 1016 |
n := 0 |
| 1017 |
for _, r := range t.R16 { |
| 1018 |
for c := rune(r.Lo); c <= rune(r.Hi); c += rune(r.Stride) { |
| 1019 |
if !v[c] { |
| 1020 |
fmt.Fprintf(os.Stderr, "unicode.%s[%q] contains %#U, should not\n", name, k, c) |
| 1021 |
} |
| 1022 |
n++ |
| 1023 |
} |
| 1024 |
} |
| 1025 |
for _, r := range t.R32 { |
| 1026 |
for c := rune(r.Lo); c <= rune(r.Hi); c += rune(r.Stride) { |
| 1027 |
if !v[c] { |
| 1028 |
fmt.Fprintf(os.Stderr, "unicode.%s[%q] contains %#U, should not\n", name, k, c) |
| 1029 |
} |
| 1030 |
n++ |
| 1031 |
} |
| 1032 |
} |
| 1033 |
if n != len(v) { |
| 1034 |
fmt.Fprintf(os.Stderr, "unicode.%s[%q] has %d code points, want %d\n", name, k, n, len(v)) |
| 1035 |
} |
| 1036 |
} |
| 1037 |
return |
| 1038 |
} |
| 1039 |
|
| 1040 |
print(comment[name]) |
| 1041 |
printf("var %s = map[string]*RangeTable{\n", name) |
| 1042 |
for _, name := range allCatFold(m) { |
| 1043 |
printf("\t%q: fold%s,\n", name, name) |
| 1044 |
} |
| 1045 |
printf("}\n\n") |
| 1046 |
for _, name := range allCatFold(m) { |
| 1047 |
class := m[name] |
| 1048 |
dumpRange("fold"+name, func(code rune) bool { return class[code] }) |
| 1049 |
} |
| 1050 |
} |
| 1051 |
|
| 1052 |
var range16Count = 0 // Number of entries in the 16-bit range tables. |
| 1053 |
var range32Count = 0 // Number of entries in the 32-bit range tables. |
| 1054 |
var foldPairCount = 0 // Number of fold pairs in the exception tables. |
| 1055 |
|
| 1056 |
func printSizes() { |
| 1057 |
if *test { |
| 1058 |
return |
| 1059 |
} |
| 1060 |
println() |
| 1061 |
printf("// Range entries: %d 16-bit, %d 32-bit, %d total.\n", range16Count, range32Count, range16Count+range32Count) |
| 1062 |
range16Bytes := range16Count * 3 * 2 |
| 1063 |
range32Bytes := range32Count * 3 * 4 |
| 1064 |
printf("// Range bytes: %d 16-bit, %d 32-bit, %d total.\n", range16Bytes, range32Bytes, range16Bytes+range32Bytes) |
| 1065 |
println() |
| 1066 |
printf("// Fold orbit bytes: %d pairs, %d bytes\n", foldPairCount, foldPairCount*2*2) |
| 1067 |
} |