| 1 |
// Copyright 2013 The Go Authors. All rights reserved. |
| 2 |
// Use of this source code is governed by a BSD-style |
| 3 |
// license that can be found in the LICENSE file. |
| 4 |
|
| 5 |
//go:build ignore |
| 6 |
|
| 7 |
package main |
| 8 |
|
| 9 |
// This program generates tables.go: |
| 10 |
// go run maketables.go | gofmt > tables.go |
| 11 |
|
| 12 |
import ( |
| 13 |
"bufio" |
| 14 |
"fmt" |
| 15 |
"log" |
| 16 |
"net/http" |
| 17 |
"sort" |
| 18 |
"strings" |
| 19 |
) |
| 20 |
|
| 21 |
func main() { |
| 22 |
fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") |
| 23 |
fmt.Printf("// Package traditionalchinese provides Traditional Chinese encodings such as Big5.\n") |
| 24 |
fmt.Printf(`package traditionalchinese // import "golang.org/x/text/encoding/traditionalchinese"` + "\n\n") |
| 25 |
|
| 26 |
res, err := http.Get("http://encoding.spec.whatwg.org/index-big5.txt") |
| 27 |
if err != nil { |
| 28 |
log.Fatalf("Get: %v", err) |
| 29 |
} |
| 30 |
defer res.Body.Close() |
| 31 |
|
| 32 |
mapping := [65536]uint32{} |
| 33 |
reverse := [65536 * 4]uint16{} |
| 34 |
|
| 35 |
scanner := bufio.NewScanner(res.Body) |
| 36 |
for scanner.Scan() { |
| 37 |
s := strings.TrimSpace(scanner.Text()) |
| 38 |
if s == "" || s[0] == '#' { |
| 39 |
continue |
| 40 |
} |
| 41 |
x, y := uint16(0), uint32(0) |
| 42 |
if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { |
| 43 |
log.Fatalf("could not parse %q", s) |
| 44 |
} |
| 45 |
if x < 0 || 126*157 <= x { |
| 46 |
log.Fatalf("Big5 code %d is out of range", x) |
| 47 |
} |
| 48 |
mapping[x] = y |
| 49 |
|
| 50 |
// The WHATWG spec http://encoding.spec.whatwg.org/#indexes says that |
| 51 |
// "The index pointer for code point in index is the first pointer |
| 52 |
// corresponding to code point in index", which would normally mean |
| 53 |
// that the code below should be guarded by "if reverse[y] == 0", but |
| 54 |
// last instead of first seems to match the behavior of |
| 55 |
// "iconv -f UTF-8 -t BIG5". For example, U+8005 者 occurs twice in |
| 56 |
// http://encoding.spec.whatwg.org/index-big5.txt, as index 2148 |
| 57 |
// (encoded as "\x8e\xcd") and index 6543 (encoded as "\xaa\xcc") |
| 58 |
// and "echo 者 | iconv -f UTF-8 -t BIG5 | xxd" gives "\xaa\xcc". |
| 59 |
c0, c1 := x/157, x%157 |
| 60 |
if c1 < 0x3f { |
| 61 |
c1 += 0x40 |
| 62 |
} else { |
| 63 |
c1 += 0x62 |
| 64 |
} |
| 65 |
reverse[y] = (0x81+c0)<<8 | c1 |
| 66 |
} |
| 67 |
if err := scanner.Err(); err != nil { |
| 68 |
log.Fatalf("scanner error: %v", err) |
| 69 |
} |
| 70 |
|
| 71 |
fmt.Printf("// decode is the decoding table from Big5 code to Unicode.\n") |
| 72 |
fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-big5.txt\n") |
| 73 |
fmt.Printf("var decode = [...]uint32{\n") |
| 74 |
for i, v := range mapping { |
| 75 |
if v != 0 { |
| 76 |
fmt.Printf("\t%d: 0x%08X,\n", i, v) |
| 77 |
} |
| 78 |
} |
| 79 |
fmt.Printf("}\n\n") |
| 80 |
|
| 81 |
// Any run of at least separation continuous zero entries in the reverse map will |
| 82 |
// be a separate encode table. |
| 83 |
const separation = 1024 |
| 84 |
|
| 85 |
intervals := []interval(nil) |
| 86 |
low, high := -1, -1 |
| 87 |
for i, v := range reverse { |
| 88 |
if v == 0 { |
| 89 |
continue |
| 90 |
} |
| 91 |
if low < 0 { |
| 92 |
low = i |
| 93 |
} else if i-high >= separation { |
| 94 |
if high >= 0 { |
| 95 |
intervals = append(intervals, interval{low, high}) |
| 96 |
} |
| 97 |
low = i |
| 98 |
} |
| 99 |
high = i + 1 |
| 100 |
} |
| 101 |
if high >= 0 { |
| 102 |
intervals = append(intervals, interval{low, high}) |
| 103 |
} |
| 104 |
sort.Sort(byDecreasingLength(intervals)) |
| 105 |
|
| 106 |
fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) |
| 107 |
fmt.Printf("// encodeX are the encoding tables from Unicode to Big5 code,\n") |
| 108 |
fmt.Printf("// sorted by decreasing length.\n") |
| 109 |
for i, v := range intervals { |
| 110 |
fmt.Printf("// encode%d: %5d entries for runes in [%6d, %6d).\n", i, v.len(), v.low, v.high) |
| 111 |
} |
| 112 |
fmt.Printf("\n") |
| 113 |
|
| 114 |
for i, v := range intervals { |
| 115 |
fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) |
| 116 |
fmt.Printf("var encode%d = [...]uint16{\n", i) |
| 117 |
for j := v.low; j < v.high; j++ { |
| 118 |
x := reverse[j] |
| 119 |
if x == 0 { |
| 120 |
continue |
| 121 |
} |
| 122 |
fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) |
| 123 |
} |
| 124 |
fmt.Printf("}\n\n") |
| 125 |
} |
| 126 |
} |
| 127 |
|
| 128 |
// interval is a half-open interval [low, high). |
| 129 |
type interval struct { |
| 130 |
low, high int |
| 131 |
} |
| 132 |
|
| 133 |
func (i interval) len() int { return i.high - i.low } |
| 134 |
|
| 135 |
// byDecreasingLength sorts intervals by decreasing length. |
| 136 |
type byDecreasingLength []interval |
| 137 |
|
| 138 |
func (b byDecreasingLength) Len() int { return len(b) } |
| 139 |
func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } |
| 140 |
func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } |