Debian Codegraph @

cases/icu_test.go

raw
1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build icu
6
7 package cases
8
9 import (
10 "path"
11 "strings"
12 "testing"
13
14 "golang.org/x/text/language"
15 "golang.org/x/text/unicode/norm"
16 )
17
18 func TestICUConformance(t *testing.T) {
19 // Build test set.
20 input := []string{
21 "a.a a_a",
22 "a\u05d0a",
23 "\u05d0'a",
24 "a\u03084a",
25 "a\u0308a",
26 "a3\u30a3a",
27 "a\u303aa",
28 "a_\u303a_a",
29 "1_a..a",
30 "1_a.a",
31 "a..a.",
32 "a--a-",
33 "a-a-",
34 "a\u200ba",
35 "a\u200b\u200ba",
36 "a\u00ad\u00ada", // Format
37 "a\u00ada",
38 "a''a", // SingleQuote
39 "a'a",
40 "a::a", // MidLetter
41 "a:a",
42 "a..a", // MidNumLet
43 "a.a",
44 "a;;a", // MidNum
45 "a;a",
46 "a__a", // ExtendNumlet
47 "a_a",
48 "ΟΣ''a",
49 }
50 add := func(x interface{}) {
51 switch v := x.(type) {
52 case string:
53 input = append(input, v)
54 case []string:
55 for _, s := range v {
56 input = append(input, s)
57 }
58 }
59 }
60 for _, tc := range testCases {
61 add(tc.src)
62 add(tc.lower)
63 add(tc.upper)
64 add(tc.title)
65 }
66 for _, tc := range bufferTests {
67 add(tc.src)
68 }
69 for _, tc := range breakTest {
70 add(strings.Replace(tc, "|", "", -1))
71 }
72 for _, tc := range foldTestCases {
73 add(tc)
74 }
75
76 // Compare ICU to Go.
77 for _, c := range []string{"lower", "upper", "title", "fold"} {
78 for _, tag := range []string{
79 "und", "af", "az", "el", "lt", "nl", "tr",
80 } {
81 for _, s := range input {
82 if exclude(c, tag, s) {
83 continue
84 }
85 t.Run(path.Join(c, tag, s), func(t *testing.T) {
86 want := doICU(tag, c, s)
87 got := doGo(tag, c, s)
88 if norm.NFC.String(got) != norm.NFC.String(want) {
89 t.Errorf("\n in %[3]q (%+[3]q)\n got %[1]q (%+[1]q)\n want %[2]q (%+[2]q)", got, want, s)
90 }
91 })
92 }
93 }
94 }
95 }
96
97 // exclude indicates if a string should be excluded from testing.
98 func exclude(cm, tag, s string) bool {
99 list := []struct{ cm, tags, pattern string }{
100 // TODO: Go does not handle certain esoteric breaks correctly. This will be
101 // fixed once we have a real word break iterator. Alternatively, it
102 // seems like we're not too far off from making it work, so we could
103 // fix these last steps. But first verify that using a separate word
104 // breaker does not hurt performance.
105 {"title", "af nl", "a''a"},
106 {"", "", "א'a"},
107
108 // All the exclusions below seem to be issues with the ICU
109 // implementation (at version 57) and thus are not marked as TODO.
110
111 // ICU does not handle leading apostrophe for Dutch and
112 // Afrikaans correctly. See https://unicode.org/cldr/trac/ticket/7078.
113 {"title", "af nl", "'n"},
114 {"title", "af nl", "'N"},
115
116 // Go terminates the final sigma check after a fixed number of
117 // ignorables have been found. This ensures that the algorithm can make
118 // progress in a streaming scenario.
119 {"lower title", "", "\u039f\u03a3...............................a"},
120 // This also applies to upper in Greek.
121 // NOTE: we could fix the following two cases by adding state to elUpper
122 // and aztrLower. However, considering a modifier to not belong to the
123 // preceding letter after the maximum modifiers count is reached is
124 // consistent with the behavior of unicode/norm.
125 {"upper", "el", "\u03bf" + strings.Repeat("\u0321", 29) + "\u0313"},
126 {"lower", "az tr lt", "I" + strings.Repeat("\u0321", 30) + "\u0307\u0300"},
127 {"upper", "lt", "i" + strings.Repeat("\u0321", 30) + "\u0307\u0300"},
128 {"lower", "lt", "I" + strings.Repeat("\u0321", 30) + "\u0300"},
129
130 // ICU title case seems to erroneously removes \u0307 from an upper case
131 // I unconditionally, instead of only when lowercasing. The ICU
132 // transform algorithm transforms these cases consistently with our
133 // implementation.
134 {"title", "az tr", "\u0307"},
135
136 // The spec says to remove \u0307 after Soft-Dotted characters. ICU
137 // transforms conform but ucasemap_utf8ToUpper does not.
138 {"upper title", "lt", "i\u0307"},
139 {"upper title", "lt", "i" + strings.Repeat("\u0321", 29) + "\u0307\u0300"},
140
141 // Both Unicode and CLDR prescribe an extra explicit dot above after a
142 // Soft_Dotted character if there are other modifiers.
143 // ucasemap_utf8ToUpper does not do this; ICU transforms do.
144 // The issue with ucasemap_utf8ToUpper seems to be that it does not
145 // consider the modifiers that are part of composition in the evaluation
146 // of More_Above. For instance, according to the More_Above rule for lt,
147 // a dotted capital I (U+0130) becomes i\u0307\u0307 (an small i with
148 // two additional dots). This seems odd, but is correct. ICU is
149 // definitely not correct as it produces different results for different
150 // normal forms. For instance, for an İ:
151 // \u0130 (NFC) -> i\u0307 (incorrect)
152 // I\u0307 (NFD) -> i\u0307\u0307 (correct)
153 // We could argue that we should not add a \u0307 if there already is
154 // one, but this may be hard to get correct and is not conform the
155 // standard.
156 {"lower title", "lt", "\u0130"},
157 {"lower title", "lt", "\u00cf"},
158
159 // We are conform ICU ucasemap_utf8ToUpper if we remove support for
160 // elUpper. However, this is clearly not conform the spec. Moreover, the
161 // ICU transforms _do_ implement this transform and produces results
162 // consistent with our implementation. Note that we still prefer to use
163 // ucasemap_utf8ToUpper instead of transforms as the latter have
164 // inconsistencies in the word breaking algorithm.
165 {"upper", "el", "\u0386"}, // GREEK CAPITAL LETTER ALPHA WITH TONOS
166 {"upper", "el", "\u0389"}, // GREEK CAPITAL LETTER ETA WITH TONOS
167 {"upper", "el", "\u038A"}, // GREEK CAPITAL LETTER IOTA WITH TONOS
168
169 {"upper", "el", "\u0391"}, // GREEK CAPITAL LETTER ALPHA
170 {"upper", "el", "\u0397"}, // GREEK CAPITAL LETTER ETA
171 {"upper", "el", "\u0399"}, // GREEK CAPITAL LETTER IOTA
172
173 {"upper", "el", "\u03AC"}, // GREEK SMALL LETTER ALPHA WITH TONOS
174 {"upper", "el", "\u03AE"}, // GREEK SMALL LETTER ALPHA WITH ETA
175 {"upper", "el", "\u03AF"}, // GREEK SMALL LETTER ALPHA WITH IOTA
176
177 {"upper", "el", "\u03B1"}, // GREEK SMALL LETTER ALPHA
178 {"upper", "el", "\u03B7"}, // GREEK SMALL LETTER ETA
179 {"upper", "el", "\u03B9"}, // GREEK SMALL LETTER IOTA
180 }
181 for _, x := range list {
182 if x.cm != "" && strings.Index(x.cm, cm) == -1 {
183 continue
184 }
185 if x.tags != "" && strings.Index(x.tags, tag) == -1 {
186 continue
187 }
188 if strings.Index(s, x.pattern) != -1 {
189 return true
190 }
191 }
192 return false
193 }
194
195 func doGo(tag, caser, input string) string {
196 var c Caser
197 t := language.MustParse(tag)
198 switch caser {
199 case "lower":
200 c = Lower(t)
201 case "upper":
202 c = Upper(t)
203 case "title":
204 c = Title(t)
205 case "fold":
206 c = Fold()
207 }
208 return c.String(input)
209 }