| 1 |
// Copyright 2016 The Go Authors. All rights reserved. |
| 2 |
// Use of this source code is governed by a BSD-style |
| 3 |
// license that can be found in the LICENSE file. |
| 4 |
|
| 5 |
//go:build icu |
| 6 |
|
| 7 |
package cases |
| 8 |
|
| 9 |
import ( |
| 10 |
"path" |
| 11 |
"strings" |
| 12 |
"testing" |
| 13 |
|
| 14 |
"golang.org/x/text/language" |
| 15 |
"golang.org/x/text/unicode/norm" |
| 16 |
) |
| 17 |
|
| 18 |
func TestICUConformance(t *testing.T) { |
| 19 |
// Build test set. |
| 20 |
input := []string{ |
| 21 |
"a.a a_a", |
| 22 |
"a\u05d0a", |
| 23 |
"\u05d0'a", |
| 24 |
"a\u03084a", |
| 25 |
"a\u0308a", |
| 26 |
"a3\u30a3a", |
| 27 |
"a\u303aa", |
| 28 |
"a_\u303a_a", |
| 29 |
"1_a..a", |
| 30 |
"1_a.a", |
| 31 |
"a..a.", |
| 32 |
"a--a-", |
| 33 |
"a-a-", |
| 34 |
"a\u200ba", |
| 35 |
"a\u200b\u200ba", |
| 36 |
"a\u00ad\u00ada", // Format |
| 37 |
"a\u00ada", |
| 38 |
"a''a", // SingleQuote |
| 39 |
"a'a", |
| 40 |
"a::a", // MidLetter |
| 41 |
"a:a", |
| 42 |
"a..a", // MidNumLet |
| 43 |
"a.a", |
| 44 |
"a;;a", // MidNum |
| 45 |
"a;a", |
| 46 |
"a__a", // ExtendNumlet |
| 47 |
"a_a", |
| 48 |
"ΟΣ''a", |
| 49 |
} |
| 50 |
add := func(x interface{}) { |
| 51 |
switch v := x.(type) { |
| 52 |
case string: |
| 53 |
input = append(input, v) |
| 54 |
case []string: |
| 55 |
for _, s := range v { |
| 56 |
input = append(input, s) |
| 57 |
} |
| 58 |
} |
| 59 |
} |
| 60 |
for _, tc := range testCases { |
| 61 |
add(tc.src) |
| 62 |
add(tc.lower) |
| 63 |
add(tc.upper) |
| 64 |
add(tc.title) |
| 65 |
} |
| 66 |
for _, tc := range bufferTests { |
| 67 |
add(tc.src) |
| 68 |
} |
| 69 |
for _, tc := range breakTest { |
| 70 |
add(strings.Replace(tc, "|", "", -1)) |
| 71 |
} |
| 72 |
for _, tc := range foldTestCases { |
| 73 |
add(tc) |
| 74 |
} |
| 75 |
|
| 76 |
// Compare ICU to Go. |
| 77 |
for _, c := range []string{"lower", "upper", "title", "fold"} { |
| 78 |
for _, tag := range []string{ |
| 79 |
"und", "af", "az", "el", "lt", "nl", "tr", |
| 80 |
} { |
| 81 |
for _, s := range input { |
| 82 |
if exclude(c, tag, s) { |
| 83 |
continue |
| 84 |
} |
| 85 |
t.Run(path.Join(c, tag, s), func(t *testing.T) { |
| 86 |
want := doICU(tag, c, s) |
| 87 |
got := doGo(tag, c, s) |
| 88 |
if norm.NFC.String(got) != norm.NFC.String(want) { |
| 89 |
t.Errorf("\n in %[3]q (%+[3]q)\n got %[1]q (%+[1]q)\n want %[2]q (%+[2]q)", got, want, s) |
| 90 |
} |
| 91 |
}) |
| 92 |
} |
| 93 |
} |
| 94 |
} |
| 95 |
} |
| 96 |
|
| 97 |
// exclude indicates if a string should be excluded from testing. |
| 98 |
func exclude(cm, tag, s string) bool { |
| 99 |
list := []struct{ cm, tags, pattern string }{ |
| 100 |
// TODO: Go does not handle certain esoteric breaks correctly. This will be |
| 101 |
// fixed once we have a real word break iterator. Alternatively, it |
| 102 |
// seems like we're not too far off from making it work, so we could |
| 103 |
// fix these last steps. But first verify that using a separate word |
| 104 |
// breaker does not hurt performance. |
| 105 |
{"title", "af nl", "a''a"}, |
| 106 |
{"", "", "א'a"}, |
| 107 |
|
| 108 |
// All the exclusions below seem to be issues with the ICU |
| 109 |
// implementation (at version 57) and thus are not marked as TODO. |
| 110 |
|
| 111 |
// ICU does not handle leading apostrophe for Dutch and |
| 112 |
// Afrikaans correctly. See https://unicode.org/cldr/trac/ticket/7078. |
| 113 |
{"title", "af nl", "'n"}, |
| 114 |
{"title", "af nl", "'N"}, |
| 115 |
|
| 116 |
// Go terminates the final sigma check after a fixed number of |
| 117 |
// ignorables have been found. This ensures that the algorithm can make |
| 118 |
// progress in a streaming scenario. |
| 119 |
{"lower title", "", "\u039f\u03a3...............................a"}, |
| 120 |
// This also applies to upper in Greek. |
| 121 |
// NOTE: we could fix the following two cases by adding state to elUpper |
| 122 |
// and aztrLower. However, considering a modifier to not belong to the |
| 123 |
// preceding letter after the maximum modifiers count is reached is |
| 124 |
// consistent with the behavior of unicode/norm. |
| 125 |
{"upper", "el", "\u03bf" + strings.Repeat("\u0321", 29) + "\u0313"}, |
| 126 |
{"lower", "az tr lt", "I" + strings.Repeat("\u0321", 30) + "\u0307\u0300"}, |
| 127 |
{"upper", "lt", "i" + strings.Repeat("\u0321", 30) + "\u0307\u0300"}, |
| 128 |
{"lower", "lt", "I" + strings.Repeat("\u0321", 30) + "\u0300"}, |
| 129 |
|
| 130 |
// ICU title case seems to erroneously removes \u0307 from an upper case |
| 131 |
// I unconditionally, instead of only when lowercasing. The ICU |
| 132 |
// transform algorithm transforms these cases consistently with our |
| 133 |
// implementation. |
| 134 |
{"title", "az tr", "\u0307"}, |
| 135 |
|
| 136 |
// The spec says to remove \u0307 after Soft-Dotted characters. ICU |
| 137 |
// transforms conform but ucasemap_utf8ToUpper does not. |
| 138 |
{"upper title", "lt", "i\u0307"}, |
| 139 |
{"upper title", "lt", "i" + strings.Repeat("\u0321", 29) + "\u0307\u0300"}, |
| 140 |
|
| 141 |
// Both Unicode and CLDR prescribe an extra explicit dot above after a |
| 142 |
// Soft_Dotted character if there are other modifiers. |
| 143 |
// ucasemap_utf8ToUpper does not do this; ICU transforms do. |
| 144 |
// The issue with ucasemap_utf8ToUpper seems to be that it does not |
| 145 |
// consider the modifiers that are part of composition in the evaluation |
| 146 |
// of More_Above. For instance, according to the More_Above rule for lt, |
| 147 |
// a dotted capital I (U+0130) becomes i\u0307\u0307 (an small i with |
| 148 |
// two additional dots). This seems odd, but is correct. ICU is |
| 149 |
// definitely not correct as it produces different results for different |
| 150 |
// normal forms. For instance, for an İ: |
| 151 |
// \u0130 (NFC) -> i\u0307 (incorrect) |
| 152 |
// I\u0307 (NFD) -> i\u0307\u0307 (correct) |
| 153 |
// We could argue that we should not add a \u0307 if there already is |
| 154 |
// one, but this may be hard to get correct and is not conform the |
| 155 |
// standard. |
| 156 |
{"lower title", "lt", "\u0130"}, |
| 157 |
{"lower title", "lt", "\u00cf"}, |
| 158 |
|
| 159 |
// We are conform ICU ucasemap_utf8ToUpper if we remove support for |
| 160 |
// elUpper. However, this is clearly not conform the spec. Moreover, the |
| 161 |
// ICU transforms _do_ implement this transform and produces results |
| 162 |
// consistent with our implementation. Note that we still prefer to use |
| 163 |
// ucasemap_utf8ToUpper instead of transforms as the latter have |
| 164 |
// inconsistencies in the word breaking algorithm. |
| 165 |
{"upper", "el", "\u0386"}, // GREEK CAPITAL LETTER ALPHA WITH TONOS |
| 166 |
{"upper", "el", "\u0389"}, // GREEK CAPITAL LETTER ETA WITH TONOS |
| 167 |
{"upper", "el", "\u038A"}, // GREEK CAPITAL LETTER IOTA WITH TONOS |
| 168 |
|
| 169 |
{"upper", "el", "\u0391"}, // GREEK CAPITAL LETTER ALPHA |
| 170 |
{"upper", "el", "\u0397"}, // GREEK CAPITAL LETTER ETA |
| 171 |
{"upper", "el", "\u0399"}, // GREEK CAPITAL LETTER IOTA |
| 172 |
|
| 173 |
{"upper", "el", "\u03AC"}, // GREEK SMALL LETTER ALPHA WITH TONOS |
| 174 |
{"upper", "el", "\u03AE"}, // GREEK SMALL LETTER ALPHA WITH ETA |
| 175 |
{"upper", "el", "\u03AF"}, // GREEK SMALL LETTER ALPHA WITH IOTA |
| 176 |
|
| 177 |
{"upper", "el", "\u03B1"}, // GREEK SMALL LETTER ALPHA |
| 178 |
{"upper", "el", "\u03B7"}, // GREEK SMALL LETTER ETA |
| 179 |
{"upper", "el", "\u03B9"}, // GREEK SMALL LETTER IOTA |
| 180 |
} |
| 181 |
for _, x := range list { |
| 182 |
if x.cm != "" && strings.Index(x.cm, cm) == -1 { |
| 183 |
continue |
| 184 |
} |
| 185 |
if x.tags != "" && strings.Index(x.tags, tag) == -1 { |
| 186 |
continue |
| 187 |
} |
| 188 |
if strings.Index(s, x.pattern) != -1 { |
| 189 |
return true |
| 190 |
} |
| 191 |
} |
| 192 |
return false |
| 193 |
} |
| 194 |
|
| 195 |
func doGo(tag, caser, input string) string { |
| 196 |
var c Caser |
| 197 |
t := language.MustParse(tag) |
| 198 |
switch caser { |
| 199 |
case "lower": |
| 200 |
c = Lower(t) |
| 201 |
case "upper": |
| 202 |
c = Upper(t) |
| 203 |
case "title": |
| 204 |
c = Title(t) |
| 205 |
case "fold": |
| 206 |
c = Fold() |
| 207 |
} |
| 208 |
return c.String(input) |
| 209 |
} |