| 1 |
// Tencent is pleased to support the open source community by making RapidJSON available. |
| 2 |
// |
| 3 |
// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. |
| 4 |
// |
| 5 |
// Licensed under the MIT License (the "License"); you may not use this file except |
| 6 |
// in compliance with the License. You may obtain a copy of the License at |
| 7 |
// |
| 8 |
// http://opensource.org/licenses/MIT |
| 9 |
// |
| 10 |
// Unless required by applicable law or agreed to in writing, software distributed |
| 11 |
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR |
| 12 |
// CONDITIONS OF ANY KIND, either express or implied. See the License for the |
| 13 |
// specific language governing permissions and limitations under the License. |
| 14 |
|
| 15 |
#include "unittest.h" |
| 16 |
#include "rapidjson/filereadstream.h" |
| 17 |
#include "rapidjson/filewritestream.h" |
| 18 |
#include "rapidjson/encodedstream.h" |
| 19 |
#include "rapidjson/stringbuffer.h" |
| 20 |
|
| 21 |
using namespace rapidjson; |
| 22 |
|
| 23 |
// Verification of encoders/decoders with Hoehrmann's UTF8 decoder |
| 24 |
|
| 25 |
// http://www.unicode.org/Public/UNIDATA/Blocks.txt |
| 26 |
static const unsigned kCodepointRanges[] = { |
| 27 |
0x0000, 0x007F, // Basic Latin |
| 28 |
0x0080, 0x00FF, // Latin-1 Supplement |
| 29 |
0x0100, 0x017F, // Latin Extended-A |
| 30 |
0x0180, 0x024F, // Latin Extended-B |
| 31 |
0x0250, 0x02AF, // IPA Extensions |
| 32 |
0x02B0, 0x02FF, // Spacing Modifier Letters |
| 33 |
0x0300, 0x036F, // Combining Diacritical Marks |
| 34 |
0x0370, 0x03FF, // Greek and Coptic |
| 35 |
0x0400, 0x04FF, // Cyrillic |
| 36 |
0x0500, 0x052F, // Cyrillic Supplement |
| 37 |
0x0530, 0x058F, // Armenian |
| 38 |
0x0590, 0x05FF, // Hebrew |
| 39 |
0x0600, 0x06FF, // Arabic |
| 40 |
0x0700, 0x074F, // Syriac |
| 41 |
0x0750, 0x077F, // Arabic Supplement |
| 42 |
0x0780, 0x07BF, // Thaana |
| 43 |
0x07C0, 0x07FF, // NKo |
| 44 |
0x0800, 0x083F, // Samaritan |
| 45 |
0x0840, 0x085F, // Mandaic |
| 46 |
0x0900, 0x097F, // Devanagari |
| 47 |
0x0980, 0x09FF, // Bengali |
| 48 |
0x0A00, 0x0A7F, // Gurmukhi |
| 49 |
0x0A80, 0x0AFF, // Gujarati |
| 50 |
0x0B00, 0x0B7F, // Oriya |
| 51 |
0x0B80, 0x0BFF, // Tamil |
| 52 |
0x0C00, 0x0C7F, // Telugu |
| 53 |
0x0C80, 0x0CFF, // Kannada |
| 54 |
0x0D00, 0x0D7F, // Malayalam |
| 55 |
0x0D80, 0x0DFF, // Sinhala |
| 56 |
0x0E00, 0x0E7F, // Thai |
| 57 |
0x0E80, 0x0EFF, // Lao |
| 58 |
0x0F00, 0x0FFF, // Tibetan |
| 59 |
0x1000, 0x109F, // Myanmar |
| 60 |
0x10A0, 0x10FF, // Georgian |
| 61 |
0x1100, 0x11FF, // Hangul Jamo |
| 62 |
0x1200, 0x137F, // Ethiopic |
| 63 |
0x1380, 0x139F, // Ethiopic Supplement |
| 64 |
0x13A0, 0x13FF, // Cherokee |
| 65 |
0x1400, 0x167F, // Unified Canadian Aboriginal Syllabics |
| 66 |
0x1680, 0x169F, // Ogham |
| 67 |
0x16A0, 0x16FF, // Runic |
| 68 |
0x1700, 0x171F, // Tagalog |
| 69 |
0x1720, 0x173F, // Hanunoo |
| 70 |
0x1740, 0x175F, // Buhid |
| 71 |
0x1760, 0x177F, // Tagbanwa |
| 72 |
0x1780, 0x17FF, // Khmer |
| 73 |
0x1800, 0x18AF, // Mongolian |
| 74 |
0x18B0, 0x18FF, // Unified Canadian Aboriginal Syllabics Extended |
| 75 |
0x1900, 0x194F, // Limbu |
| 76 |
0x1950, 0x197F, // Tai Le |
| 77 |
0x1980, 0x19DF, // New Tai Lue |
| 78 |
0x19E0, 0x19FF, // Khmer Symbols |
| 79 |
0x1A00, 0x1A1F, // Buginese |
| 80 |
0x1A20, 0x1AAF, // Tai Tham |
| 81 |
0x1B00, 0x1B7F, // Balinese |
| 82 |
0x1B80, 0x1BBF, // Sundanese |
| 83 |
0x1BC0, 0x1BFF, // Batak |
| 84 |
0x1C00, 0x1C4F, // Lepcha |
| 85 |
0x1C50, 0x1C7F, // Ol Chiki |
| 86 |
0x1CD0, 0x1CFF, // Vedic Extensions |
| 87 |
0x1D00, 0x1D7F, // Phonetic Extensions |
| 88 |
0x1D80, 0x1DBF, // Phonetic Extensions Supplement |
| 89 |
0x1DC0, 0x1DFF, // Combining Diacritical Marks Supplement |
| 90 |
0x1E00, 0x1EFF, // Latin Extended Additional |
| 91 |
0x1F00, 0x1FFF, // Greek Extended |
| 92 |
0x2000, 0x206F, // General Punctuation |
| 93 |
0x2070, 0x209F, // Superscripts and Subscripts |
| 94 |
0x20A0, 0x20CF, // Currency Symbols |
| 95 |
0x20D0, 0x20FF, // Combining Diacritical Marks for Symbols |
| 96 |
0x2100, 0x214F, // Letterlike Symbols |
| 97 |
0x2150, 0x218F, // Number Forms |
| 98 |
0x2190, 0x21FF, // Arrows |
| 99 |
0x2200, 0x22FF, // Mathematical Operators |
| 100 |
0x2300, 0x23FF, // Miscellaneous Technical |
| 101 |
0x2400, 0x243F, // Control Pictures |
| 102 |
0x2440, 0x245F, // Optical Character Recognition |
| 103 |
0x2460, 0x24FF, // Enclosed Alphanumerics |
| 104 |
0x2500, 0x257F, // Box Drawing |
| 105 |
0x2580, 0x259F, // Block Elements |
| 106 |
0x25A0, 0x25FF, // Geometric Shapes |
| 107 |
0x2600, 0x26FF, // Miscellaneous Symbols |
| 108 |
0x2700, 0x27BF, // Dingbats |
| 109 |
0x27C0, 0x27EF, // Miscellaneous Mathematical Symbols-A |
| 110 |
0x27F0, 0x27FF, // Supplemental Arrows-A |
| 111 |
0x2800, 0x28FF, // Braille Patterns |
| 112 |
0x2900, 0x297F, // Supplemental Arrows-B |
| 113 |
0x2980, 0x29FF, // Miscellaneous Mathematical Symbols-B |
| 114 |
0x2A00, 0x2AFF, // Supplemental Mathematical Operators |
| 115 |
0x2B00, 0x2BFF, // Miscellaneous Symbols and Arrows |
| 116 |
0x2C00, 0x2C5F, // Glagolitic |
| 117 |
0x2C60, 0x2C7F, // Latin Extended-C |
| 118 |
0x2C80, 0x2CFF, // Coptic |
| 119 |
0x2D00, 0x2D2F, // Georgian Supplement |
| 120 |
0x2D30, 0x2D7F, // Tifinagh |
| 121 |
0x2D80, 0x2DDF, // Ethiopic Extended |
| 122 |
0x2DE0, 0x2DFF, // Cyrillic Extended-A |
| 123 |
0x2E00, 0x2E7F, // Supplemental Punctuation |
| 124 |
0x2E80, 0x2EFF, // CJK Radicals Supplement |
| 125 |
0x2F00, 0x2FDF, // Kangxi Radicals |
| 126 |
0x2FF0, 0x2FFF, // Ideographic Description Characters |
| 127 |
0x3000, 0x303F, // CJK Symbols and Punctuation |
| 128 |
0x3040, 0x309F, // Hiragana |
| 129 |
0x30A0, 0x30FF, // Katakana |
| 130 |
0x3100, 0x312F, // Bopomofo |
| 131 |
0x3130, 0x318F, // Hangul Compatibility Jamo |
| 132 |
0x3190, 0x319F, // Kanbun |
| 133 |
0x31A0, 0x31BF, // Bopomofo Extended |
| 134 |
0x31C0, 0x31EF, // CJK Strokes |
| 135 |
0x31F0, 0x31FF, // Katakana Phonetic Extensions |
| 136 |
0x3200, 0x32FF, // Enclosed CJK Letters and Months |
| 137 |
0x3300, 0x33FF, // CJK Compatibility |
| 138 |
0x3400, 0x4DBF, // CJK Unified Ideographs Extension A |
| 139 |
0x4DC0, 0x4DFF, // Yijing Hexagram Symbols |
| 140 |
0x4E00, 0x9FFF, // CJK Unified Ideographs |
| 141 |
0xA000, 0xA48F, // Yi Syllables |
| 142 |
0xA490, 0xA4CF, // Yi Radicals |
| 143 |
0xA4D0, 0xA4FF, // Lisu |
| 144 |
0xA500, 0xA63F, // Vai |
| 145 |
0xA640, 0xA69F, // Cyrillic Extended-B |
| 146 |
0xA6A0, 0xA6FF, // Bamum |
| 147 |
0xA700, 0xA71F, // Modifier Tone Letters |
| 148 |
0xA720, 0xA7FF, // Latin Extended-D |
| 149 |
0xA800, 0xA82F, // Syloti Nagri |
| 150 |
0xA830, 0xA83F, // Common Indic Number Forms |
| 151 |
0xA840, 0xA87F, // Phags-pa |
| 152 |
0xA880, 0xA8DF, // Saurashtra |
| 153 |
0xA8E0, 0xA8FF, // Devanagari Extended |
| 154 |
0xA900, 0xA92F, // Kayah Li |
| 155 |
0xA930, 0xA95F, // Rejang |
| 156 |
0xA960, 0xA97F, // Hangul Jamo Extended-A |
| 157 |
0xA980, 0xA9DF, // Javanese |
| 158 |
0xAA00, 0xAA5F, // Cham |
| 159 |
0xAA60, 0xAA7F, // Myanmar Extended-A |
| 160 |
0xAA80, 0xAADF, // Tai Viet |
| 161 |
0xAB00, 0xAB2F, // Ethiopic Extended-A |
| 162 |
0xABC0, 0xABFF, // Meetei Mayek |
| 163 |
0xAC00, 0xD7AF, // Hangul Syllables |
| 164 |
0xD7B0, 0xD7FF, // Hangul Jamo Extended-B |
| 165 |
//0xD800, 0xDB7F, // High Surrogates |
| 166 |
//0xDB80, 0xDBFF, // High Private Use Surrogates |
| 167 |
//0xDC00, 0xDFFF, // Low Surrogates |
| 168 |
0xE000, 0xF8FF, // Private Use Area |
| 169 |
0xF900, 0xFAFF, // CJK Compatibility Ideographs |
| 170 |
0xFB00, 0xFB4F, // Alphabetic Presentation Forms |
| 171 |
0xFB50, 0xFDFF, // Arabic Presentation Forms-A |
| 172 |
0xFE00, 0xFE0F, // Variation Selectors |
| 173 |
0xFE10, 0xFE1F, // Vertical Forms |
| 174 |
0xFE20, 0xFE2F, // Combining Half Marks |
| 175 |
0xFE30, 0xFE4F, // CJK Compatibility Forms |
| 176 |
0xFE50, 0xFE6F, // Small Form Variants |
| 177 |
0xFE70, 0xFEFF, // Arabic Presentation Forms-B |
| 178 |
0xFF00, 0xFFEF, // Halfwidth and Fullwidth Forms |
| 179 |
0xFFF0, 0xFFFF, // Specials |
| 180 |
0x10000, 0x1007F, // Linear B Syllabary |
| 181 |
0x10080, 0x100FF, // Linear B Ideograms |
| 182 |
0x10100, 0x1013F, // Aegean Numbers |
| 183 |
0x10140, 0x1018F, // Ancient Greek Numbers |
| 184 |
0x10190, 0x101CF, // Ancient Symbols |
| 185 |
0x101D0, 0x101FF, // Phaistos Disc |
| 186 |
0x10280, 0x1029F, // Lycian |
| 187 |
0x102A0, 0x102DF, // Carian |
| 188 |
0x10300, 0x1032F, // Old Italic |
| 189 |
0x10330, 0x1034F, // Gothic |
| 190 |
0x10380, 0x1039F, // Ugaritic |
| 191 |
0x103A0, 0x103DF, // Old Persian |
| 192 |
0x10400, 0x1044F, // Deseret |
| 193 |
0x10450, 0x1047F, // Shavian |
| 194 |
0x10480, 0x104AF, // Osmanya |
| 195 |
0x10800, 0x1083F, // Cypriot Syllabary |
| 196 |
0x10840, 0x1085F, // Imperial Aramaic |
| 197 |
0x10900, 0x1091F, // Phoenician |
| 198 |
0x10920, 0x1093F, // Lydian |
| 199 |
0x10A00, 0x10A5F, // Kharoshthi |
| 200 |
0x10A60, 0x10A7F, // Old South Arabian |
| 201 |
0x10B00, 0x10B3F, // Avestan |
| 202 |
0x10B40, 0x10B5F, // Inscriptional Parthian |
| 203 |
0x10B60, 0x10B7F, // Inscriptional Pahlavi |
| 204 |
0x10C00, 0x10C4F, // Old Turkic |
| 205 |
0x10E60, 0x10E7F, // Rumi Numeral Symbols |
| 206 |
0x11000, 0x1107F, // Brahmi |
| 207 |
0x11080, 0x110CF, // Kaithi |
| 208 |
0x12000, 0x123FF, // Cuneiform |
| 209 |
0x12400, 0x1247F, // Cuneiform Numbers and Punctuation |
| 210 |
0x13000, 0x1342F, // Egyptian Hieroglyphs |
| 211 |
0x16800, 0x16A3F, // Bamum Supplement |
| 212 |
0x1B000, 0x1B0FF, // Kana Supplement |
| 213 |
0x1D000, 0x1D0FF, // Byzantine Musical Symbols |
| 214 |
0x1D100, 0x1D1FF, // Musical Symbols |
| 215 |
0x1D200, 0x1D24F, // Ancient Greek Musical Notation |
| 216 |
0x1D300, 0x1D35F, // Tai Xuan Jing Symbols |
| 217 |
0x1D360, 0x1D37F, // Counting Rod Numerals |
| 218 |
0x1D400, 0x1D7FF, // Mathematical Alphanumeric Symbols |
| 219 |
0x1F000, 0x1F02F, // Mahjong Tiles |
| 220 |
0x1F030, 0x1F09F, // Domino Tiles |
| 221 |
0x1F0A0, 0x1F0FF, // Playing Cards |
| 222 |
0x1F100, 0x1F1FF, // Enclosed Alphanumeric Supplement |
| 223 |
0x1F200, 0x1F2FF, // Enclosed Ideographic Supplement |
| 224 |
0x1F300, 0x1F5FF, // Miscellaneous Symbols And Pictographs |
| 225 |
0x1F600, 0x1F64F, // Emoticons |
| 226 |
0x1F680, 0x1F6FF, // Transport And Map Symbols |
| 227 |
0x1F700, 0x1F77F, // Alchemical Symbols |
| 228 |
0x20000, 0x2A6DF, // CJK Unified Ideographs Extension B |
| 229 |
0x2A700, 0x2B73F, // CJK Unified Ideographs Extension C |
| 230 |
0x2B740, 0x2B81F, // CJK Unified Ideographs Extension D |
| 231 |
0x2F800, 0x2FA1F, // CJK Compatibility Ideographs Supplement |
| 232 |
0xE0000, 0xE007F, // Tags |
| 233 |
0xE0100, 0xE01EF, // Variation Selectors Supplement |
| 234 |
0xF0000, 0xFFFFF, // Supplementary Private Use Area-A |
| 235 |
0x100000, 0x10FFFF, // Supplementary Private Use Area-B |
| 236 |
0xFFFFFFFF |
| 237 |
}; |
| 238 |
|
| 239 |
// Copyright (c) 2008-2010 Bjoern Hoehrmann <[email protected]> |
| 240 |
// See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details. |
| 241 |
|
| 242 |
#define UTF8_ACCEPT 0u |
| 243 |
|
| 244 |
static const unsigned char utf8d[] = { |
| 245 |
// The first part of the table maps bytes to character classes that |
| 246 |
// to reduce the size of the transition table and create bitmasks. |
| 247 |
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
| 248 |
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
| 249 |
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
| 250 |
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
| 251 |
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, |
| 252 |
7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, |
| 253 |
8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, |
| 254 |
10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, |
| 255 |
|
| 256 |
// The second part is a transition table that maps a combination |
| 257 |
// of a state of the automaton and a character class to a state. |
| 258 |
0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12, |
| 259 |
12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12, |
| 260 |
12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12, |
| 261 |
12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12, |
| 262 |
12,36,12,12,12,12,12,12,12,12,12,12, |
| 263 |
}; |
| 264 |
|
| 265 |
static unsigned inline decode(unsigned* state, unsigned* codep, unsigned byte) { |
| 266 |
unsigned type = utf8d[byte]; |
| 267 |
|
| 268 |
*codep = (*state != UTF8_ACCEPT) ? |
| 269 |
(byte & 0x3fu) | (*codep << 6) : |
| 270 |
(0xffu >> type) & (byte); |
| 271 |
|
| 272 |
*state = utf8d[256 + *state + type]; |
| 273 |
return *state; |
| 274 |
} |
| 275 |
|
| 276 |
//static bool IsUTF8(unsigned char* s) { |
| 277 |
// unsigned codepoint, state = 0; |
| 278 |
// |
| 279 |
// while (*s) |
| 280 |
// decode(&state, &codepoint, *s++); |
| 281 |
// |
| 282 |
// return state == UTF8_ACCEPT; |
| 283 |
//} |
| 284 |
|
| 285 |
TEST(EncodingsTest, UTF8) { |
| 286 |
StringBuffer os, os2; |
| 287 |
for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) { |
| 288 |
for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) { |
| 289 |
os.Clear(); |
| 290 |
UTF8<>::Encode(os, codepoint); |
| 291 |
const char* encodedStr = os.GetString(); |
| 292 |
|
| 293 |
// Decode with Hoehrmann |
| 294 |
{ |
| 295 |
unsigned decodedCodepoint = 0; |
| 296 |
unsigned state = 0; |
| 297 |
|
| 298 |
unsigned decodedCount = 0; |
| 299 |
for (const char* s = encodedStr; *s; ++s) |
| 300 |
if (!decode(&state, &decodedCodepoint, static_cast<unsigned char>(*s))) { |
| 301 |
EXPECT_EQ(codepoint, decodedCodepoint); |
| 302 |
decodedCount++; |
| 303 |
} |
| 304 |
|
| 305 |
if (*encodedStr) { // This decoder cannot handle U+0000 |
| 306 |
EXPECT_EQ(1u, decodedCount); // Should only contain one code point |
| 307 |
} |
| 308 |
|
| 309 |
EXPECT_EQ(UTF8_ACCEPT, state); |
| 310 |
if (UTF8_ACCEPT != state) |
| 311 |
std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; |
| 312 |
} |
| 313 |
|
| 314 |
// Decode |
| 315 |
{ |
| 316 |
StringStream is(encodedStr); |
| 317 |
unsigned decodedCodepoint; |
| 318 |
bool result = UTF8<>::Decode(is, &decodedCodepoint); |
| 319 |
EXPECT_TRUE(result); |
| 320 |
EXPECT_EQ(codepoint, decodedCodepoint); |
| 321 |
if (!result || codepoint != decodedCodepoint) |
| 322 |
std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; |
| 323 |
} |
| 324 |
|
| 325 |
// Validate |
| 326 |
{ |
| 327 |
StringStream is(encodedStr); |
| 328 |
os2.Clear(); |
| 329 |
bool result = UTF8<>::Validate(is, os2); |
| 330 |
EXPECT_TRUE(result); |
| 331 |
EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); |
| 332 |
} |
| 333 |
} |
| 334 |
} |
| 335 |
} |
| 336 |
|
| 337 |
TEST(EncodingsTest, UTF16) { |
| 338 |
GenericStringBuffer<UTF16<> > os, os2; |
| 339 |
GenericStringBuffer<UTF8<> > utf8os; |
| 340 |
for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) { |
| 341 |
for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) { |
| 342 |
os.Clear(); |
| 343 |
UTF16<>::Encode(os, codepoint); |
| 344 |
const UTF16<>::Ch* encodedStr = os.GetString(); |
| 345 |
|
| 346 |
// Encode with Hoehrmann's code |
| 347 |
if (codepoint != 0) // cannot handle U+0000 |
| 348 |
{ |
| 349 |
// encode with UTF8<> first |
| 350 |
utf8os.Clear(); |
| 351 |
UTF8<>::Encode(utf8os, codepoint); |
| 352 |
|
| 353 |
// transcode from UTF8 to UTF16 with Hoehrmann's code |
| 354 |
unsigned decodedCodepoint = 0; |
| 355 |
unsigned state = 0; |
| 356 |
UTF16<>::Ch buffer[3], *p = &buffer[0]; |
| 357 |
for (const char* s = utf8os.GetString(); *s; ++s) { |
| 358 |
if (!decode(&state, &decodedCodepoint, static_cast<unsigned char>(*s))) |
| 359 |
break; |
| 360 |
} |
| 361 |
|
| 362 |
if (codepoint <= 0xFFFF) |
| 363 |
*p++ = static_cast<UTF16<>::Ch>(decodedCodepoint); |
| 364 |
else { |
| 365 |
// Encode code points above U+FFFF as surrogate pair. |
| 366 |
*p++ = static_cast<UTF16<>::Ch>(0xD7C0 + (decodedCodepoint >> 10)); |
| 367 |
*p++ = static_cast<UTF16<>::Ch>(0xDC00 + (decodedCodepoint & 0x3FF)); |
| 368 |
} |
| 369 |
*p++ = '\0'; |
| 370 |
|
| 371 |
EXPECT_EQ(0, StrCmp(buffer, encodedStr)); |
| 372 |
} |
| 373 |
|
| 374 |
// Decode |
| 375 |
{ |
| 376 |
GenericStringStream<UTF16<> > is(encodedStr); |
| 377 |
unsigned decodedCodepoint; |
| 378 |
bool result = UTF16<>::Decode(is, &decodedCodepoint); |
| 379 |
EXPECT_TRUE(result); |
| 380 |
EXPECT_EQ(codepoint, decodedCodepoint); |
| 381 |
if (!result || codepoint != decodedCodepoint) |
| 382 |
std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; |
| 383 |
} |
| 384 |
|
| 385 |
// Validate |
| 386 |
{ |
| 387 |
GenericStringStream<UTF16<> > is(encodedStr); |
| 388 |
os2.Clear(); |
| 389 |
bool result = UTF16<>::Validate(is, os2); |
| 390 |
EXPECT_TRUE(result); |
| 391 |
EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); |
| 392 |
} |
| 393 |
} |
| 394 |
} |
| 395 |
} |
| 396 |
|
| 397 |
TEST(EncodingsTest, UTF32) { |
| 398 |
GenericStringBuffer<UTF32<> > os, os2; |
| 399 |
for (const unsigned* range = kCodepointRanges; *range != 0xFFFFFFFF; range += 2) { |
| 400 |
for (unsigned codepoint = range[0]; codepoint <= range[1]; ++codepoint) { |
| 401 |
os.Clear(); |
| 402 |
UTF32<>::Encode(os, codepoint); |
| 403 |
const UTF32<>::Ch* encodedStr = os.GetString(); |
| 404 |
|
| 405 |
// Decode |
| 406 |
{ |
| 407 |
GenericStringStream<UTF32<> > is(encodedStr); |
| 408 |
unsigned decodedCodepoint; |
| 409 |
bool result = UTF32<>::Decode(is, &decodedCodepoint); |
| 410 |
EXPECT_TRUE(result); |
| 411 |
EXPECT_EQ(codepoint, decodedCodepoint); |
| 412 |
if (!result || codepoint != decodedCodepoint) |
| 413 |
std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; |
| 414 |
} |
| 415 |
|
| 416 |
// Validate |
| 417 |
{ |
| 418 |
GenericStringStream<UTF32<> > is(encodedStr); |
| 419 |
os2.Clear(); |
| 420 |
bool result = UTF32<>::Validate(is, os2); |
| 421 |
EXPECT_TRUE(result); |
| 422 |
EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); |
| 423 |
} |
| 424 |
} |
| 425 |
} |
| 426 |
} |
| 427 |
|
| 428 |
TEST(EncodingsTest, ASCII) { |
| 429 |
StringBuffer os, os2; |
| 430 |
for (unsigned codepoint = 0; codepoint < 128; codepoint++) { |
| 431 |
os.Clear(); |
| 432 |
ASCII<>::Encode(os, codepoint); |
| 433 |
const ASCII<>::Ch* encodedStr = os.GetString(); |
| 434 |
{ |
| 435 |
StringStream is(encodedStr); |
| 436 |
unsigned decodedCodepoint; |
| 437 |
bool result = ASCII<>::Decode(is, &decodedCodepoint); |
| 438 |
if (!result || codepoint != decodedCodepoint) |
| 439 |
std::cout << std::hex << codepoint << " " << decodedCodepoint << std::endl; |
| 440 |
} |
| 441 |
|
| 442 |
// Validate |
| 443 |
{ |
| 444 |
StringStream is(encodedStr); |
| 445 |
os2.Clear(); |
| 446 |
bool result = ASCII<>::Validate(is, os2); |
| 447 |
EXPECT_TRUE(result); |
| 448 |
EXPECT_EQ(0, StrCmp(encodedStr, os2.GetString())); |
| 449 |
} |
| 450 |
} |
| 451 |
} |