| 1 |
// Tencent is pleased to support the open source community by making RapidJSON available. |
| 2 |
// |
| 3 |
// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. |
| 4 |
// |
| 5 |
// Licensed under the MIT License (the "License"); you may not use this file except |
| 6 |
// in compliance with the License. You may obtain a copy of the License at |
| 7 |
// |
| 8 |
// http://opensource.org/licenses/MIT |
| 9 |
// |
| 10 |
// Unless required by applicable law or agreed to in writing, software distributed |
| 11 |
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR |
| 12 |
// CONDITIONS OF ANY KIND, either express or implied. See the License for the |
| 13 |
// specific language governing permissions and limitations under the License. |
| 14 |
|
| 15 |
#ifndef RAPIDJSON_ENCODINGS_H_ |
| 16 |
#define RAPIDJSON_ENCODINGS_H_ |
| 17 |
|
| 18 |
#include "rapidjson.h" |
| 19 |
|
| 20 |
#ifdef _MSC_VER |
| 21 |
RAPIDJSON_DIAG_PUSH |
| 22 |
RAPIDJSON_DIAG_OFF(4244) // conversion from 'type1' to 'type2', possible loss of data |
| 23 |
RAPIDJSON_DIAG_OFF(4702) // unreachable code |
| 24 |
#elif defined(__GNUC__) |
| 25 |
RAPIDJSON_DIAG_PUSH |
| 26 |
RAPIDJSON_DIAG_OFF(effc++) |
| 27 |
RAPIDJSON_DIAG_OFF(overflow) |
| 28 |
#endif |
| 29 |
|
| 30 |
RAPIDJSON_NAMESPACE_BEGIN |
| 31 |
|
| 32 |
/////////////////////////////////////////////////////////////////////////////// |
| 33 |
// Encoding |
| 34 |
|
| 35 |
/*! \class rapidjson::Encoding |
| 36 |
\brief Concept for encoding of Unicode characters. |
| 37 |
|
| 38 |
\code |
| 39 |
concept Encoding { |
| 40 |
typename Ch; //! Type of character. A "character" is actually a code unit in unicode's definition. |
| 41 |
|
| 42 |
enum { supportUnicode = 1 }; // or 0 if not supporting unicode |
| 43 |
|
| 44 |
//! \brief Encode a Unicode codepoint to an output stream. |
| 45 |
//! \param os Output stream. |
| 46 |
//! \param codepoint An unicode codepoint, ranging from 0x0 to 0x10FFFF inclusively. |
| 47 |
template<typename OutputStream> |
| 48 |
static void Encode(OutputStream& os, unsigned codepoint); |
| 49 |
|
| 50 |
//! \brief Decode a Unicode codepoint from an input stream. |
| 51 |
//! \param is Input stream. |
| 52 |
//! \param codepoint Output of the unicode codepoint. |
| 53 |
//! \return true if a valid codepoint can be decoded from the stream. |
| 54 |
template <typename InputStream> |
| 55 |
static bool Decode(InputStream& is, unsigned* codepoint); |
| 56 |
|
| 57 |
//! \brief Validate one Unicode codepoint from an encoded stream. |
| 58 |
//! \param is Input stream to obtain codepoint. |
| 59 |
//! \param os Output for copying one codepoint. |
| 60 |
//! \return true if it is valid. |
| 61 |
//! \note This function just validating and copying the codepoint without actually decode it. |
| 62 |
template <typename InputStream, typename OutputStream> |
| 63 |
static bool Validate(InputStream& is, OutputStream& os); |
| 64 |
|
| 65 |
// The following functions are deal with byte streams. |
| 66 |
|
| 67 |
//! Take a character from input byte stream, skip BOM if exist. |
| 68 |
template <typename InputByteStream> |
| 69 |
static CharType TakeBOM(InputByteStream& is); |
| 70 |
|
| 71 |
//! Take a character from input byte stream. |
| 72 |
template <typename InputByteStream> |
| 73 |
static Ch Take(InputByteStream& is); |
| 74 |
|
| 75 |
//! Put BOM to output byte stream. |
| 76 |
template <typename OutputByteStream> |
| 77 |
static void PutBOM(OutputByteStream& os); |
| 78 |
|
| 79 |
//! Put a character to output byte stream. |
| 80 |
template <typename OutputByteStream> |
| 81 |
static void Put(OutputByteStream& os, Ch c); |
| 82 |
}; |
| 83 |
\endcode |
| 84 |
*/ |
| 85 |
|
| 86 |
/////////////////////////////////////////////////////////////////////////////// |
| 87 |
// UTF8 |
| 88 |
|
| 89 |
//! UTF-8 encoding. |
| 90 |
/*! http://en.wikipedia.org/wiki/UTF-8 |
| 91 |
http://tools.ietf.org/html/rfc3629 |
| 92 |
\tparam CharType Code unit for storing 8-bit UTF-8 data. Default is char. |
| 93 |
\note implements Encoding concept |
| 94 |
*/ |
| 95 |
template<typename CharType = char> |
| 96 |
struct UTF8 { |
| 97 |
typedef CharType Ch; |
| 98 |
|
| 99 |
enum { supportUnicode = 1 }; |
| 100 |
|
| 101 |
template<typename OutputStream> |
| 102 |
static void Encode(OutputStream& os, unsigned codepoint) { |
| 103 |
if (codepoint <= 0x7F) |
| 104 |
os.Put(static_cast<Ch>(codepoint & 0xFF)); |
| 105 |
else if (codepoint <= 0x7FF) { |
| 106 |
os.Put(static_cast<Ch>(0xC0 | ((codepoint >> 6) & 0xFF))); |
| 107 |
os.Put(static_cast<Ch>(0x80 | ((codepoint & 0x3F)))); |
| 108 |
} |
| 109 |
else if (codepoint <= 0xFFFF) { |
| 110 |
os.Put(static_cast<Ch>(0xE0 | ((codepoint >> 12) & 0xFF))); |
| 111 |
os.Put(static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F))); |
| 112 |
os.Put(static_cast<Ch>(0x80 | (codepoint & 0x3F))); |
| 113 |
} |
| 114 |
else { |
| 115 |
RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); |
| 116 |
os.Put(static_cast<Ch>(0xF0 | ((codepoint >> 18) & 0xFF))); |
| 117 |
os.Put(static_cast<Ch>(0x80 | ((codepoint >> 12) & 0x3F))); |
| 118 |
os.Put(static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F))); |
| 119 |
os.Put(static_cast<Ch>(0x80 | (codepoint & 0x3F))); |
| 120 |
} |
| 121 |
} |
| 122 |
|
| 123 |
template<typename OutputStream> |
| 124 |
static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { |
| 125 |
if (codepoint <= 0x7F) |
| 126 |
PutUnsafe(os, static_cast<Ch>(codepoint & 0xFF)); |
| 127 |
else if (codepoint <= 0x7FF) { |
| 128 |
PutUnsafe(os, static_cast<Ch>(0xC0 | ((codepoint >> 6) & 0xFF))); |
| 129 |
PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint & 0x3F)))); |
| 130 |
} |
| 131 |
else if (codepoint <= 0xFFFF) { |
| 132 |
PutUnsafe(os, static_cast<Ch>(0xE0 | ((codepoint >> 12) & 0xFF))); |
| 133 |
PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F))); |
| 134 |
PutUnsafe(os, static_cast<Ch>(0x80 | (codepoint & 0x3F))); |
| 135 |
} |
| 136 |
else { |
| 137 |
RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); |
| 138 |
PutUnsafe(os, static_cast<Ch>(0xF0 | ((codepoint >> 18) & 0xFF))); |
| 139 |
PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 12) & 0x3F))); |
| 140 |
PutUnsafe(os, static_cast<Ch>(0x80 | ((codepoint >> 6) & 0x3F))); |
| 141 |
PutUnsafe(os, static_cast<Ch>(0x80 | (codepoint & 0x3F))); |
| 142 |
} |
| 143 |
} |
| 144 |
|
| 145 |
template <typename InputStream> |
| 146 |
static bool Decode(InputStream& is, unsigned* codepoint) { |
| 147 |
#define COPY() c = is.Take(); *codepoint = (*codepoint << 6) | (static_cast<unsigned char>(c) & 0x3Fu) |
| 148 |
#define TRANS(mask) result &= ((GetRange(static_cast<unsigned char>(c)) & mask) != 0) |
| 149 |
#define TAIL() COPY(); TRANS(0x70) |
| 150 |
typename InputStream::Ch c = is.Take(); |
| 151 |
if (!(c & 0x80)) { |
| 152 |
*codepoint = static_cast<unsigned char>(c); |
| 153 |
return true; |
| 154 |
} |
| 155 |
|
| 156 |
unsigned char type = GetRange(static_cast<unsigned char>(c)); |
| 157 |
if (type >= 32) { |
| 158 |
*codepoint = 0; |
| 159 |
} else { |
| 160 |
*codepoint = (0xFF >> type) & static_cast<unsigned char>(c); |
| 161 |
} |
| 162 |
bool result = true; |
| 163 |
switch (type) { |
| 164 |
case 2: TAIL(); return result; |
| 165 |
case 3: TAIL(); TAIL(); return result; |
| 166 |
case 4: COPY(); TRANS(0x50); TAIL(); return result; |
| 167 |
case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; |
| 168 |
case 6: TAIL(); TAIL(); TAIL(); return result; |
| 169 |
case 10: COPY(); TRANS(0x20); TAIL(); return result; |
| 170 |
case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; |
| 171 |
default: return false; |
| 172 |
} |
| 173 |
#undef COPY |
| 174 |
#undef TRANS |
| 175 |
#undef TAIL |
| 176 |
} |
| 177 |
|
| 178 |
template <typename InputStream, typename OutputStream> |
| 179 |
static bool Validate(InputStream& is, OutputStream& os) { |
| 180 |
#define COPY() os.Put(c = is.Take()) |
| 181 |
#define TRANS(mask) result &= ((GetRange(static_cast<unsigned char>(c)) & mask) != 0) |
| 182 |
#define TAIL() COPY(); TRANS(0x70) |
| 183 |
Ch c; |
| 184 |
COPY(); |
| 185 |
if (!(c & 0x80)) |
| 186 |
return true; |
| 187 |
|
| 188 |
bool result = true; |
| 189 |
switch (GetRange(static_cast<unsigned char>(c))) { |
| 190 |
case 2: TAIL(); return result; |
| 191 |
case 3: TAIL(); TAIL(); return result; |
| 192 |
case 4: COPY(); TRANS(0x50); TAIL(); return result; |
| 193 |
case 5: COPY(); TRANS(0x10); TAIL(); TAIL(); return result; |
| 194 |
case 6: TAIL(); TAIL(); TAIL(); return result; |
| 195 |
case 10: COPY(); TRANS(0x20); TAIL(); return result; |
| 196 |
case 11: COPY(); TRANS(0x60); TAIL(); TAIL(); return result; |
| 197 |
default: return false; |
| 198 |
} |
| 199 |
#undef COPY |
| 200 |
#undef TRANS |
| 201 |
#undef TAIL |
| 202 |
} |
| 203 |
|
| 204 |
static unsigned char GetRange(unsigned char c) { |
| 205 |
// Referring to DFA of http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ |
| 206 |
// With new mapping 1 -> 0x10, 7 -> 0x20, 9 -> 0x40, such that AND operation can test multiple types. |
| 207 |
static const unsigned char type[] = { |
| 208 |
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
| 209 |
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
| 210 |
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
| 211 |
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, |
| 212 |
0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10, |
| 213 |
0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40,0x40, |
| 214 |
0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, |
| 215 |
0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20,0x20, |
| 216 |
8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, |
| 217 |
10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8, |
| 218 |
}; |
| 219 |
return type[c]; |
| 220 |
} |
| 221 |
|
| 222 |
template <typename InputByteStream> |
| 223 |
static CharType TakeBOM(InputByteStream& is) { |
| 224 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 225 |
typename InputByteStream::Ch c = Take(is); |
| 226 |
if (static_cast<unsigned char>(c) != 0xEFu) return c; |
| 227 |
c = is.Take(); |
| 228 |
if (static_cast<unsigned char>(c) != 0xBBu) return c; |
| 229 |
c = is.Take(); |
| 230 |
if (static_cast<unsigned char>(c) != 0xBFu) return c; |
| 231 |
c = is.Take(); |
| 232 |
return c; |
| 233 |
} |
| 234 |
|
| 235 |
template <typename InputByteStream> |
| 236 |
static Ch Take(InputByteStream& is) { |
| 237 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 238 |
return static_cast<Ch>(is.Take()); |
| 239 |
} |
| 240 |
|
| 241 |
template <typename OutputByteStream> |
| 242 |
static void PutBOM(OutputByteStream& os) { |
| 243 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 244 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xEFu)); |
| 245 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xBBu)); |
| 246 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xBFu)); |
| 247 |
} |
| 248 |
|
| 249 |
template <typename OutputByteStream> |
| 250 |
static void Put(OutputByteStream& os, Ch c) { |
| 251 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 252 |
os.Put(static_cast<typename OutputByteStream::Ch>(c)); |
| 253 |
} |
| 254 |
}; |
| 255 |
|
| 256 |
/////////////////////////////////////////////////////////////////////////////// |
| 257 |
// UTF16 |
| 258 |
|
| 259 |
//! UTF-16 encoding. |
| 260 |
/*! http://en.wikipedia.org/wiki/UTF-16 |
| 261 |
http://tools.ietf.org/html/rfc2781 |
| 262 |
\tparam CharType Type for storing 16-bit UTF-16 data. Default is wchar_t. C++11 may use char16_t instead. |
| 263 |
\note implements Encoding concept |
| 264 |
|
| 265 |
\note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. |
| 266 |
For streaming, use UTF16LE and UTF16BE, which handle endianness. |
| 267 |
*/ |
| 268 |
template<typename CharType = wchar_t> |
| 269 |
struct UTF16 { |
| 270 |
typedef CharType Ch; |
| 271 |
RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 2); |
| 272 |
|
| 273 |
enum { supportUnicode = 1 }; |
| 274 |
|
| 275 |
template<typename OutputStream> |
| 276 |
static void Encode(OutputStream& os, unsigned codepoint) { |
| 277 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); |
| 278 |
if (codepoint <= 0xFFFF) { |
| 279 |
RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair |
| 280 |
os.Put(static_cast<typename OutputStream::Ch>(codepoint)); |
| 281 |
} |
| 282 |
else { |
| 283 |
RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); |
| 284 |
unsigned v = codepoint - 0x10000; |
| 285 |
os.Put(static_cast<typename OutputStream::Ch>((v >> 10) | 0xD800)); |
| 286 |
os.Put((v & 0x3FF) | 0xDC00); |
| 287 |
} |
| 288 |
} |
| 289 |
|
| 290 |
|
| 291 |
template<typename OutputStream> |
| 292 |
static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { |
| 293 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); |
| 294 |
if (codepoint <= 0xFFFF) { |
| 295 |
RAPIDJSON_ASSERT(codepoint < 0xD800 || codepoint > 0xDFFF); // Code point itself cannot be surrogate pair |
| 296 |
PutUnsafe(os, static_cast<typename OutputStream::Ch>(codepoint)); |
| 297 |
} |
| 298 |
else { |
| 299 |
RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); |
| 300 |
unsigned v = codepoint - 0x10000; |
| 301 |
PutUnsafe(os, static_cast<typename OutputStream::Ch>((v >> 10) | 0xD800)); |
| 302 |
PutUnsafe(os, (v & 0x3FF) | 0xDC00); |
| 303 |
} |
| 304 |
} |
| 305 |
|
| 306 |
template <typename InputStream> |
| 307 |
static bool Decode(InputStream& is, unsigned* codepoint) { |
| 308 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); |
| 309 |
typename InputStream::Ch c = is.Take(); |
| 310 |
if (c < 0xD800 || c > 0xDFFF) { |
| 311 |
*codepoint = static_cast<unsigned>(c); |
| 312 |
return true; |
| 313 |
} |
| 314 |
else if (c <= 0xDBFF) { |
| 315 |
*codepoint = (static_cast<unsigned>(c) & 0x3FF) << 10; |
| 316 |
c = is.Take(); |
| 317 |
*codepoint |= (static_cast<unsigned>(c) & 0x3FF); |
| 318 |
*codepoint += 0x10000; |
| 319 |
return c >= 0xDC00 && c <= 0xDFFF; |
| 320 |
} |
| 321 |
return false; |
| 322 |
} |
| 323 |
|
| 324 |
template <typename InputStream, typename OutputStream> |
| 325 |
static bool Validate(InputStream& is, OutputStream& os) { |
| 326 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 2); |
| 327 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 2); |
| 328 |
typename InputStream::Ch c; |
| 329 |
os.Put(static_cast<typename OutputStream::Ch>(c = is.Take())); |
| 330 |
if (c < 0xD800 || c > 0xDFFF) |
| 331 |
return true; |
| 332 |
else if (c <= 0xDBFF) { |
| 333 |
os.Put(c = is.Take()); |
| 334 |
return c >= 0xDC00 && c <= 0xDFFF; |
| 335 |
} |
| 336 |
return false; |
| 337 |
} |
| 338 |
}; |
| 339 |
|
| 340 |
//! UTF-16 little endian encoding. |
| 341 |
template<typename CharType = wchar_t> |
| 342 |
struct UTF16LE : UTF16<CharType> { |
| 343 |
template <typename InputByteStream> |
| 344 |
static CharType TakeBOM(InputByteStream& is) { |
| 345 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 346 |
CharType c = Take(is); |
| 347 |
return static_cast<uint16_t>(c) == 0xFEFFu ? Take(is) : c; |
| 348 |
} |
| 349 |
|
| 350 |
template <typename InputByteStream> |
| 351 |
static CharType Take(InputByteStream& is) { |
| 352 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 353 |
unsigned c = static_cast<uint8_t>(is.Take()); |
| 354 |
c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8; |
| 355 |
return static_cast<CharType>(c); |
| 356 |
} |
| 357 |
|
| 358 |
template <typename OutputByteStream> |
| 359 |
static void PutBOM(OutputByteStream& os) { |
| 360 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 361 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu)); |
| 362 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu)); |
| 363 |
} |
| 364 |
|
| 365 |
template <typename OutputByteStream> |
| 366 |
static void Put(OutputByteStream& os, CharType c) { |
| 367 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 368 |
os.Put(static_cast<typename OutputByteStream::Ch>(static_cast<unsigned>(c) & 0xFFu)); |
| 369 |
os.Put(static_cast<typename OutputByteStream::Ch>((static_cast<unsigned>(c) >> 8) & 0xFFu)); |
| 370 |
} |
| 371 |
}; |
| 372 |
|
| 373 |
//! UTF-16 big endian encoding. |
| 374 |
template<typename CharType = wchar_t> |
| 375 |
struct UTF16BE : UTF16<CharType> { |
| 376 |
template <typename InputByteStream> |
| 377 |
static CharType TakeBOM(InputByteStream& is) { |
| 378 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 379 |
CharType c = Take(is); |
| 380 |
return static_cast<uint16_t>(c) == 0xFEFFu ? Take(is) : c; |
| 381 |
} |
| 382 |
|
| 383 |
template <typename InputByteStream> |
| 384 |
static CharType Take(InputByteStream& is) { |
| 385 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 386 |
unsigned c = static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8; |
| 387 |
c |= static_cast<uint8_t>(is.Take()); |
| 388 |
return static_cast<CharType>(c); |
| 389 |
} |
| 390 |
|
| 391 |
template <typename OutputByteStream> |
| 392 |
static void PutBOM(OutputByteStream& os) { |
| 393 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 394 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu)); |
| 395 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu)); |
| 396 |
} |
| 397 |
|
| 398 |
template <typename OutputByteStream> |
| 399 |
static void Put(OutputByteStream& os, CharType c) { |
| 400 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 401 |
os.Put(static_cast<typename OutputByteStream::Ch>((static_cast<unsigned>(c) >> 8) & 0xFFu)); |
| 402 |
os.Put(static_cast<typename OutputByteStream::Ch>(static_cast<unsigned>(c) & 0xFFu)); |
| 403 |
} |
| 404 |
}; |
| 405 |
|
| 406 |
/////////////////////////////////////////////////////////////////////////////// |
| 407 |
// UTF32 |
| 408 |
|
| 409 |
//! UTF-32 encoding. |
| 410 |
/*! http://en.wikipedia.org/wiki/UTF-32 |
| 411 |
\tparam CharType Type for storing 32-bit UTF-32 data. Default is unsigned. C++11 may use char32_t instead. |
| 412 |
\note implements Encoding concept |
| 413 |
|
| 414 |
\note For in-memory access, no need to concern endianness. The code units and code points are represented by CPU's endianness. |
| 415 |
For streaming, use UTF32LE and UTF32BE, which handle endianness. |
| 416 |
*/ |
| 417 |
template<typename CharType = unsigned> |
| 418 |
struct UTF32 { |
| 419 |
typedef CharType Ch; |
| 420 |
RAPIDJSON_STATIC_ASSERT(sizeof(Ch) >= 4); |
| 421 |
|
| 422 |
enum { supportUnicode = 1 }; |
| 423 |
|
| 424 |
template<typename OutputStream> |
| 425 |
static void Encode(OutputStream& os, unsigned codepoint) { |
| 426 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); |
| 427 |
RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); |
| 428 |
os.Put(codepoint); |
| 429 |
} |
| 430 |
|
| 431 |
template<typename OutputStream> |
| 432 |
static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { |
| 433 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputStream::Ch) >= 4); |
| 434 |
RAPIDJSON_ASSERT(codepoint <= 0x10FFFF); |
| 435 |
PutUnsafe(os, codepoint); |
| 436 |
} |
| 437 |
|
| 438 |
template <typename InputStream> |
| 439 |
static bool Decode(InputStream& is, unsigned* codepoint) { |
| 440 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); |
| 441 |
Ch c = is.Take(); |
| 442 |
*codepoint = c; |
| 443 |
return c <= 0x10FFFF; |
| 444 |
} |
| 445 |
|
| 446 |
template <typename InputStream, typename OutputStream> |
| 447 |
static bool Validate(InputStream& is, OutputStream& os) { |
| 448 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputStream::Ch) >= 4); |
| 449 |
Ch c; |
| 450 |
os.Put(c = is.Take()); |
| 451 |
return c <= 0x10FFFF; |
| 452 |
} |
| 453 |
}; |
| 454 |
|
| 455 |
//! UTF-32 little endian enocoding. |
| 456 |
template<typename CharType = unsigned> |
| 457 |
struct UTF32LE : UTF32<CharType> { |
| 458 |
template <typename InputByteStream> |
| 459 |
static CharType TakeBOM(InputByteStream& is) { |
| 460 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 461 |
CharType c = Take(is); |
| 462 |
return static_cast<uint32_t>(c) == 0x0000FEFFu ? Take(is) : c; |
| 463 |
} |
| 464 |
|
| 465 |
template <typename InputByteStream> |
| 466 |
static CharType Take(InputByteStream& is) { |
| 467 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 468 |
unsigned c = static_cast<uint8_t>(is.Take()); |
| 469 |
c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8; |
| 470 |
c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 16; |
| 471 |
c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 24; |
| 472 |
return static_cast<CharType>(c); |
| 473 |
} |
| 474 |
|
| 475 |
template <typename OutputByteStream> |
| 476 |
static void PutBOM(OutputByteStream& os) { |
| 477 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 478 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu)); |
| 479 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu)); |
| 480 |
os.Put(static_cast<typename OutputByteStream::Ch>(0x00u)); |
| 481 |
os.Put(static_cast<typename OutputByteStream::Ch>(0x00u)); |
| 482 |
} |
| 483 |
|
| 484 |
template <typename OutputByteStream> |
| 485 |
static void Put(OutputByteStream& os, CharType c) { |
| 486 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 487 |
os.Put(static_cast<typename OutputByteStream::Ch>(c & 0xFFu)); |
| 488 |
os.Put(static_cast<typename OutputByteStream::Ch>((c >> 8) & 0xFFu)); |
| 489 |
os.Put(static_cast<typename OutputByteStream::Ch>((c >> 16) & 0xFFu)); |
| 490 |
os.Put(static_cast<typename OutputByteStream::Ch>((c >> 24) & 0xFFu)); |
| 491 |
} |
| 492 |
}; |
| 493 |
|
| 494 |
//! UTF-32 big endian encoding. |
| 495 |
template<typename CharType = unsigned> |
| 496 |
struct UTF32BE : UTF32<CharType> { |
| 497 |
template <typename InputByteStream> |
| 498 |
static CharType TakeBOM(InputByteStream& is) { |
| 499 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 500 |
CharType c = Take(is); |
| 501 |
return static_cast<uint32_t>(c) == 0x0000FEFFu ? Take(is) : c; |
| 502 |
} |
| 503 |
|
| 504 |
template <typename InputByteStream> |
| 505 |
static CharType Take(InputByteStream& is) { |
| 506 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 507 |
unsigned c = static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 24; |
| 508 |
c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 16; |
| 509 |
c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())) << 8; |
| 510 |
c |= static_cast<unsigned>(static_cast<uint8_t>(is.Take())); |
| 511 |
return static_cast<CharType>(c); |
| 512 |
} |
| 513 |
|
| 514 |
template <typename OutputByteStream> |
| 515 |
static void PutBOM(OutputByteStream& os) { |
| 516 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 517 |
os.Put(static_cast<typename OutputByteStream::Ch>(0x00u)); |
| 518 |
os.Put(static_cast<typename OutputByteStream::Ch>(0x00u)); |
| 519 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xFEu)); |
| 520 |
os.Put(static_cast<typename OutputByteStream::Ch>(0xFFu)); |
| 521 |
} |
| 522 |
|
| 523 |
template <typename OutputByteStream> |
| 524 |
static void Put(OutputByteStream& os, CharType c) { |
| 525 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 526 |
os.Put(static_cast<typename OutputByteStream::Ch>((c >> 24) & 0xFFu)); |
| 527 |
os.Put(static_cast<typename OutputByteStream::Ch>((c >> 16) & 0xFFu)); |
| 528 |
os.Put(static_cast<typename OutputByteStream::Ch>((c >> 8) & 0xFFu)); |
| 529 |
os.Put(static_cast<typename OutputByteStream::Ch>(c & 0xFFu)); |
| 530 |
} |
| 531 |
}; |
| 532 |
|
| 533 |
/////////////////////////////////////////////////////////////////////////////// |
| 534 |
// ASCII |
| 535 |
|
| 536 |
//! ASCII encoding. |
| 537 |
/*! http://en.wikipedia.org/wiki/ASCII |
| 538 |
\tparam CharType Code unit for storing 7-bit ASCII data. Default is char. |
| 539 |
\note implements Encoding concept |
| 540 |
*/ |
| 541 |
template<typename CharType = char> |
| 542 |
struct ASCII { |
| 543 |
typedef CharType Ch; |
| 544 |
|
| 545 |
enum { supportUnicode = 0 }; |
| 546 |
|
| 547 |
template<typename OutputStream> |
| 548 |
static void Encode(OutputStream& os, unsigned codepoint) { |
| 549 |
RAPIDJSON_ASSERT(codepoint <= 0x7F); |
| 550 |
os.Put(static_cast<Ch>(codepoint & 0xFF)); |
| 551 |
} |
| 552 |
|
| 553 |
template<typename OutputStream> |
| 554 |
static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { |
| 555 |
RAPIDJSON_ASSERT(codepoint <= 0x7F); |
| 556 |
PutUnsafe(os, static_cast<Ch>(codepoint & 0xFF)); |
| 557 |
} |
| 558 |
|
| 559 |
template <typename InputStream> |
| 560 |
static bool Decode(InputStream& is, unsigned* codepoint) { |
| 561 |
uint8_t c = static_cast<uint8_t>(is.Take()); |
| 562 |
*codepoint = c; |
| 563 |
return c <= 0X7F; |
| 564 |
} |
| 565 |
|
| 566 |
template <typename InputStream, typename OutputStream> |
| 567 |
static bool Validate(InputStream& is, OutputStream& os) { |
| 568 |
uint8_t c = static_cast<uint8_t>(is.Take()); |
| 569 |
os.Put(static_cast<typename OutputStream::Ch>(c)); |
| 570 |
return c <= 0x7F; |
| 571 |
} |
| 572 |
|
| 573 |
template <typename InputByteStream> |
| 574 |
static CharType TakeBOM(InputByteStream& is) { |
| 575 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 576 |
uint8_t c = static_cast<uint8_t>(Take(is)); |
| 577 |
return static_cast<Ch>(c); |
| 578 |
} |
| 579 |
|
| 580 |
template <typename InputByteStream> |
| 581 |
static Ch Take(InputByteStream& is) { |
| 582 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); |
| 583 |
return static_cast<Ch>(is.Take()); |
| 584 |
} |
| 585 |
|
| 586 |
template <typename OutputByteStream> |
| 587 |
static void PutBOM(OutputByteStream& os) { |
| 588 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 589 |
(void)os; |
| 590 |
} |
| 591 |
|
| 592 |
template <typename OutputByteStream> |
| 593 |
static void Put(OutputByteStream& os, Ch c) { |
| 594 |
RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); |
| 595 |
os.Put(static_cast<typename OutputByteStream::Ch>(c)); |
| 596 |
} |
| 597 |
}; |
| 598 |
|
| 599 |
/////////////////////////////////////////////////////////////////////////////// |
| 600 |
// AutoUTF |
| 601 |
|
| 602 |
//! Runtime-specified UTF encoding type of a stream. |
| 603 |
enum UTFType { |
| 604 |
kUTF8 = 0, //!< UTF-8. |
| 605 |
kUTF16LE = 1, //!< UTF-16 little endian. |
| 606 |
kUTF16BE = 2, //!< UTF-16 big endian. |
| 607 |
kUTF32LE = 3, //!< UTF-32 little endian. |
| 608 |
kUTF32BE = 4 //!< UTF-32 big endian. |
| 609 |
}; |
| 610 |
|
| 611 |
//! Dynamically select encoding according to stream's runtime-specified UTF encoding type. |
| 612 |
/*! \note This class can be used with AutoUTFInputtStream and AutoUTFOutputStream, which provides GetType(). |
| 613 |
*/ |
| 614 |
template<typename CharType> |
| 615 |
struct AutoUTF { |
| 616 |
typedef CharType Ch; |
| 617 |
|
| 618 |
enum { supportUnicode = 1 }; |
| 619 |
|
| 620 |
#define RAPIDJSON_ENCODINGS_FUNC(x) UTF8<Ch>::x, UTF16LE<Ch>::x, UTF16BE<Ch>::x, UTF32LE<Ch>::x, UTF32BE<Ch>::x |
| 621 |
|
| 622 |
template<typename OutputStream> |
| 623 |
RAPIDJSON_FORCEINLINE static void Encode(OutputStream& os, unsigned codepoint) { |
| 624 |
typedef void (*EncodeFunc)(OutputStream&, unsigned); |
| 625 |
static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Encode) }; |
| 626 |
(*f[os.GetType()])(os, codepoint); |
| 627 |
} |
| 628 |
|
| 629 |
template<typename OutputStream> |
| 630 |
RAPIDJSON_FORCEINLINE static void EncodeUnsafe(OutputStream& os, unsigned codepoint) { |
| 631 |
typedef void (*EncodeFunc)(OutputStream&, unsigned); |
| 632 |
static const EncodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(EncodeUnsafe) }; |
| 633 |
(*f[os.GetType()])(os, codepoint); |
| 634 |
} |
| 635 |
|
| 636 |
template <typename InputStream> |
| 637 |
RAPIDJSON_FORCEINLINE static bool Decode(InputStream& is, unsigned* codepoint) { |
| 638 |
typedef bool (*DecodeFunc)(InputStream&, unsigned*); |
| 639 |
static const DecodeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Decode) }; |
| 640 |
return (*f[is.GetType()])(is, codepoint); |
| 641 |
} |
| 642 |
|
| 643 |
template <typename InputStream, typename OutputStream> |
| 644 |
RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { |
| 645 |
typedef bool (*ValidateFunc)(InputStream&, OutputStream&); |
| 646 |
static const ValidateFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Validate) }; |
| 647 |
return (*f[is.GetType()])(is, os); |
| 648 |
} |
| 649 |
|
| 650 |
#undef RAPIDJSON_ENCODINGS_FUNC |
| 651 |
}; |
| 652 |
|
| 653 |
/////////////////////////////////////////////////////////////////////////////// |
| 654 |
// Transcoder |
| 655 |
|
| 656 |
//! Encoding conversion. |
| 657 |
template<typename SourceEncoding, typename TargetEncoding> |
| 658 |
struct Transcoder { |
| 659 |
//! Take one Unicode codepoint from source encoding, convert it to target encoding and put it to the output stream. |
| 660 |
template<typename InputStream, typename OutputStream> |
| 661 |
RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { |
| 662 |
unsigned codepoint; |
| 663 |
if (!SourceEncoding::Decode(is, &codepoint)) |
| 664 |
return false; |
| 665 |
TargetEncoding::Encode(os, codepoint); |
| 666 |
return true; |
| 667 |
} |
| 668 |
|
| 669 |
template<typename InputStream, typename OutputStream> |
| 670 |
RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { |
| 671 |
unsigned codepoint; |
| 672 |
if (!SourceEncoding::Decode(is, &codepoint)) |
| 673 |
return false; |
| 674 |
TargetEncoding::EncodeUnsafe(os, codepoint); |
| 675 |
return true; |
| 676 |
} |
| 677 |
|
| 678 |
//! Validate one Unicode codepoint from an encoded stream. |
| 679 |
template<typename InputStream, typename OutputStream> |
| 680 |
RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { |
| 681 |
return Transcode(is, os); // Since source/target encoding is different, must transcode. |
| 682 |
} |
| 683 |
}; |
| 684 |
|
| 685 |
// Forward declaration. |
| 686 |
template<typename Stream> |
| 687 |
inline void PutUnsafe(Stream& stream, typename Stream::Ch c); |
| 688 |
|
| 689 |
//! Specialization of Transcoder with same source and target encoding. |
| 690 |
template<typename Encoding> |
| 691 |
struct Transcoder<Encoding, Encoding> { |
| 692 |
template<typename InputStream, typename OutputStream> |
| 693 |
RAPIDJSON_FORCEINLINE static bool Transcode(InputStream& is, OutputStream& os) { |
| 694 |
os.Put(is.Take()); // Just copy one code unit. This semantic is different from primary template class. |
| 695 |
return true; |
| 696 |
} |
| 697 |
|
| 698 |
template<typename InputStream, typename OutputStream> |
| 699 |
RAPIDJSON_FORCEINLINE static bool TranscodeUnsafe(InputStream& is, OutputStream& os) { |
| 700 |
PutUnsafe(os, is.Take()); // Just copy one code unit. This semantic is different from primary template class. |
| 701 |
return true; |
| 702 |
} |
| 703 |
|
| 704 |
template<typename InputStream, typename OutputStream> |
| 705 |
RAPIDJSON_FORCEINLINE static bool Validate(InputStream& is, OutputStream& os) { |
| 706 |
return Encoding::Validate(is, os); // source/target encoding are the same |
| 707 |
} |
| 708 |
}; |
| 709 |
|
| 710 |
RAPIDJSON_NAMESPACE_END |
| 711 |
|
| 712 |
#if defined(__GNUC__) || defined(_MSC_VER) |
| 713 |
RAPIDJSON_DIAG_POP |
| 714 |
#endif |
| 715 |
|
| 716 |
#endif // RAPIDJSON_ENCODINGS_H_ |