| 1 |
// Copyright 2006 Nemanja Trifunovic |
| 2 |
|
| 3 |
/* |
| 4 |
Permission is hereby granted, free of charge, to any person or organization |
| 5 |
obtaining a copy of the software and accompanying documentation covered by |
| 6 |
this license (the "Software") to use, reproduce, display, distribute, |
| 7 |
execute, and transmit the Software, and to prepare derivative works of the |
| 8 |
Software, and to permit third-parties to whom the Software is furnished to |
| 9 |
do so, all subject to the following: |
| 10 |
|
| 11 |
The copyright notices in the Software and this entire statement, including |
| 12 |
the above license grant, this restriction and the following disclaimer, |
| 13 |
must be included in all copies of the Software, in whole or in part, and |
| 14 |
all derivative works of the Software, unless such copies or derivative |
| 15 |
works are solely in the form of machine-executable object code generated by |
| 16 |
a source language processor. |
| 17 |
|
| 18 |
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 19 |
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 20 |
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT |
| 21 |
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE |
| 22 |
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, |
| 23 |
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| 24 |
DEALINGS IN THE SOFTWARE. |
| 25 |
*/ |
| 26 |
|
| 27 |
|
| 28 |
#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 |
| 29 |
#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 |
| 30 |
|
| 31 |
#include "core.h" |
| 32 |
|
| 33 |
namespace utf8 |
| 34 |
{ |
| 35 |
namespace unchecked |
| 36 |
{ |
| 37 |
template <typename octet_iterator> |
| 38 |
octet_iterator append(uint32_t cp, octet_iterator result) |
| 39 |
{ |
| 40 |
return internal::append(cp, result); |
| 41 |
} |
| 42 |
|
| 43 |
template <typename octet_iterator, typename output_iterator> |
| 44 |
output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, uint32_t replacement) |
| 45 |
{ |
| 46 |
while (start != end) { |
| 47 |
octet_iterator sequence_start = start; |
| 48 |
internal::utf_error err_code = utf8::internal::validate_next(start, end); |
| 49 |
switch (err_code) { |
| 50 |
case internal::UTF8_OK : |
| 51 |
for (octet_iterator it = sequence_start; it != start; ++it) |
| 52 |
*out++ = *it; |
| 53 |
break; |
| 54 |
case internal::NOT_ENOUGH_ROOM: |
| 55 |
out = utf8::unchecked::append (replacement, out); |
| 56 |
start = end; |
| 57 |
break; |
| 58 |
case internal::INVALID_LEAD: |
| 59 |
out = utf8::unchecked::append (replacement, out); |
| 60 |
++start; |
| 61 |
break; |
| 62 |
case internal::INCOMPLETE_SEQUENCE: |
| 63 |
case internal::OVERLONG_SEQUENCE: |
| 64 |
case internal::INVALID_CODE_POINT: |
| 65 |
out = utf8::unchecked::append (replacement, out); |
| 66 |
++start; |
| 67 |
// just one replacement mark for the sequence |
| 68 |
while (start != end && utf8::internal::is_trail(*start)) |
| 69 |
++start; |
| 70 |
break; |
| 71 |
} |
| 72 |
} |
| 73 |
return out; |
| 74 |
} |
| 75 |
|
| 76 |
template <typename octet_iterator, typename output_iterator> |
| 77 |
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) |
| 78 |
{ |
| 79 |
static const uint32_t replacement_marker = utf8::internal::mask16(0xfffd); |
| 80 |
return utf8::unchecked::replace_invalid(start, end, out, replacement_marker); |
| 81 |
} |
| 82 |
|
| 83 |
template <typename octet_iterator> |
| 84 |
uint32_t next(octet_iterator& it) |
| 85 |
{ |
| 86 |
uint32_t cp = utf8::internal::mask8(*it); |
| 87 |
typename std::iterator_traits<octet_iterator>::difference_type length = utf8::internal::sequence_length(it); |
| 88 |
switch (length) { |
| 89 |
case 1: |
| 90 |
break; |
| 91 |
case 2: |
| 92 |
it++; |
| 93 |
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); |
| 94 |
break; |
| 95 |
case 3: |
| 96 |
++it; |
| 97 |
cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); |
| 98 |
++it; |
| 99 |
cp += (*it) & 0x3f; |
| 100 |
break; |
| 101 |
case 4: |
| 102 |
++it; |
| 103 |
cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); |
| 104 |
++it; |
| 105 |
cp += (utf8::internal::mask8(*it) << 6) & 0xfff; |
| 106 |
++it; |
| 107 |
cp += (*it) & 0x3f; |
| 108 |
break; |
| 109 |
} |
| 110 |
++it; |
| 111 |
return cp; |
| 112 |
} |
| 113 |
|
| 114 |
template <typename octet_iterator> |
| 115 |
uint32_t peek_next(octet_iterator it) |
| 116 |
{ |
| 117 |
return utf8::unchecked::next(it); |
| 118 |
} |
| 119 |
|
| 120 |
template <typename octet_iterator> |
| 121 |
uint32_t prior(octet_iterator& it) |
| 122 |
{ |
| 123 |
while (utf8::internal::is_trail(*(--it))) ; |
| 124 |
octet_iterator temp = it; |
| 125 |
return utf8::unchecked::next(temp); |
| 126 |
} |
| 127 |
|
| 128 |
template <typename octet_iterator, typename distance_type> |
| 129 |
void advance (octet_iterator& it, distance_type n) |
| 130 |
{ |
| 131 |
const distance_type zero(0); |
| 132 |
if (n < zero) { |
| 133 |
// backward |
| 134 |
for (distance_type i = n; i < zero; ++i) |
| 135 |
utf8::unchecked::prior(it); |
| 136 |
} else { |
| 137 |
// forward |
| 138 |
for (distance_type i = zero; i < n; ++i) |
| 139 |
utf8::unchecked::next(it); |
| 140 |
} |
| 141 |
} |
| 142 |
|
| 143 |
template <typename octet_iterator> |
| 144 |
typename std::iterator_traits<octet_iterator>::difference_type |
| 145 |
distance (octet_iterator first, octet_iterator last) |
| 146 |
{ |
| 147 |
typename std::iterator_traits<octet_iterator>::difference_type dist; |
| 148 |
for (dist = 0; first < last; ++dist) |
| 149 |
utf8::unchecked::next(first); |
| 150 |
return dist; |
| 151 |
} |
| 152 |
|
| 153 |
template <typename u16bit_iterator, typename octet_iterator> |
| 154 |
octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) |
| 155 |
{ |
| 156 |
while (start != end) { |
| 157 |
uint32_t cp = utf8::internal::mask16(*start++); |
| 158 |
// Take care of surrogate pairs first |
| 159 |
if (utf8::internal::is_lead_surrogate(cp)) { |
| 160 |
uint32_t trail_surrogate = utf8::internal::mask16(*start++); |
| 161 |
cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; |
| 162 |
} |
| 163 |
result = utf8::unchecked::append(cp, result); |
| 164 |
} |
| 165 |
return result; |
| 166 |
} |
| 167 |
|
| 168 |
template <typename u16bit_iterator, typename octet_iterator> |
| 169 |
u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) |
| 170 |
{ |
| 171 |
while (start < end) { |
| 172 |
uint32_t cp = utf8::unchecked::next(start); |
| 173 |
if (cp > 0xffff) { //make a surrogate pair |
| 174 |
*result++ = static_cast<uint16_t>((cp >> 10) + internal::LEAD_OFFSET); |
| 175 |
*result++ = static_cast<uint16_t>((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); |
| 176 |
} |
| 177 |
else |
| 178 |
*result++ = static_cast<uint16_t>(cp); |
| 179 |
} |
| 180 |
return result; |
| 181 |
} |
| 182 |
|
| 183 |
template <typename octet_iterator, typename u32bit_iterator> |
| 184 |
octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) |
| 185 |
{ |
| 186 |
while (start != end) |
| 187 |
result = utf8::unchecked::append(*(start++), result); |
| 188 |
|
| 189 |
return result; |
| 190 |
} |
| 191 |
|
| 192 |
template <typename octet_iterator, typename u32bit_iterator> |
| 193 |
u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) |
| 194 |
{ |
| 195 |
while (start < end) |
| 196 |
(*result++) = utf8::unchecked::next(start); |
| 197 |
|
| 198 |
return result; |
| 199 |
} |
| 200 |
|
| 201 |
// The iterator class |
| 202 |
template <typename octet_iterator> |
| 203 |
class iterator { |
| 204 |
octet_iterator it; |
| 205 |
public: |
| 206 |
typedef uint32_t value_type; |
| 207 |
typedef uint32_t* pointer; |
| 208 |
typedef uint32_t& reference; |
| 209 |
typedef std::ptrdiff_t difference_type; |
| 210 |
typedef std::bidirectional_iterator_tag iterator_category; |
| 211 |
iterator () {} |
| 212 |
explicit iterator (const octet_iterator& octet_it): it(octet_it) {} |
| 213 |
// the default "big three" are OK |
| 214 |
octet_iterator base () const { return it; } |
| 215 |
uint32_t operator * () const |
| 216 |
{ |
| 217 |
octet_iterator temp = it; |
| 218 |
return utf8::unchecked::next(temp); |
| 219 |
} |
| 220 |
bool operator == (const iterator& rhs) const |
| 221 |
{ |
| 222 |
return (it == rhs.it); |
| 223 |
} |
| 224 |
bool operator != (const iterator& rhs) const |
| 225 |
{ |
| 226 |
return !(operator == (rhs)); |
| 227 |
} |
| 228 |
iterator& operator ++ () |
| 229 |
{ |
| 230 |
::std::advance(it, utf8::internal::sequence_length(it)); |
| 231 |
return *this; |
| 232 |
} |
| 233 |
iterator operator ++ (int) |
| 234 |
{ |
| 235 |
iterator temp = *this; |
| 236 |
::std::advance(it, utf8::internal::sequence_length(it)); |
| 237 |
return temp; |
| 238 |
} |
| 239 |
iterator& operator -- () |
| 240 |
{ |
| 241 |
utf8::unchecked::prior(it); |
| 242 |
return *this; |
| 243 |
} |
| 244 |
iterator operator -- (int) |
| 245 |
{ |
| 246 |
iterator temp = *this; |
| 247 |
utf8::unchecked::prior(it); |
| 248 |
return temp; |
| 249 |
} |
| 250 |
}; // class iterator |
| 251 |
|
| 252 |
} // namespace utf8::unchecked |
| 253 |
} // namespace utf8 |
| 254 |
|
| 255 |
|
| 256 |
#endif // header guard |
| 257 |
|