| 1 |
/* |
| 2 |
* Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/> |
| 3 |
* (C) 2020 Vladimir Sadovnikov <[email protected]> |
| 4 |
* |
| 5 |
* This file is part of lsp-runtime-lib |
| 6 |
* Created on: 14 окт. 2019 г. |
| 7 |
* |
| 8 |
* lsp-runtime-lib is free software: you can redistribute it and/or modify |
| 9 |
* it under the terms of the GNU Lesser General Public License as published by |
| 10 |
* the Free Software Foundation, either version 3 of the License, or |
| 11 |
* any later version. |
| 12 |
* |
| 13 |
* lsp-runtime-lib is distributed in the hope that it will be useful, |
| 14 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 |
* GNU Lesser General Public License for more details. |
| 17 |
* |
| 18 |
* You should have received a copy of the GNU Lesser General Public License |
| 19 |
* along with lsp-runtime-lib. If not, see <https://www.gnu.org/licenses/>. |
| 20 |
*/ |
| 21 |
|
| 22 |
#include <lsp-plug.in/stdlib/math.h> |
| 23 |
#include <lsp-plug.in/fmt/json/Tokenizer.h> |
| 24 |
#include <wctype.h> |
| 25 |
|
| 26 |
namespace lsp |
| 27 |
{ |
| 28 |
namespace json |
| 29 |
{ |
| 30 |
static const char *ecma_reserved[] = |
| 31 |
{ |
| 32 |
"Infinity", |
| 33 |
"abstract", |
| 34 |
"arguments", |
| 35 |
"await", |
| 36 |
"boolean", |
| 37 |
"break", |
| 38 |
"byte", |
| 39 |
"case", |
| 40 |
"catch", |
| 41 |
"char", |
| 42 |
"class", |
| 43 |
"const", |
| 44 |
"continue", |
| 45 |
"debugger", |
| 46 |
"default", |
| 47 |
"delete", |
| 48 |
"do", |
| 49 |
"double", |
| 50 |
"instanceof", |
| 51 |
"interface", |
| 52 |
"else", |
| 53 |
"enum", |
| 54 |
"eval", |
| 55 |
"export", |
| 56 |
"extends", |
| 57 |
"false", |
| 58 |
"final", |
| 59 |
"finally", |
| 60 |
"float", |
| 61 |
"for", |
| 62 |
"function", |
| 63 |
"goto", |
| 64 |
"if", |
| 65 |
"implements", |
| 66 |
"import", |
| 67 |
"in", |
| 68 |
"int", |
| 69 |
"let", |
| 70 |
"long", |
| 71 |
"native", |
| 72 |
"new", |
| 73 |
"null", |
| 74 |
"package", |
| 75 |
"private", |
| 76 |
"protected", |
| 77 |
"public", |
| 78 |
"return", |
| 79 |
"short", |
| 80 |
"static", |
| 81 |
"super", |
| 82 |
"switch", |
| 83 |
"synchronized", |
| 84 |
"this", |
| 85 |
"throw", |
| 86 |
"transient", |
| 87 |
"try", |
| 88 |
"true", |
| 89 |
"typeof", |
| 90 |
"var", |
| 91 |
"void", |
| 92 |
"volatile", |
| 93 |
"while", |
| 94 |
"with", |
| 95 |
"yield" |
| 96 |
}; |
| 97 |
|
| 98 |
Tokenizer::Tokenizer(io::IInSequence *in) |
| 99 |
{ |
| 100 |
pIn = in; |
| 101 |
cCurrent = -1; |
| 102 |
enToken = JT_UNKNOWN; |
| 103 |
nError = STATUS_OK; |
| 104 |
fValue = 0; |
| 105 |
iValue = 0; |
| 106 |
nUnget = 0; |
| 107 |
vPending = NULL; |
| 108 |
nPending = 0; |
| 109 |
nCapacity = 0; |
| 110 |
} |
| 111 |
|
| 112 |
Tokenizer::~Tokenizer() |
| 113 |
{ |
| 114 |
pIn = NULL; |
| 115 |
if (vPending != NULL) |
| 116 |
{ |
| 117 |
::free(vPending); |
| 118 |
vPending = NULL; |
| 119 |
} |
| 120 |
nPending = 0; |
| 121 |
nCapacity = 0; |
| 122 |
} |
| 123 |
|
| 124 |
token_t Tokenizer::set_error(status_t code) |
| 125 |
{ |
| 126 |
nError = code; |
| 127 |
return enToken = JT_ERROR; |
| 128 |
} |
| 129 |
|
| 130 |
lsp_swchar_t Tokenizer::skip_whitespace() |
| 131 |
{ |
| 132 |
if (cCurrent < 0) |
| 133 |
cCurrent = pIn->read(); |
| 134 |
|
| 135 |
while (true) |
| 136 |
{ |
| 137 |
// Skip whitespace |
| 138 |
if (::iswspace(cCurrent) || (::iswblank(cCurrent))) |
| 139 |
cCurrent = pIn->read(); |
| 140 |
else |
| 141 |
return cCurrent; |
| 142 |
} |
| 143 |
} |
| 144 |
|
| 145 |
lsp_swchar_t Tokenizer::lookup() |
| 146 |
{ |
| 147 |
if (cCurrent < 0) |
| 148 |
cCurrent = pIn->read(); |
| 149 |
return cCurrent; |
| 150 |
} |
| 151 |
|
| 152 |
token_t Tokenizer::commit(token_t token) |
| 153 |
{ |
| 154 |
if (cCurrent < 0) |
| 155 |
return set_error(STATUS_BAD_STATE); |
| 156 |
|
| 157 |
if (!sValue.append(cCurrent)) |
| 158 |
return set_error(STATUS_NO_MEM); |
| 159 |
|
| 160 |
enToken = token; |
| 161 |
cCurrent = -1; |
| 162 |
return token; |
| 163 |
} |
| 164 |
|
| 165 |
lsp_swchar_t Tokenizer::commit_lookup(token_t token) |
| 166 |
{ |
| 167 |
token_t x = commit(token); |
| 168 |
return (x != JT_ERROR) ? lookup() : -1; |
| 169 |
} |
| 170 |
|
| 171 |
bool Tokenizer::is_identifier_start(lsp_wchar_t ch) |
| 172 |
{ |
| 173 |
if (::iswupper(ch)) |
| 174 |
return true; |
| 175 |
if (::iswlower(ch)) |
| 176 |
return true; |
| 177 |
return (ch == '_') || (ch == '$') || (ch == '\\'); |
| 178 |
} |
| 179 |
|
| 180 |
bool Tokenizer::is_identifier(lsp_wchar_t ch) |
| 181 |
{ |
| 182 |
if (::iswupper(ch)) |
| 183 |
return true; |
| 184 |
if (::iswlower(ch)) |
| 185 |
return true; |
| 186 |
if (::iswdigit(ch)) |
| 187 |
return true; |
| 188 |
return (ch == '_') || (ch == '$'); |
| 189 |
} |
| 190 |
|
| 191 |
bool Tokenizer::is_reserved_word(const LSPString *text) |
| 192 |
{ |
| 193 |
ssize_t first = 0, last = sizeof(ecma_reserved) / sizeof (const char *) - 1; |
| 194 |
while (first <= last) |
| 195 |
{ |
| 196 |
ssize_t center = (first + last) >> 1; |
| 197 |
int cmp = text->compare_to_ascii(ecma_reserved[center]); |
| 198 |
if (cmp < 0) |
| 199 |
last = center - 1; |
| 200 |
else if (cmp > 0) |
| 201 |
first = center + 1; |
| 202 |
else |
| 203 |
return true; |
| 204 |
} |
| 205 |
|
| 206 |
return false; |
| 207 |
} |
| 208 |
|
| 209 |
bool Tokenizer::is_valid_identifier(const LSPString *text) |
| 210 |
{ |
| 211 |
size_t len = text->length(); |
| 212 |
if (len <= 0) |
| 213 |
return false; |
| 214 |
if (!is_identifier_start(text->char_at(0))) |
| 215 |
return false; |
| 216 |
for (size_t i=1; i<len; ++i) |
| 217 |
{ |
| 218 |
if (!is_identifier(text->char_at(i))) |
| 219 |
return false; |
| 220 |
} |
| 221 |
|
| 222 |
return !is_reserved_word(text); |
| 223 |
} |
| 224 |
|
| 225 |
bool Tokenizer::parse_digit(int *digit, lsp_wchar_t ch, int radix) |
| 226 |
{ |
| 227 |
int res; |
| 228 |
if ((ch >= '0') && (ch <= '9')) |
| 229 |
res = ch - '0'; |
| 230 |
else if ((ch >= 'a') && (ch <= 'f')) |
| 231 |
res = ch - 'a' + 10; |
| 232 |
else if ((ch >= 'A') && (ch <= 'F')) |
| 233 |
res = ch - 'A' + 10; |
| 234 |
else |
| 235 |
return false; |
| 236 |
|
| 237 |
if (res >= radix) |
| 238 |
return false; |
| 239 |
|
| 240 |
*digit = res; |
| 241 |
return true; |
| 242 |
} |
| 243 |
|
| 244 |
status_t Tokenizer::add_pending_character(lsp_utf16_t ch) |
| 245 |
{ |
| 246 |
if (nPending >= nCapacity) |
| 247 |
{ |
| 248 |
size_t cap = ((nPending + 0x10) & ~0xf) + nCapacity; |
| 249 |
lsp_utf16_t *buf = reinterpret_cast<lsp_utf16_t *>(::realloc(vPending, sizeof(lsp_utf16_t) * cap)); |
| 250 |
if (buf == NULL) |
| 251 |
return STATUS_NO_MEM; |
| 252 |
vPending = buf; |
| 253 |
} |
| 254 |
|
| 255 |
vPending[nPending++] = ch; |
| 256 |
return STATUS_OK; |
| 257 |
} |
| 258 |
|
| 259 |
status_t Tokenizer::commit_pending_characters() |
| 260 |
{ |
| 261 |
// Are there any pending characters? |
| 262 |
if (nPending <= 0) |
| 263 |
return STATUS_OK; |
| 264 |
|
| 265 |
// Initialize temporary string |
| 266 |
LSPString tmp; |
| 267 |
bool res = tmp.set_utf16(vPending, nPending); |
| 268 |
nPending = 0; |
| 269 |
if (!res) |
| 270 |
return STATUS_BAD_FORMAT; |
| 271 |
|
| 272 |
// Append string to result |
| 273 |
return (sValue.append(&tmp)) ? STATUS_OK : STATUS_NO_MEM; |
| 274 |
} |
| 275 |
|
| 276 |
token_t Tokenizer::parse_string(token_t type) |
| 277 |
{ |
| 278 |
sValue.clear(); |
| 279 |
skip(type); |
| 280 |
|
| 281 |
status_t res; |
| 282 |
bool protector = false; |
| 283 |
|
| 284 |
// Parse string |
| 285 |
while (true) |
| 286 |
{ |
| 287 |
// Read character |
| 288 |
lsp_swchar_t c = lookup(); |
| 289 |
if (c < 0) |
| 290 |
return set_error(-c); |
| 291 |
|
| 292 |
if (protector) |
| 293 |
{ |
| 294 |
protector = false; |
| 295 |
|
| 296 |
if ((c == 'u') || (c == 'U')) |
| 297 |
{ |
| 298 |
token_t tok = parse_unicode_escape_sequence(type); |
| 299 |
if (tok == JT_ERROR) |
| 300 |
return tok; |
| 301 |
} |
| 302 |
else if ((c == 'x') || (c == 'X')) |
| 303 |
{ |
| 304 |
token_t tok = parse_hexadecimal_escape_sequence(type); |
| 305 |
if (tok == JT_ERROR) |
| 306 |
return tok; |
| 307 |
} |
| 308 |
else |
| 309 |
{ |
| 310 |
// Commit previously pending unicode characters |
| 311 |
if ((res = commit_pending_characters()) != STATUS_OK) |
| 312 |
return set_error(res); |
| 313 |
|
| 314 |
switch (c) |
| 315 |
{ |
| 316 |
// Escaped characters: ' " \ b f n r t v |
| 317 |
case 'b': c = '\b'; break; |
| 318 |
case 'f': c = '\f'; break; |
| 319 |
case 'n': c = '\n'; break; |
| 320 |
case 'r': c = '\r'; break; |
| 321 |
case 't': c = '\t'; break; |
| 322 |
case 'v': c = '\v'; break; |
| 323 |
case '0': c = '\0'; break; |
| 324 |
case '\n': |
| 325 |
skip(type); // Skip end of line |
| 326 |
c = lookup(); |
| 327 |
if (c < 0) |
| 328 |
{ |
| 329 |
if (c != -STATUS_EOF) |
| 330 |
return set_error(-c); |
| 331 |
} |
| 332 |
else if (c == '\r') |
| 333 |
skip(type); |
| 334 |
c = -1; |
| 335 |
break; |
| 336 |
|
| 337 |
case '\r': |
| 338 |
case 0x2028: |
| 339 |
case 0x2029: |
| 340 |
skip(type); |
| 341 |
c = -1; // Line terminator |
| 342 |
break; |
| 343 |
|
| 344 |
default: // Any other characters just omit the protector character in ECMA script |
| 345 |
break; |
| 346 |
} |
| 347 |
|
| 348 |
// Append character (if it is present) |
| 349 |
if (c >= 0) |
| 350 |
{ |
| 351 |
if (!sValue.append(c)) |
| 352 |
return set_error(STATUS_NO_MEM); |
| 353 |
skip(type); |
| 354 |
} |
| 355 |
} |
| 356 |
} |
| 357 |
else if (c == '\\') |
| 358 |
{ |
| 359 |
skip(type); |
| 360 |
protector = true; |
| 361 |
} |
| 362 |
else |
| 363 |
{ |
| 364 |
// Commit previously pending unicode characters |
| 365 |
if ((res = commit_pending_characters()) != STATUS_OK) |
| 366 |
return set_error(res); |
| 367 |
|
| 368 |
switch (c) |
| 369 |
{ |
| 370 |
case '\n': |
| 371 |
return set_error(STATUS_BAD_TOKEN); |
| 372 |
case '\"': |
| 373 |
if (type == JT_DQ_STRING) |
| 374 |
return skip(type); |
| 375 |
break; |
| 376 |
case '\'': |
| 377 |
if (type == JT_SQ_STRING) |
| 378 |
return skip(type); |
| 379 |
break; |
| 380 |
default: |
| 381 |
break; |
| 382 |
} |
| 383 |
|
| 384 |
if ((type = commit(type)) == JT_ERROR) |
| 385 |
return type; |
| 386 |
} |
| 387 |
} |
| 388 |
|
| 389 |
return enToken = type; |
| 390 |
} |
| 391 |
|
| 392 |
token_t Tokenizer::parse_unicode_escape_sequence(token_t type) |
| 393 |
{ |
| 394 |
// Seek for 'u' or 'U' |
| 395 |
lsp_swchar_t c = lookup(); |
| 396 |
if (c < 0) |
| 397 |
return set_error(-c); |
| 398 |
else if ((c != 'u') && (c != 'U')) |
| 399 |
return set_error(STATUS_BAD_TOKEN); |
| 400 |
skip(type); |
| 401 |
|
| 402 |
// Read 4 mandatory digits |
| 403 |
int digit = 0; |
| 404 |
lsp_utf16_t cp = 0; |
| 405 |
for (size_t i=0; i<4; ++i) |
| 406 |
{ |
| 407 |
c = lookup(); |
| 408 |
if (c < 0) |
| 409 |
return set_error(-c); |
| 410 |
|
| 411 |
skip(type); |
| 412 |
if (!parse_digit(&digit, c, 16)) |
| 413 |
return set_error(STATUS_BAD_TOKEN); |
| 414 |
|
| 415 |
cp = (cp << 4) + digit; |
| 416 |
} |
| 417 |
|
| 418 |
// All is fine, store, truncate value and return result |
| 419 |
status_t res = add_pending_character(cp); |
| 420 |
return (res != STATUS_OK) ? set_error(res) : enToken = type; |
| 421 |
} |
| 422 |
|
| 423 |
token_t Tokenizer::parse_hexadecimal_escape_sequence(token_t type) |
| 424 |
{ |
| 425 |
// Seek for 'u' or 'U' |
| 426 |
lsp_swchar_t c = lookup(); |
| 427 |
if (c < 0) |
| 428 |
return set_error(-c); |
| 429 |
else if ((c != 'x') && (c != 'X')) |
| 430 |
return set_error(STATUS_BAD_TOKEN); |
| 431 |
skip(type); |
| 432 |
|
| 433 |
// Read 4 mandatory digits |
| 434 |
int digit = 0; |
| 435 |
size_t cp = 0; |
| 436 |
for (size_t i=0; i<2; ++i) |
| 437 |
{ |
| 438 |
c = lookup(); |
| 439 |
if (c < 0) |
| 440 |
return set_error(-c); |
| 441 |
|
| 442 |
skip(type); |
| 443 |
if (!parse_digit(&digit, c, 16)) |
| 444 |
return set_error(STATUS_BAD_TOKEN); |
| 445 |
|
| 446 |
cp = (cp << 4) + digit; |
| 447 |
} |
| 448 |
|
| 449 |
// All is fine, store, truncate value and return result |
| 450 |
status_t res = add_pending_character(cp); |
| 451 |
return (res != STATUS_OK) ? set_error(res) : enToken = type; |
| 452 |
} |
| 453 |
|
| 454 |
token_t Tokenizer::parse_identifier() |
| 455 |
{ |
| 456 |
// Commit identifier's start character |
| 457 |
status_t res; |
| 458 |
token_t tok; |
| 459 |
|
| 460 |
while (true) |
| 461 |
{ |
| 462 |
// Read character |
| 463 |
lsp_swchar_t c = lookup(); |
| 464 |
if (c < 0) |
| 465 |
{ |
| 466 |
if (c != -STATUS_EOF) |
| 467 |
return set_error(-c); |
| 468 |
|
| 469 |
// Commit pending characters first |
| 470 |
res = commit_pending_characters(); |
| 471 |
if (res != STATUS_OK) |
| 472 |
return set_error(res); |
| 473 |
break; |
| 474 |
} |
| 475 |
|
| 476 |
// Check that character is an identifier |
| 477 |
if (!is_identifier(c)) |
| 478 |
{ |
| 479 |
if (c != '\\') |
| 480 |
break; |
| 481 |
|
| 482 |
skip(JT_IDENTIFIER); |
| 483 |
tok = parse_unicode_escape_sequence(JT_IDENTIFIER); |
| 484 |
} |
| 485 |
else |
| 486 |
{ |
| 487 |
// Commit pending unicode characters first |
| 488 |
res = commit_pending_characters(); |
| 489 |
if (res != STATUS_OK) |
| 490 |
return set_error(res); |
| 491 |
|
| 492 |
// Now, commit pending character |
| 493 |
tok = commit(JT_IDENTIFIER); |
| 494 |
} |
| 495 |
|
| 496 |
if (tok == JT_ERROR) |
| 497 |
return tok; |
| 498 |
} |
| 499 |
|
| 500 |
// Analyze identifier |
| 501 |
if (sValue.equals_ascii("true")) |
| 502 |
return enToken = JT_TRUE; |
| 503 |
else if (sValue.equals_ascii("false")) |
| 504 |
return enToken = JT_FALSE; |
| 505 |
else if (sValue.equals_ascii("null")) |
| 506 |
return enToken = JT_NULL; |
| 507 |
else if (sValue.equals_ascii("NaN")) |
| 508 |
{ |
| 509 |
fValue = NAN; |
| 510 |
return enToken = JT_DOUBLE; |
| 511 |
} |
| 512 |
else if (sValue.equals_ascii("Infinity")) |
| 513 |
{ |
| 514 |
fValue = INFINITY; |
| 515 |
return enToken = JT_DOUBLE; |
| 516 |
} |
| 517 |
else if (is_reserved_word(&sValue)) |
| 518 |
return enToken = JT_RESERVED; |
| 519 |
|
| 520 |
return enToken = JT_IDENTIFIER; |
| 521 |
} |
| 522 |
|
| 523 |
token_t Tokenizer::parse_single_line_comment() |
| 524 |
{ |
| 525 |
// Reset the length of string literal |
| 526 |
sValue.set_length(0); |
| 527 |
skip(JT_SL_COMMENT); |
| 528 |
|
| 529 |
while (true) |
| 530 |
{ |
| 531 |
// Read character |
| 532 |
lsp_swchar_t c = lookup(); |
| 533 |
if (c < 0) |
| 534 |
return (c == -STATUS_EOF) ? JT_SL_COMMENT : set_error(-c); |
| 535 |
|
| 536 |
// Analyze character |
| 537 |
switch (c) { |
| 538 |
case '\n': |
| 539 |
{ |
| 540 |
status_t res = commit_pending_characters(); |
| 541 |
return (res == STATUS_OK) ? skip(JT_SL_COMMENT) : set_error(STATUS_BAD_TOKEN); |
| 542 |
} |
| 543 |
case '\\': |
| 544 |
{ |
| 545 |
skip(JT_SL_COMMENT); |
| 546 |
token_t tok = parse_unicode_escape_sequence(JT_SL_COMMENT); |
| 547 |
if (tok == JT_ERROR) |
| 548 |
return tok; |
| 549 |
break; |
| 550 |
} |
| 551 |
default: |
| 552 |
{ |
| 553 |
status_t res = commit_pending_characters(); |
| 554 |
if (res != STATUS_OK) |
| 555 |
return set_error(STATUS_BAD_TOKEN); |
| 556 |
|
| 557 |
token_t tok = commit(JT_SL_COMMENT); |
| 558 |
if (tok == JT_ERROR) |
| 559 |
return tok; |
| 560 |
break; |
| 561 |
} |
| 562 |
} |
| 563 |
} |
| 564 |
|
| 565 |
return JT_UNKNOWN; |
| 566 |
} |
| 567 |
|
| 568 |
token_t Tokenizer::parse_multiline_comment() |
| 569 |
{ |
| 570 |
// Reset the length of string literal |
| 571 |
sValue.set_length(0); |
| 572 |
skip(JT_SL_COMMENT); |
| 573 |
|
| 574 |
lsp_swchar_t last = -1; |
| 575 |
while (true) |
| 576 |
{ |
| 577 |
// Read character |
| 578 |
lsp_swchar_t c = lookup(); |
| 579 |
if (c < 0) |
| 580 |
return set_error(-c); |
| 581 |
|
| 582 |
// Check state |
| 583 |
if (last == '*') |
| 584 |
{ |
| 585 |
if (c == '/') |
| 586 |
{ |
| 587 |
// Remove last stored character (asterisk) |
| 588 |
sValue.remove_last(); |
| 589 |
return skip(JT_ML_COMMENT); |
| 590 |
} |
| 591 |
} |
| 592 |
else if (last == '\n') |
| 593 |
{ |
| 594 |
if (c == '\r') |
| 595 |
{ |
| 596 |
skip(JT_ML_COMMENT); |
| 597 |
continue; |
| 598 |
} |
| 599 |
last = -1; |
| 600 |
} |
| 601 |
|
| 602 |
// Append current character to the comment |
| 603 |
switch (c) { |
| 604 |
case '\\': |
| 605 |
{ |
| 606 |
skip(JT_ML_COMMENT); |
| 607 |
token_t tok = parse_unicode_escape_sequence(JT_ML_COMMENT); |
| 608 |
if (tok == JT_ERROR) |
| 609 |
return tok; |
| 610 |
break; |
| 611 |
} |
| 612 |
|
| 613 |
default: |
| 614 |
{ |
| 615 |
status_t res = commit_pending_characters(); |
| 616 |
if (res != STATUS_OK) |
| 617 |
return set_error(STATUS_BAD_TOKEN); |
| 618 |
|
| 619 |
token_t tok = commit(JT_ML_COMMENT); |
| 620 |
if (tok == JT_ERROR) |
| 621 |
return tok; |
| 622 |
} |
| 623 |
} |
| 624 |
|
| 625 |
// Remember last character |
| 626 |
last = c; |
| 627 |
} |
| 628 |
|
| 629 |
return JT_UNKNOWN; |
| 630 |
} |
| 631 |
|
| 632 |
token_t Tokenizer::parse_number() |
| 633 |
{ |
| 634 |
enum flags_t |
| 635 |
{ |
| 636 |
F_NEGATIVE = 1 << 0, |
| 637 |
F_SIGN = 1 << 1, |
| 638 |
F_INTEGER = 1 << 2, |
| 639 |
F_INT = 1 << 3, |
| 640 |
F_DOT = 1 << 4, |
| 641 |
F_FRAC = 1 << 5, |
| 642 |
F_EXP = 1 << 6, |
| 643 |
F_ESIGN = 1 << 7, |
| 644 |
F_ENEGATIVE = 1 << 8 |
| 645 |
}; |
| 646 |
|
| 647 |
lsp_swchar_t c = lookup(); |
| 648 |
size_t flags = 0; |
| 649 |
int radix = 10; |
| 650 |
int digit = 0; |
| 651 |
ssize_t ivalue = 0; |
| 652 |
double ifrac = 0; |
| 653 |
double ifpow = 1.0; |
| 654 |
double rradix = 0.1; |
| 655 |
ssize_t iexp = 0; |
| 656 |
|
| 657 |
// Has sign? |
| 658 |
if (c == '-') |
| 659 |
{ |
| 660 |
flags |= F_NEGATIVE | F_SIGN; |
| 661 |
c = commit_lookup(JT_UNKNOWN); |
| 662 |
} |
| 663 |
else if (c == '+') |
| 664 |
{ |
| 665 |
flags |= F_SIGN; |
| 666 |
c = commit_lookup(JT_UNKNOWN); |
| 667 |
} |
| 668 |
|
| 669 |
// Has prefix ? |
| 670 |
if (c == '0') |
| 671 |
{ |
| 672 |
c = commit_lookup(JT_UNKNOWN); |
| 673 |
switch (c) |
| 674 |
{ |
| 675 |
case 'x': // hexadecimal |
| 676 |
case 'X': |
| 677 |
radix = 16; |
| 678 |
rradix = 0.0625; |
| 679 |
c = commit_lookup(JT_UNKNOWN); |
| 680 |
flags |= F_INTEGER; |
| 681 |
break; |
| 682 |
default: |
| 683 |
flags |= F_INT; |
| 684 |
break; |
| 685 |
} |
| 686 |
} |
| 687 |
else if ((c == 'I') || (c == 'N')) // Infinity or NaN? |
| 688 |
{ |
| 689 |
LSPString tmp; |
| 690 |
tmp.swap(&sValue); |
| 691 |
token_t tok = parse_identifier(); |
| 692 |
if (!tmp.append(&sValue)) |
| 693 |
return set_error(STATUS_NO_MEM); |
| 694 |
sValue.swap(&tmp); |
| 695 |
if (tok != JT_DOUBLE) |
| 696 |
return enToken = JT_UNKNOWN; |
| 697 |
if (flags & F_NEGATIVE) |
| 698 |
fValue = -fValue; |
| 699 |
return tok; |
| 700 |
} |
| 701 |
|
| 702 |
// Read the integer part |
| 703 |
while (parse_digit(&digit, c, radix)) |
| 704 |
{ |
| 705 |
ivalue = ivalue*radix + digit; |
| 706 |
flags |= F_INT; |
| 707 |
c = commit_lookup(JT_DECIMAL); |
| 708 |
} |
| 709 |
|
| 710 |
// Is integer only? |
| 711 |
if (flags & F_INTEGER) |
| 712 |
{ |
| 713 |
if (!(flags & F_INT)) // There should be at least one integer character |
| 714 |
return enToken = JT_UNKNOWN; |
| 715 |
iValue = (flags & F_NEGATIVE) ? -ivalue : ivalue; |
| 716 |
return enToken = (radix != 16) ? JT_DECIMAL : JT_HEXADECIMAL; |
| 717 |
} |
| 718 |
|
| 719 |
// Has a fraction part? |
| 720 |
if (c == '.') |
| 721 |
{ |
| 722 |
flags |= F_DOT; |
| 723 |
c = commit_lookup(JT_DOUBLE); |
| 724 |
|
| 725 |
while (parse_digit(&digit, c, radix)) |
| 726 |
{ |
| 727 |
ifpow *= rradix; |
| 728 |
ifrac += digit * ifpow; |
| 729 |
flags |= F_FRAC; |
| 730 |
c = commit_lookup(JT_DOUBLE); |
| 731 |
} |
| 732 |
} |
| 733 |
|
| 734 |
// Is there at least INT or FRAC part defined? |
| 735 |
if ((flags & (F_INT | F_FRAC)) == 0) |
| 736 |
return enToken = JT_UNKNOWN; |
| 737 |
|
| 738 |
// Has an exponent part? |
| 739 |
if ((c == 'e') || (c == 'E')) |
| 740 |
{ |
| 741 |
c = commit_lookup(JT_DOUBLE); |
| 742 |
|
| 743 |
// Has sign? |
| 744 |
if (c == '-') |
| 745 |
{ |
| 746 |
flags |= F_ENEGATIVE | F_ESIGN; |
| 747 |
c = commit_lookup(JT_UNKNOWN); |
| 748 |
} |
| 749 |
else if (c == '+') |
| 750 |
{ |
| 751 |
flags |= F_ESIGN; |
| 752 |
c = commit_lookup(JT_UNKNOWN); |
| 753 |
} |
| 754 |
|
| 755 |
// Parse exponent |
| 756 |
while (parse_digit(&digit, c, radix)) |
| 757 |
{ |
| 758 |
iexp = iexp*radix + digit; |
| 759 |
flags |= F_EXP; |
| 760 |
c = commit_lookup(JT_DOUBLE); |
| 761 |
} |
| 762 |
|
| 763 |
// Analyze post-condition: if exponent sign is defined, |
| 764 |
// the exponent value also should be defined |
| 765 |
if ((flags & (F_ESIGN | F_EXP)) == F_ESIGN) |
| 766 |
return enToken = JT_UNKNOWN; |
| 767 |
else if (flags & F_ENEGATIVE) |
| 768 |
iexp = -iexp; |
| 769 |
} |
| 770 |
|
| 771 |
// Ensure that the next character is not an identifier character (ECMA) |
| 772 |
c = lookup(); |
| 773 |
if (c < 0) |
| 774 |
{ |
| 775 |
if (c != (-STATUS_EOF)) |
| 776 |
return set_error(-c); |
| 777 |
} |
| 778 |
else if (is_identifier_start(c)) |
| 779 |
return set_error(STATUS_BAD_FORMAT); |
| 780 |
|
| 781 |
// Now analyze parsing state |
| 782 |
if ((flags & (F_INT | F_FRAC | F_EXP | F_DOT)) == F_INT) |
| 783 |
{ |
| 784 |
iValue = (flags & F_NEGATIVE) ? -ivalue : ivalue; |
| 785 |
return enToken = JT_DECIMAL; |
| 786 |
} |
| 787 |
|
| 788 |
// Form the floating-point value |
| 789 |
double fv = (double(ivalue) + ifrac) * pow(radix, iexp); |
| 790 |
fValue = (flags & F_NEGATIVE) ? -fv : fv; |
| 791 |
return enToken = JT_DOUBLE; |
| 792 |
} |
| 793 |
|
| 794 |
token_t Tokenizer::get_token(bool get) |
| 795 |
{ |
| 796 |
// Pre-checks |
| 797 |
if (!get) |
| 798 |
return enToken; |
| 799 |
else if (nUnget > 0) |
| 800 |
{ |
| 801 |
--nUnget; |
| 802 |
return enToken; |
| 803 |
} |
| 804 |
|
| 805 |
// Skip whitespaces |
| 806 |
lsp_swchar_t c = skip_whitespace(); |
| 807 |
if (c < 0) |
| 808 |
{ |
| 809 |
enToken = (c == -STATUS_EOF) ? JT_EOF : JT_ERROR; |
| 810 |
nError = -c; |
| 811 |
return enToken; |
| 812 |
} |
| 813 |
|
| 814 |
sValue.set_length(0); |
| 815 |
|
| 816 |
switch (c) |
| 817 |
{ |
| 818 |
case '[': return commit(JT_LQ_BRACE); |
| 819 |
case ']': return commit(JT_RQ_BRACE); |
| 820 |
case '{': return commit(JT_LC_BRACE); |
| 821 |
case '}': return commit(JT_RC_BRACE); |
| 822 |
case ':': return commit(JT_COLON); |
| 823 |
case ',': return commit(JT_COMMA); |
| 824 |
case '\'': return parse_string(JT_SQ_STRING); |
| 825 |
case '\"': return parse_string(JT_DQ_STRING); |
| 826 |
case '/': |
| 827 |
c = commit_lookup(JT_UNKNOWN); |
| 828 |
if (c == '/') |
| 829 |
return parse_single_line_comment(); |
| 830 |
else if (c == '*') |
| 831 |
return parse_multiline_comment(); |
| 832 |
commit(JT_UNKNOWN); |
| 833 |
break; |
| 834 |
|
| 835 |
default: |
| 836 |
if (is_identifier_start(c)) |
| 837 |
return parse_identifier(); |
| 838 |
return parse_number(); |
| 839 |
} |
| 840 |
|
| 841 |
return enToken = JT_UNKNOWN; |
| 842 |
} |
| 843 |
|
| 844 |
} /* namespace json */ |
| 845 |
} /* namespace lsp */ |