/*
* Copyright (C) 2020 Linux Studio Plugins Project
* (C) 2020 Vladimir Sadovnikov
*
* This file is part of lsp-runtime-lib
* Created on: 14 окт. 2019 г.
*
* lsp-runtime-lib is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* lsp-runtime-lib is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with lsp-runtime-lib. If not, see .
*/
#include
#include
#include
namespace lsp
{
namespace json
{
static const char *ecma_reserved[] =
{
"Infinity",
"abstract",
"arguments",
"await",
"boolean",
"break",
"byte",
"case",
"catch",
"char",
"class",
"const",
"continue",
"debugger",
"default",
"delete",
"do",
"double",
"instanceof",
"interface",
"else",
"enum",
"eval",
"export",
"extends",
"false",
"final",
"finally",
"float",
"for",
"function",
"goto",
"if",
"implements",
"import",
"in",
"int",
"let",
"long",
"native",
"new",
"null",
"package",
"private",
"protected",
"public",
"return",
"short",
"static",
"super",
"switch",
"synchronized",
"this",
"throw",
"transient",
"try",
"true",
"typeof",
"var",
"void",
"volatile",
"while",
"with",
"yield"
};
Tokenizer::Tokenizer(io::IInSequence *in)
{
pIn = in;
cCurrent = -1;
enToken = JT_UNKNOWN;
nError = STATUS_OK;
fValue = 0;
iValue = 0;
nUnget = 0;
vPending = NULL;
nPending = 0;
nCapacity = 0;
}
Tokenizer::~Tokenizer()
{
pIn = NULL;
if (vPending != NULL)
{
::free(vPending);
vPending = NULL;
}
nPending = 0;
nCapacity = 0;
}
token_t Tokenizer::set_error(status_t code)
{
nError = code;
return enToken = JT_ERROR;
}
lsp_swchar_t Tokenizer::skip_whitespace()
{
if (cCurrent < 0)
cCurrent = pIn->read();
while (true)
{
// Skip whitespace
if (::iswspace(cCurrent) || (::iswblank(cCurrent)))
cCurrent = pIn->read();
else
return cCurrent;
}
}
lsp_swchar_t Tokenizer::lookup()
{
if (cCurrent < 0)
cCurrent = pIn->read();
return cCurrent;
}
token_t Tokenizer::commit(token_t token)
{
if (cCurrent < 0)
return set_error(STATUS_BAD_STATE);
if (!sValue.append(cCurrent))
return set_error(STATUS_NO_MEM);
enToken = token;
cCurrent = -1;
return token;
}
lsp_swchar_t Tokenizer::commit_lookup(token_t token)
{
token_t x = commit(token);
return (x != JT_ERROR) ? lookup() : -1;
}
bool Tokenizer::is_identifier_start(lsp_wchar_t ch)
{
if (::iswupper(ch))
return true;
if (::iswlower(ch))
return true;
return (ch == '_') || (ch == '$') || (ch == '\\');
}
bool Tokenizer::is_identifier(lsp_wchar_t ch)
{
if (::iswupper(ch))
return true;
if (::iswlower(ch))
return true;
if (::iswdigit(ch))
return true;
return (ch == '_') || (ch == '$');
}
bool Tokenizer::is_reserved_word(const LSPString *text)
{
ssize_t first = 0, last = sizeof(ecma_reserved) / sizeof (const char *) - 1;
while (first <= last)
{
ssize_t center = (first + last) >> 1;
int cmp = text->compare_to_ascii(ecma_reserved[center]);
if (cmp < 0)
last = center - 1;
else if (cmp > 0)
first = center + 1;
else
return true;
}
return false;
}
bool Tokenizer::is_valid_identifier(const LSPString *text)
{
size_t len = text->length();
if (len <= 0)
return false;
if (!is_identifier_start(text->char_at(0)))
return false;
for (size_t i=1; ichar_at(i)))
return false;
}
return !is_reserved_word(text);
}
bool Tokenizer::parse_digit(int *digit, lsp_wchar_t ch, int radix)
{
int res;
if ((ch >= '0') && (ch <= '9'))
res = ch - '0';
else if ((ch >= 'a') && (ch <= 'f'))
res = ch - 'a' + 10;
else if ((ch >= 'A') && (ch <= 'F'))
res = ch - 'A' + 10;
else
return false;
if (res >= radix)
return false;
*digit = res;
return true;
}
status_t Tokenizer::add_pending_character(lsp_utf16_t ch)
{
if (nPending >= nCapacity)
{
size_t cap = ((nPending + 0x10) & ~0xf) + nCapacity;
lsp_utf16_t *buf = reinterpret_cast(::realloc(vPending, sizeof(lsp_utf16_t) * cap));
if (buf == NULL)
return STATUS_NO_MEM;
vPending = buf;
}
vPending[nPending++] = ch;
return STATUS_OK;
}
status_t Tokenizer::commit_pending_characters()
{
// Are there any pending characters?
if (nPending <= 0)
return STATUS_OK;
// Initialize temporary string
LSPString tmp;
bool res = tmp.set_utf16(vPending, nPending);
nPending = 0;
if (!res)
return STATUS_BAD_FORMAT;
// Append string to result
return (sValue.append(&tmp)) ? STATUS_OK : STATUS_NO_MEM;
}
token_t Tokenizer::parse_string(token_t type)
{
sValue.clear();
skip(type);
status_t res;
bool protector = false;
// Parse string
while (true)
{
// Read character
lsp_swchar_t c = lookup();
if (c < 0)
return set_error(-c);
if (protector)
{
protector = false;
if ((c == 'u') || (c == 'U'))
{
token_t tok = parse_unicode_escape_sequence(type);
if (tok == JT_ERROR)
return tok;
}
else if ((c == 'x') || (c == 'X'))
{
token_t tok = parse_hexadecimal_escape_sequence(type);
if (tok == JT_ERROR)
return tok;
}
else
{
// Commit previously pending unicode characters
if ((res = commit_pending_characters()) != STATUS_OK)
return set_error(res);
switch (c)
{
// Escaped characters: ' " \ b f n r t v
case 'b': c = '\b'; break;
case 'f': c = '\f'; break;
case 'n': c = '\n'; break;
case 'r': c = '\r'; break;
case 't': c = '\t'; break;
case 'v': c = '\v'; break;
case '0': c = '\0'; break;
case '\n':
skip(type); // Skip end of line
c = lookup();
if (c < 0)
{
if (c != -STATUS_EOF)
return set_error(-c);
}
else if (c == '\r')
skip(type);
c = -1;
break;
case '\r':
case 0x2028:
case 0x2029:
skip(type);
c = -1; // Line terminator
break;
default: // Any other characters just omit the protector character in ECMA script
break;
}
// Append character (if it is present)
if (c >= 0)
{
if (!sValue.append(c))
return set_error(STATUS_NO_MEM);
skip(type);
}
}
}
else if (c == '\\')
{
skip(type);
protector = true;
}
else
{
// Commit previously pending unicode characters
if ((res = commit_pending_characters()) != STATUS_OK)
return set_error(res);
switch (c)
{
case '\n':
return set_error(STATUS_BAD_TOKEN);
case '\"':
if (type == JT_DQ_STRING)
return skip(type);
break;
case '\'':
if (type == JT_SQ_STRING)
return skip(type);
break;
default:
break;
}
if ((type = commit(type)) == JT_ERROR)
return type;
}
}
return enToken = type;
}
token_t Tokenizer::parse_unicode_escape_sequence(token_t type)
{
// Seek for 'u' or 'U'
lsp_swchar_t c = lookup();
if (c < 0)
return set_error(-c);
else if ((c != 'u') && (c != 'U'))
return set_error(STATUS_BAD_TOKEN);
skip(type);
// Read 4 mandatory digits
int digit = 0;
lsp_utf16_t cp = 0;
for (size_t i=0; i<4; ++i)
{
c = lookup();
if (c < 0)
return set_error(-c);
skip(type);
if (!parse_digit(&digit, c, 16))
return set_error(STATUS_BAD_TOKEN);
cp = (cp << 4) + digit;
}
// All is fine, store, truncate value and return result
status_t res = add_pending_character(cp);
return (res != STATUS_OK) ? set_error(res) : enToken = type;
}
token_t Tokenizer::parse_hexadecimal_escape_sequence(token_t type)
{
// Seek for 'u' or 'U'
lsp_swchar_t c = lookup();
if (c < 0)
return set_error(-c);
else if ((c != 'x') && (c != 'X'))
return set_error(STATUS_BAD_TOKEN);
skip(type);
// Read 4 mandatory digits
int digit = 0;
size_t cp = 0;
for (size_t i=0; i<2; ++i)
{
c = lookup();
if (c < 0)
return set_error(-c);
skip(type);
if (!parse_digit(&digit, c, 16))
return set_error(STATUS_BAD_TOKEN);
cp = (cp << 4) + digit;
}
// All is fine, store, truncate value and return result
status_t res = add_pending_character(cp);
return (res != STATUS_OK) ? set_error(res) : enToken = type;
}
token_t Tokenizer::parse_identifier()
{
// Commit identifier's start character
status_t res;
token_t tok;
while (true)
{
// Read character
lsp_swchar_t c = lookup();
if (c < 0)
{
if (c != -STATUS_EOF)
return set_error(-c);
// Commit pending characters first
res = commit_pending_characters();
if (res != STATUS_OK)
return set_error(res);
break;
}
// Check that character is an identifier
if (!is_identifier(c))
{
if (c != '\\')
break;
skip(JT_IDENTIFIER);
tok = parse_unicode_escape_sequence(JT_IDENTIFIER);
}
else
{
// Commit pending unicode characters first
res = commit_pending_characters();
if (res != STATUS_OK)
return set_error(res);
// Now, commit pending character
tok = commit(JT_IDENTIFIER);
}
if (tok == JT_ERROR)
return tok;
}
// Analyze identifier
if (sValue.equals_ascii("true"))
return enToken = JT_TRUE;
else if (sValue.equals_ascii("false"))
return enToken = JT_FALSE;
else if (sValue.equals_ascii("null"))
return enToken = JT_NULL;
else if (sValue.equals_ascii("NaN"))
{
fValue = NAN;
return enToken = JT_DOUBLE;
}
else if (sValue.equals_ascii("Infinity"))
{
fValue = INFINITY;
return enToken = JT_DOUBLE;
}
else if (is_reserved_word(&sValue))
return enToken = JT_RESERVED;
return enToken = JT_IDENTIFIER;
}
token_t Tokenizer::parse_single_line_comment()
{
// Reset the length of string literal
sValue.set_length(0);
skip(JT_SL_COMMENT);
while (true)
{
// Read character
lsp_swchar_t c = lookup();
if (c < 0)
return (c == -STATUS_EOF) ? JT_SL_COMMENT : set_error(-c);
// Analyze character
switch (c) {
case '\n':
{
status_t res = commit_pending_characters();
return (res == STATUS_OK) ? skip(JT_SL_COMMENT) : set_error(STATUS_BAD_TOKEN);
}
case '\\':
{
skip(JT_SL_COMMENT);
token_t tok = parse_unicode_escape_sequence(JT_SL_COMMENT);
if (tok == JT_ERROR)
return tok;
break;
}
default:
{
status_t res = commit_pending_characters();
if (res != STATUS_OK)
return set_error(STATUS_BAD_TOKEN);
token_t tok = commit(JT_SL_COMMENT);
if (tok == JT_ERROR)
return tok;
break;
}
}
}
return JT_UNKNOWN;
}
token_t Tokenizer::parse_multiline_comment()
{
// Reset the length of string literal
sValue.set_length(0);
skip(JT_SL_COMMENT);
lsp_swchar_t last = -1;
while (true)
{
// Read character
lsp_swchar_t c = lookup();
if (c < 0)
return set_error(-c);
// Check state
if (last == '*')
{
if (c == '/')
{
// Remove last stored character (asterisk)
sValue.remove_last();
return skip(JT_ML_COMMENT);
}
}
else if (last == '\n')
{
if (c == '\r')
{
skip(JT_ML_COMMENT);
continue;
}
last = -1;
}
// Append current character to the comment
switch (c) {
case '\\':
{
skip(JT_ML_COMMENT);
token_t tok = parse_unicode_escape_sequence(JT_ML_COMMENT);
if (tok == JT_ERROR)
return tok;
break;
}
default:
{
status_t res = commit_pending_characters();
if (res != STATUS_OK)
return set_error(STATUS_BAD_TOKEN);
token_t tok = commit(JT_ML_COMMENT);
if (tok == JT_ERROR)
return tok;
}
}
// Remember last character
last = c;
}
return JT_UNKNOWN;
}
token_t Tokenizer::parse_number()
{
enum flags_t
{
F_NEGATIVE = 1 << 0,
F_SIGN = 1 << 1,
F_INTEGER = 1 << 2,
F_INT = 1 << 3,
F_DOT = 1 << 4,
F_FRAC = 1 << 5,
F_EXP = 1 << 6,
F_ESIGN = 1 << 7,
F_ENEGATIVE = 1 << 8
};
lsp_swchar_t c = lookup();
size_t flags = 0;
int radix = 10;
int digit = 0;
ssize_t ivalue = 0;
double ifrac = 0;
double ifpow = 1.0;
double rradix = 0.1;
ssize_t iexp = 0;
// Has sign?
if (c == '-')
{
flags |= F_NEGATIVE | F_SIGN;
c = commit_lookup(JT_UNKNOWN);
}
else if (c == '+')
{
flags |= F_SIGN;
c = commit_lookup(JT_UNKNOWN);
}
// Has prefix ?
if (c == '0')
{
c = commit_lookup(JT_UNKNOWN);
switch (c)
{
case 'x': // hexadecimal
case 'X':
radix = 16;
rradix = 0.0625;
c = commit_lookup(JT_UNKNOWN);
flags |= F_INTEGER;
break;
default:
flags |= F_INT;
break;
}
}
else if ((c == 'I') || (c == 'N')) // Infinity or NaN?
{
LSPString tmp;
tmp.swap(&sValue);
token_t tok = parse_identifier();
if (!tmp.append(&sValue))
return set_error(STATUS_NO_MEM);
sValue.swap(&tmp);
if (tok != JT_DOUBLE)
return enToken = JT_UNKNOWN;
if (flags & F_NEGATIVE)
fValue = -fValue;
return tok;
}
// Read the integer part
while (parse_digit(&digit, c, radix))
{
ivalue = ivalue*radix + digit;
flags |= F_INT;
c = commit_lookup(JT_DECIMAL);
}
// Is integer only?
if (flags & F_INTEGER)
{
if (!(flags & F_INT)) // There should be at least one integer character
return enToken = JT_UNKNOWN;
iValue = (flags & F_NEGATIVE) ? -ivalue : ivalue;
return enToken = (radix != 16) ? JT_DECIMAL : JT_HEXADECIMAL;
}
// Has a fraction part?
if (c == '.')
{
flags |= F_DOT;
c = commit_lookup(JT_DOUBLE);
while (parse_digit(&digit, c, radix))
{
ifpow *= rradix;
ifrac += digit * ifpow;
flags |= F_FRAC;
c = commit_lookup(JT_DOUBLE);
}
}
// Is there at least INT or FRAC part defined?
if ((flags & (F_INT | F_FRAC)) == 0)
return enToken = JT_UNKNOWN;
// Has an exponent part?
if ((c == 'e') || (c == 'E'))
{
c = commit_lookup(JT_DOUBLE);
// Has sign?
if (c == '-')
{
flags |= F_ENEGATIVE | F_ESIGN;
c = commit_lookup(JT_UNKNOWN);
}
else if (c == '+')
{
flags |= F_ESIGN;
c = commit_lookup(JT_UNKNOWN);
}
// Parse exponent
while (parse_digit(&digit, c, radix))
{
iexp = iexp*radix + digit;
flags |= F_EXP;
c = commit_lookup(JT_DOUBLE);
}
// Analyze post-condition: if exponent sign is defined,
// the exponent value also should be defined
if ((flags & (F_ESIGN | F_EXP)) == F_ESIGN)
return enToken = JT_UNKNOWN;
else if (flags & F_ENEGATIVE)
iexp = -iexp;
}
// Ensure that the next character is not an identifier character (ECMA)
c = lookup();
if (c < 0)
{
if (c != (-STATUS_EOF))
return set_error(-c);
}
else if (is_identifier_start(c))
return set_error(STATUS_BAD_FORMAT);
// Now analyze parsing state
if ((flags & (F_INT | F_FRAC | F_EXP | F_DOT)) == F_INT)
{
iValue = (flags & F_NEGATIVE) ? -ivalue : ivalue;
return enToken = JT_DECIMAL;
}
// Form the floating-point value
double fv = (double(ivalue) + ifrac) * pow(radix, iexp);
fValue = (flags & F_NEGATIVE) ? -fv : fv;
return enToken = JT_DOUBLE;
}
token_t Tokenizer::get_token(bool get)
{
// Pre-checks
if (!get)
return enToken;
else if (nUnget > 0)
{
--nUnget;
return enToken;
}
// Skip whitespaces
lsp_swchar_t c = skip_whitespace();
if (c < 0)
{
enToken = (c == -STATUS_EOF) ? JT_EOF : JT_ERROR;
nError = -c;
return enToken;
}
sValue.set_length(0);
switch (c)
{
case '[': return commit(JT_LQ_BRACE);
case ']': return commit(JT_RQ_BRACE);
case '{': return commit(JT_LC_BRACE);
case '}': return commit(JT_RC_BRACE);
case ':': return commit(JT_COLON);
case ',': return commit(JT_COMMA);
case '\'': return parse_string(JT_SQ_STRING);
case '\"': return parse_string(JT_DQ_STRING);
case '/':
c = commit_lookup(JT_UNKNOWN);
if (c == '/')
return parse_single_line_comment();
else if (c == '*')
return parse_multiline_comment();
commit(JT_UNKNOWN);
break;
default:
if (is_identifier_start(c))
return parse_identifier();
return parse_number();
}
return enToken = JT_UNKNOWN;
}
} /* namespace json */
} /* namespace lsp */