| 1 |
/* |
| 2 |
* Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/> |
| 3 |
* (C) 2020 Vladimir Sadovnikov <[email protected]> |
| 4 |
* |
| 5 |
* This file is part of lsp-runtime-lib |
| 6 |
* Created on: 15 окт. 2019 г. |
| 7 |
* |
| 8 |
* lsp-runtime-lib is free software: you can redistribute it and/or modify |
| 9 |
* it under the terms of the GNU Lesser General Public License as published by |
| 10 |
* the Free Software Foundation, either version 3 of the License, or |
| 11 |
* any later version. |
| 12 |
* |
| 13 |
* lsp-runtime-lib is distributed in the hope that it will be useful, |
| 14 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 |
* GNU Lesser General Public License for more details. |
| 17 |
* |
| 18 |
* You should have received a copy of the GNU Lesser General Public License |
| 19 |
* along with lsp-runtime-lib. If not, see <https://www.gnu.org/licenses/>. |
| 20 |
*/ |
| 21 |
|
| 22 |
#include <lsp-plug.in/test-fw/utest.h> |
| 23 |
#include <lsp-plug.in/test-fw/helpers.h> |
| 24 |
#include <lsp-plug.in/io/InStringSequence.h> |
| 25 |
#include <lsp-plug.in/fmt/json/token.h> |
| 26 |
#include <lsp-plug.in/fmt/json/Tokenizer.h> |
| 27 |
#include <lsp-plug.in/stdlib/math.h> |
| 28 |
//#include <locale.h> |
| 29 |
|
| 30 |
using namespace lsp; |
| 31 |
using namespace lsp::json; |
| 32 |
|
| 33 |
UTEST_BEGIN("runtime.fmt.json", tokenizer) |
| 34 |
|
| 35 |
void ck_token(Tokenizer &t, const char *s, token_t token) |
| 36 |
{ |
| 37 |
printf(" checking token: %s\n", s); |
| 38 |
token_t tok = t.get_token(true); |
| 39 |
UTEST_ASSERT_MSG(tok == token, "Error testing token: %s", s); |
| 40 |
LSPString tmp; |
| 41 |
UTEST_ASSERT(tmp.set_utf8(s)); |
| 42 |
UTEST_ASSERT_MSG(t.text_value()->equals(&tmp), "Error testing token: %s, got: %s", s, t.text_value()->get_utf8()); |
| 43 |
} |
| 44 |
|
| 45 |
void ck_int(Tokenizer &t, ssize_t value) |
| 46 |
{ |
| 47 |
printf(" checking integer: %d\n", int(value)); |
| 48 |
token_t tok = t.get_token(true); |
| 49 |
UTEST_ASSERT_MSG((tok == JT_DECIMAL) || (tok == JT_HEXADECIMAL), "Error testing token: not decimal/hexadecimal value"); |
| 50 |
UTEST_ASSERT_MSG(t.int_value() == value, "Error testing token: %d != %d", int(t.int_value()), int(value)); |
| 51 |
} |
| 52 |
|
| 53 |
void ck_float(Tokenizer &t, double value) |
| 54 |
{ |
| 55 |
printf(" checking float: %f\n", double(value)); |
| 56 |
token_t tok = t.get_token(true); |
| 57 |
UTEST_ASSERT_MSG(tok == JT_DOUBLE, "Error testing token: not double value"); |
| 58 |
UTEST_ASSERT_MSG(float_equals_relative(t.float_value(), value), "Error testing token: %f != %f", t.float_value(), value); |
| 59 |
} |
| 60 |
|
| 61 |
void ck_invalid(const char *s, token_t token) |
| 62 |
{ |
| 63 |
printf(" checking invalid token: %s\n", s); |
| 64 |
|
| 65 |
io::InStringSequence sq; |
| 66 |
UTEST_ASSERT(sq.wrap(s) == STATUS_OK); |
| 67 |
|
| 68 |
Tokenizer t(&sq); |
| 69 |
token_t tok = t.get_token(true); |
| 70 |
UTEST_ASSERT(tok == token); |
| 71 |
} |
| 72 |
|
| 73 |
void test_basic_tokens() |
| 74 |
{ |
| 75 |
static const char *tokens = |
| 76 |
" [ ] { } : , 'sq_string' \"dq_string\" " |
| 77 |
" identifier true false null break " |
| 78 |
" Infinity NaN " |
| 79 |
" //comment\n\r /*Multiline\n\rcomment*/ "; |
| 80 |
|
| 81 |
io::InStringSequence sq; |
| 82 |
UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); |
| 83 |
Tokenizer t(&sq); |
| 84 |
|
| 85 |
ck_token(t, "[", JT_LQ_BRACE); |
| 86 |
ck_token(t, "]", JT_RQ_BRACE); |
| 87 |
ck_token(t, "{", JT_LC_BRACE); |
| 88 |
ck_token(t, "}", JT_RC_BRACE); |
| 89 |
ck_token(t, ":", JT_COLON); |
| 90 |
ck_token(t, ",", JT_COMMA); |
| 91 |
ck_token(t, "sq_string", JT_SQ_STRING); |
| 92 |
ck_token(t, "dq_string", JT_DQ_STRING); |
| 93 |
ck_token(t, "identifier", JT_IDENTIFIER); |
| 94 |
ck_token(t, "true", JT_TRUE); |
| 95 |
ck_token(t, "false", JT_FALSE); |
| 96 |
ck_token(t, "null", JT_NULL); |
| 97 |
ck_token(t, "break", JT_RESERVED); |
| 98 |
ck_token(t, "Infinity", JT_DOUBLE); |
| 99 |
ck_token(t, "NaN", JT_DOUBLE); |
| 100 |
ck_token(t, "comment", JT_SL_COMMENT); |
| 101 |
ck_token(t, "Multiline\ncomment", JT_ML_COMMENT); |
| 102 |
|
| 103 |
UTEST_ASSERT(t.get_token(true) == JT_EOF); |
| 104 |
} |
| 105 |
|
| 106 |
void test_string_tokens() |
| 107 |
{ |
| 108 |
static const char *tokens = |
| 109 |
// "'Lorem ipsum dolor sit amet, \\\n\rconsectetur adipiscing elit.'\n\r" |
| 110 |
// "'\\n\\t\\r\\f\"\\\"\\\'\\v\\b\\f'\n\r" |
| 111 |
"'\\x20\\X41\\u000a\\U000D\\u042F\\uD83C\\uDFBC'\n\r" |
| 112 |
"'\\x20\\X41\\u000a\\U000D\\u042F\\uD83C\\uDFBCx'\n\r" |
| 113 |
"\"\\A\\C/\\D\\C\'\\\'\\\"\"\n\r" |
| 114 |
"\"\\0\"\n\r"; |
| 115 |
|
| 116 |
io::InStringSequence sq; |
| 117 |
UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); |
| 118 |
Tokenizer t(&sq); |
| 119 |
|
| 120 |
// ck_token(t, "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", JT_SQ_STRING); |
| 121 |
// ck_token(t, "\n\t\r\f\"\"\'\v\b\f", JT_SQ_STRING); |
| 122 |
ck_token(t, " A\n\rЯ\xF0\x9F\x8E\xBC", JT_SQ_STRING); |
| 123 |
ck_token(t, " A\n\rЯ\xF0\x9F\x8E\xBCx", JT_SQ_STRING); |
| 124 |
ck_token(t, "AC/DC\'\'\"", JT_DQ_STRING); |
| 125 |
|
| 126 |
UTEST_ASSERT(t.get_token(true) == JT_DQ_STRING); |
| 127 |
UTEST_ASSERT(t.text_value()->length() == 1); |
| 128 |
UTEST_ASSERT(t.text_value()->char_at(0) == '\0'); |
| 129 |
|
| 130 |
UTEST_ASSERT(t.get_token(true) == JT_EOF); |
| 131 |
} |
| 132 |
|
| 133 |
void test_identifier_tokens() |
| 134 |
{ |
| 135 |
static const char *tokens = |
| 136 |
"Identifier_001\n\r" |
| 137 |
// "Identifier\\u0041\\u0042\n\r" |
| 138 |
// "Identifier\\u0041\\u0042C\n\r" |
| 139 |
"$valid_identifier$\n\r" |
| 140 |
"\\u042F8"; |
| 141 |
|
| 142 |
io::InStringSequence sq; |
| 143 |
UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); |
| 144 |
Tokenizer t(&sq); |
| 145 |
|
| 146 |
// char *locale = ::setlocale(LC_ALL, NULL); |
| 147 |
// ::setlocale(LC_ALL, "ru_RU.UTF-8"); |
| 148 |
ck_token(t, "Identifier_001", JT_IDENTIFIER); |
| 149 |
// ck_token(t, "IdentifierAB", JT_IDENTIFIER); |
| 150 |
// ck_token(t, "IdentifierABC", JT_IDENTIFIER); |
| 151 |
ck_token(t, "$valid_identifier$", JT_IDENTIFIER); |
| 152 |
ck_token(t, "Я8", JT_IDENTIFIER); |
| 153 |
|
| 154 |
UTEST_ASSERT(t.get_token(true) == JT_EOF); |
| 155 |
|
| 156 |
// ::setlocale(LC_ALL, locale); |
| 157 |
} |
| 158 |
|
| 159 |
void test_numeric_tokens() |
| 160 |
{ |
| 161 |
static const char *tokens = |
| 162 |
"0\n\r" |
| 163 |
"10\n\r" |
| 164 |
"+20\n\r" |
| 165 |
"-20\n\r" |
| 166 |
"0x10\n\r" |
| 167 |
"+0x20\n\r" |
| 168 |
"-0x20\n\r" |
| 169 |
"123.\n\r" |
| 170 |
".123\n\r" |
| 171 |
"12.34\n\r" |
| 172 |
"12.34E+1\n\r" |
| 173 |
"12.34e-1\n\r" |
| 174 |
"1e-1\n\r" |
| 175 |
"2e+1\n\r" |
| 176 |
"+Infinity\n\r" |
| 177 |
"-Infinity\n\r" |
| 178 |
"+NaN\n\r" |
| 179 |
"-NaN\n\r" |
| 180 |
; |
| 181 |
|
| 182 |
io::InStringSequence sq; |
| 183 |
UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); |
| 184 |
Tokenizer t(&sq); |
| 185 |
|
| 186 |
ck_int(t, 0); |
| 187 |
ck_int(t, 10); |
| 188 |
ck_int(t, +20); |
| 189 |
ck_int(t, -20); |
| 190 |
ck_int(t, 0x10); |
| 191 |
ck_int(t, 0x20); |
| 192 |
ck_int(t, -0x20); |
| 193 |
ck_float(t, 123.0); |
| 194 |
ck_float(t, 0.123); |
| 195 |
ck_float(t, 12.34); |
| 196 |
ck_float(t, 12.34E+1); |
| 197 |
ck_float(t, 12.34E-1); |
| 198 |
ck_float(t, 1E-1); |
| 199 |
ck_float(t, 2E+1); |
| 200 |
|
| 201 |
UTEST_ASSERT(t.get_token(true) == JT_DOUBLE); |
| 202 |
UTEST_ASSERT(isinf(t.float_value()) && (t.float_value() > 0.0)); |
| 203 |
UTEST_ASSERT(t.get_token(true) == JT_DOUBLE); |
| 204 |
UTEST_ASSERT(isinf(t.float_value()) && (t.float_value() < 0.0)); |
| 205 |
UTEST_ASSERT(t.get_token(true) == JT_DOUBLE); |
| 206 |
UTEST_ASSERT(isnan(t.float_value())); |
| 207 |
UTEST_ASSERT(t.get_token(true) == JT_DOUBLE); |
| 208 |
UTEST_ASSERT(isnan(t.float_value())); |
| 209 |
|
| 210 |
UTEST_ASSERT(t.get_token(true) == JT_EOF); |
| 211 |
} |
| 212 |
|
| 213 |
void test_unicode_comments() |
| 214 |
{ |
| 215 |
static const char *tokens = |
| 216 |
"// \\u0041\\u0042\n\r" |
| 217 |
"/* \\u0041\\u0042 */\n\r" |
| 218 |
"//\\uD83C\\uDFBC\n\r" |
| 219 |
"/*\\uD83C\\uDFBC*/\n\r" |
| 220 |
"//\\uD83C\\uDFBCX\n\r" |
| 221 |
"/*\\uD83C\\uDFBCX*/\n\r" |
| 222 |
; |
| 223 |
|
| 224 |
io::InStringSequence sq; |
| 225 |
UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK); |
| 226 |
Tokenizer t(&sq); |
| 227 |
ck_token(t, " AB", JT_SL_COMMENT); |
| 228 |
ck_token(t, " AB ", JT_ML_COMMENT); |
| 229 |
ck_token(t, "\xF0\x9F\x8E\xBC", JT_SL_COMMENT); |
| 230 |
ck_token(t, "\xF0\x9F\x8E\xBC", JT_ML_COMMENT); |
| 231 |
ck_token(t, "\xF0\x9F\x8E\xBCX", JT_SL_COMMENT); |
| 232 |
ck_token(t, "\xF0\x9F\x8E\xBCX", JT_ML_COMMENT); |
| 233 |
} |
| 234 |
|
| 235 |
void test_invalid_tokens() |
| 236 |
{ |
| 237 |
ck_invalid("\"", JT_ERROR); |
| 238 |
ck_invalid("\'", JT_ERROR); |
| 239 |
ck_invalid("\\", JT_ERROR); |
| 240 |
ck_invalid("\"\\\"", JT_ERROR); |
| 241 |
ck_invalid("'\\x1'", JT_ERROR); |
| 242 |
ck_invalid("'\\u1'", JT_ERROR); |
| 243 |
ck_invalid("'\\u12'", JT_ERROR); |
| 244 |
ck_invalid("'\\u123'", JT_ERROR); |
| 245 |
ck_invalid("\\u123", JT_ERROR); |
| 246 |
ck_invalid("\\u12", JT_ERROR); |
| 247 |
ck_invalid("'\n\r'", JT_ERROR); |
| 248 |
ck_invalid("+ 1", JT_UNKNOWN); |
| 249 |
ck_invalid("- 1", JT_UNKNOWN); |
| 250 |
ck_invalid(".", JT_UNKNOWN); |
| 251 |
ck_invalid("12identifier", JT_ERROR); |
| 252 |
ck_invalid("/* test comment", JT_ERROR); |
| 253 |
} |
| 254 |
|
| 255 |
UTEST_MAIN |
| 256 |
{ |
| 257 |
printf("Testing basic tokens...\n"); |
| 258 |
// test_basic_tokens(); |
| 259 |
printf("Testing string tokens...\n"); |
| 260 |
test_string_tokens(); |
| 261 |
printf("Testing identifier tokens...\n"); |
| 262 |
test_identifier_tokens(); |
| 263 |
printf("Testing numeric tokens...\n"); |
| 264 |
test_numeric_tokens(); |
| 265 |
printf("Testing unicode comments...\n"); |
| 266 |
test_unicode_comments(); |
| 267 |
printf("Testing invalid tokens...\n"); |
| 268 |
test_invalid_tokens(); |
| 269 |
} |
| 270 |
|
| 271 |
UTEST_END |