Debian Codegraph @

modules/lsp-runtime-lib/src/test/utest/fmt/json/tokenizer.cpp

raw
1 /*
2 * Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/>
3 * (C) 2020 Vladimir Sadovnikov <[email protected]>
4 *
5 * This file is part of lsp-runtime-lib
6 * Created on: 15 окт. 2019 г.
7 *
8 * lsp-runtime-lib is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * any later version.
12 *
13 * lsp-runtime-lib is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with lsp-runtime-lib. If not, see <https://www.gnu.org/licenses/>.
20 */
21
22 #include <lsp-plug.in/test-fw/utest.h>
23 #include <lsp-plug.in/test-fw/helpers.h>
24 #include <lsp-plug.in/io/InStringSequence.h>
25 #include <lsp-plug.in/fmt/json/token.h>
26 #include <lsp-plug.in/fmt/json/Tokenizer.h>
27 #include <lsp-plug.in/stdlib/math.h>
28 //#include <locale.h>
29
30 using namespace lsp;
31 using namespace lsp::json;
32
33 UTEST_BEGIN("runtime.fmt.json", tokenizer)
34
35 void ck_token(Tokenizer &t, const char *s, token_t token)
36 {
37 printf(" checking token: %s\n", s);
38 token_t tok = t.get_token(true);
39 UTEST_ASSERT_MSG(tok == token, "Error testing token: %s", s);
40 LSPString tmp;
41 UTEST_ASSERT(tmp.set_utf8(s));
42 UTEST_ASSERT_MSG(t.text_value()->equals(&tmp), "Error testing token: %s, got: %s", s, t.text_value()->get_utf8());
43 }
44
45 void ck_int(Tokenizer &t, ssize_t value)
46 {
47 printf(" checking integer: %d\n", int(value));
48 token_t tok = t.get_token(true);
49 UTEST_ASSERT_MSG((tok == JT_DECIMAL) || (tok == JT_HEXADECIMAL), "Error testing token: not decimal/hexadecimal value");
50 UTEST_ASSERT_MSG(t.int_value() == value, "Error testing token: %d != %d", int(t.int_value()), int(value));
51 }
52
53 void ck_float(Tokenizer &t, double value)
54 {
55 printf(" checking float: %f\n", double(value));
56 token_t tok = t.get_token(true);
57 UTEST_ASSERT_MSG(tok == JT_DOUBLE, "Error testing token: not double value");
58 UTEST_ASSERT_MSG(float_equals_relative(t.float_value(), value), "Error testing token: %f != %f", t.float_value(), value);
59 }
60
61 void ck_invalid(const char *s, token_t token)
62 {
63 printf(" checking invalid token: %s\n", s);
64
65 io::InStringSequence sq;
66 UTEST_ASSERT(sq.wrap(s) == STATUS_OK);
67
68 Tokenizer t(&sq);
69 token_t tok = t.get_token(true);
70 UTEST_ASSERT(tok == token);
71 }
72
73 void test_basic_tokens()
74 {
75 static const char *tokens =
76 " [ ] { } : , 'sq_string' \"dq_string\" "
77 " identifier true false null break "
78 " Infinity NaN "
79 " //comment\n\r /*Multiline\n\rcomment*/ ";
80
81 io::InStringSequence sq;
82 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
83 Tokenizer t(&sq);
84
85 ck_token(t, "[", JT_LQ_BRACE);
86 ck_token(t, "]", JT_RQ_BRACE);
87 ck_token(t, "{", JT_LC_BRACE);
88 ck_token(t, "}", JT_RC_BRACE);
89 ck_token(t, ":", JT_COLON);
90 ck_token(t, ",", JT_COMMA);
91 ck_token(t, "sq_string", JT_SQ_STRING);
92 ck_token(t, "dq_string", JT_DQ_STRING);
93 ck_token(t, "identifier", JT_IDENTIFIER);
94 ck_token(t, "true", JT_TRUE);
95 ck_token(t, "false", JT_FALSE);
96 ck_token(t, "null", JT_NULL);
97 ck_token(t, "break", JT_RESERVED);
98 ck_token(t, "Infinity", JT_DOUBLE);
99 ck_token(t, "NaN", JT_DOUBLE);
100 ck_token(t, "comment", JT_SL_COMMENT);
101 ck_token(t, "Multiline\ncomment", JT_ML_COMMENT);
102
103 UTEST_ASSERT(t.get_token(true) == JT_EOF);
104 }
105
106 void test_string_tokens()
107 {
108 static const char *tokens =
109 // "'Lorem ipsum dolor sit amet, \\\n\rconsectetur adipiscing elit.'\n\r"
110 // "'\\n\\t\\r\\f\"\\\"\\\'\\v\\b\\f'\n\r"
111 "'\\x20\\X41\\u000a\\U000D\\u042F\\uD83C\\uDFBC'\n\r"
112 "'\\x20\\X41\\u000a\\U000D\\u042F\\uD83C\\uDFBCx'\n\r"
113 "\"\\A\\C/\\D\\C\'\\\'\\\"\"\n\r"
114 "\"\\0\"\n\r";
115
116 io::InStringSequence sq;
117 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
118 Tokenizer t(&sq);
119
120 // ck_token(t, "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", JT_SQ_STRING);
121 // ck_token(t, "\n\t\r\f\"\"\'\v\b\f", JT_SQ_STRING);
122 ck_token(t, " A\n\rЯ\xF0\x9F\x8E\xBC", JT_SQ_STRING);
123 ck_token(t, " A\n\rЯ\xF0\x9F\x8E\xBCx", JT_SQ_STRING);
124 ck_token(t, "AC/DC\'\'\"", JT_DQ_STRING);
125
126 UTEST_ASSERT(t.get_token(true) == JT_DQ_STRING);
127 UTEST_ASSERT(t.text_value()->length() == 1);
128 UTEST_ASSERT(t.text_value()->char_at(0) == '\0');
129
130 UTEST_ASSERT(t.get_token(true) == JT_EOF);
131 }
132
133 void test_identifier_tokens()
134 {
135 static const char *tokens =
136 "Identifier_001\n\r"
137 // "Identifier\\u0041\\u0042\n\r"
138 // "Identifier\\u0041\\u0042C\n\r"
139 "$valid_identifier$\n\r"
140 "\\u042F8";
141
142 io::InStringSequence sq;
143 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
144 Tokenizer t(&sq);
145
146 // char *locale = ::setlocale(LC_ALL, NULL);
147 // ::setlocale(LC_ALL, "ru_RU.UTF-8");
148 ck_token(t, "Identifier_001", JT_IDENTIFIER);
149 // ck_token(t, "IdentifierAB", JT_IDENTIFIER);
150 // ck_token(t, "IdentifierABC", JT_IDENTIFIER);
151 ck_token(t, "$valid_identifier$", JT_IDENTIFIER);
152 ck_token(t, "Я8", JT_IDENTIFIER);
153
154 UTEST_ASSERT(t.get_token(true) == JT_EOF);
155
156 // ::setlocale(LC_ALL, locale);
157 }
158
159 void test_numeric_tokens()
160 {
161 static const char *tokens =
162 "0\n\r"
163 "10\n\r"
164 "+20\n\r"
165 "-20\n\r"
166 "0x10\n\r"
167 "+0x20\n\r"
168 "-0x20\n\r"
169 "123.\n\r"
170 ".123\n\r"
171 "12.34\n\r"
172 "12.34E+1\n\r"
173 "12.34e-1\n\r"
174 "1e-1\n\r"
175 "2e+1\n\r"
176 "+Infinity\n\r"
177 "-Infinity\n\r"
178 "+NaN\n\r"
179 "-NaN\n\r"
180 ;
181
182 io::InStringSequence sq;
183 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
184 Tokenizer t(&sq);
185
186 ck_int(t, 0);
187 ck_int(t, 10);
188 ck_int(t, +20);
189 ck_int(t, -20);
190 ck_int(t, 0x10);
191 ck_int(t, 0x20);
192 ck_int(t, -0x20);
193 ck_float(t, 123.0);
194 ck_float(t, 0.123);
195 ck_float(t, 12.34);
196 ck_float(t, 12.34E+1);
197 ck_float(t, 12.34E-1);
198 ck_float(t, 1E-1);
199 ck_float(t, 2E+1);
200
201 UTEST_ASSERT(t.get_token(true) == JT_DOUBLE);
202 UTEST_ASSERT(isinf(t.float_value()) && (t.float_value() > 0.0));
203 UTEST_ASSERT(t.get_token(true) == JT_DOUBLE);
204 UTEST_ASSERT(isinf(t.float_value()) && (t.float_value() < 0.0));
205 UTEST_ASSERT(t.get_token(true) == JT_DOUBLE);
206 UTEST_ASSERT(isnan(t.float_value()));
207 UTEST_ASSERT(t.get_token(true) == JT_DOUBLE);
208 UTEST_ASSERT(isnan(t.float_value()));
209
210 UTEST_ASSERT(t.get_token(true) == JT_EOF);
211 }
212
213 void test_unicode_comments()
214 {
215 static const char *tokens =
216 "// \\u0041\\u0042\n\r"
217 "/* \\u0041\\u0042 */\n\r"
218 "//\\uD83C\\uDFBC\n\r"
219 "/*\\uD83C\\uDFBC*/\n\r"
220 "//\\uD83C\\uDFBCX\n\r"
221 "/*\\uD83C\\uDFBCX*/\n\r"
222 ;
223
224 io::InStringSequence sq;
225 UTEST_ASSERT(sq.wrap(tokens, "UTF-8") == STATUS_OK);
226 Tokenizer t(&sq);
227 ck_token(t, " AB", JT_SL_COMMENT);
228 ck_token(t, " AB ", JT_ML_COMMENT);
229 ck_token(t, "\xF0\x9F\x8E\xBC", JT_SL_COMMENT);
230 ck_token(t, "\xF0\x9F\x8E\xBC", JT_ML_COMMENT);
231 ck_token(t, "\xF0\x9F\x8E\xBCX", JT_SL_COMMENT);
232 ck_token(t, "\xF0\x9F\x8E\xBCX", JT_ML_COMMENT);
233 }
234
235 void test_invalid_tokens()
236 {
237 ck_invalid("\"", JT_ERROR);
238 ck_invalid("\'", JT_ERROR);
239 ck_invalid("\\", JT_ERROR);
240 ck_invalid("\"\\\"", JT_ERROR);
241 ck_invalid("'\\x1'", JT_ERROR);
242 ck_invalid("'\\u1'", JT_ERROR);
243 ck_invalid("'\\u12'", JT_ERROR);
244 ck_invalid("'\\u123'", JT_ERROR);
245 ck_invalid("\\u123", JT_ERROR);
246 ck_invalid("\\u12", JT_ERROR);
247 ck_invalid("'\n\r'", JT_ERROR);
248 ck_invalid("+ 1", JT_UNKNOWN);
249 ck_invalid("- 1", JT_UNKNOWN);
250 ck_invalid(".", JT_UNKNOWN);
251 ck_invalid("12identifier", JT_ERROR);
252 ck_invalid("/* test comment", JT_ERROR);
253 }
254
255 UTEST_MAIN
256 {
257 printf("Testing basic tokens...\n");
258 // test_basic_tokens();
259 printf("Testing string tokens...\n");
260 test_string_tokens();
261 printf("Testing identifier tokens...\n");
262 test_identifier_tokens();
263 printf("Testing numeric tokens...\n");
264 test_numeric_tokens();
265 printf("Testing unicode comments...\n");
266 test_unicode_comments();
267 printf("Testing invalid tokens...\n");
268 test_invalid_tokens();
269 }
270
271 UTEST_END