| 1 |
// Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 3 |
|
| 4 |
#include <assert.h> |
| 5 |
|
| 6 |
#include "test_util.h" |
| 7 |
#include "../array.h" |
| 8 |
#include "../c_lexer.h" |
| 9 |
#include "../lexer.h" |
| 10 |
#include "../pp.h" |
| 11 |
|
| 12 |
#suite language_c |
| 13 |
|
| 14 |
#tcase lexer |
| 15 |
|
| 16 |
struct test_token { |
| 17 |
int kind; |
| 18 |
const char *s; |
| 19 |
}; |
| 20 |
|
| 21 |
static void assert_lexes(const char *s, const struct test_token *tokens, |
| 22 |
bool cpp) |
| 23 |
{ |
| 24 |
DRGN_C_FAMILY_LEXER(lexer, s, cpp); |
| 25 |
for (;;) { |
| 26 |
struct drgn_token token; |
| 27 |
drgn_ck_no_err(drgn_lexer_pop(&lexer.lexer, &token)); |
| 28 |
ck_assert_int_eq(token.kind, tokens->kind); |
| 29 |
ck_assert_uint_eq(token.len, strlen(tokens->s)); |
| 30 |
ck_assert_mem_eq(token.value, tokens->s, strlen(tokens->s)); |
| 31 |
if (tokens->kind == C_TOKEN_EOF) |
| 32 |
break; |
| 33 |
tokens++; |
| 34 |
} |
| 35 |
} |
| 36 |
|
| 37 |
#define assert_lexes_c(s, tokens) assert_lexes(s, tokens, false) |
| 38 |
#define assert_lexes_cpp(s, tokens) assert_lexes(s, tokens, true) |
| 39 |
|
| 40 |
static const char c_keywords[] = |
| 41 |
"void char short int long signed unsigned _Bool float double " |
| 42 |
"const restrict volatile _Atomic struct union class enum"; |
| 43 |
|
| 44 |
#test lexer_empty |
| 45 |
{ |
| 46 |
DRGN_C_FAMILY_LEXER(lexer, "", false); |
| 47 |
for (int i = 0; i < 64; i++) { |
| 48 |
struct drgn_token token; |
| 49 |
drgn_ck_no_err(drgn_lexer_pop(&lexer.lexer, &token)); |
| 50 |
ck_assert_int_eq(token.kind, C_TOKEN_EOF); |
| 51 |
ck_assert_uint_eq(token.len, 0); |
| 52 |
} |
| 53 |
} |
| 54 |
|
| 55 |
#test lexer_symbols |
| 56 |
{ |
| 57 |
static const char s[] = "()[]*."; |
| 58 |
static struct test_token tokens[] = { |
| 59 |
{ C_TOKEN_LPAREN, "(" }, |
| 60 |
{ C_TOKEN_RPAREN, ")" }, |
| 61 |
{ C_TOKEN_LBRACKET, "["}, |
| 62 |
{ C_TOKEN_RBRACKET, "]"}, |
| 63 |
{ C_TOKEN_ASTERISK, "*" }, |
| 64 |
{ C_TOKEN_DOT, "." }, |
| 65 |
{ C_TOKEN_EOF, "" }, |
| 66 |
}; |
| 67 |
assert_lexes_c(s, tokens); |
| 68 |
} |
| 69 |
|
| 70 |
#test lexer_keywords |
| 71 |
{ |
| 72 |
static const struct test_token tokens[] = { |
| 73 |
{ C_TOKEN_VOID, "void" }, |
| 74 |
{ C_TOKEN_CHAR, "char" }, |
| 75 |
{ C_TOKEN_SHORT, "short" }, |
| 76 |
{ C_TOKEN_INT, "int" }, |
| 77 |
{ C_TOKEN_LONG, "long" }, |
| 78 |
{ C_TOKEN_SIGNED, "signed" }, |
| 79 |
{ C_TOKEN_UNSIGNED, "unsigned" }, |
| 80 |
{ C_TOKEN_BOOL, "_Bool" }, |
| 81 |
{ C_TOKEN_FLOAT, "float" }, |
| 82 |
{ C_TOKEN_DOUBLE, "double" }, |
| 83 |
{ C_TOKEN_CONST, "const" }, |
| 84 |
{ C_TOKEN_RESTRICT, "restrict" }, |
| 85 |
{ C_TOKEN_VOLATILE, "volatile" }, |
| 86 |
{ C_TOKEN_ATOMIC, "_Atomic" }, |
| 87 |
{ C_TOKEN_STRUCT, "struct" }, |
| 88 |
{ C_TOKEN_UNION, "union" }, |
| 89 |
{ C_TOKEN_IDENTIFIER, "class" }, // Not C_TOKEN_CLASS |
| 90 |
{ C_TOKEN_ENUM, "enum" }, |
| 91 |
{ C_TOKEN_EOF, "" }, |
| 92 |
}; |
| 93 |
static_assert(array_size(tokens) |
| 94 |
== MAX_KEYWORD_TOKEN - MIN_KEYWORD_TOKEN + 2, |
| 95 |
"keyword tokens changed"); |
| 96 |
assert_lexes_c(c_keywords, tokens); |
| 97 |
} |
| 98 |
|
| 99 |
#test lexer_keywords_cpp |
| 100 |
{ |
| 101 |
static const struct test_token tokens[] = { |
| 102 |
{ C_TOKEN_VOID, "void" }, |
| 103 |
{ C_TOKEN_CHAR, "char" }, |
| 104 |
{ C_TOKEN_SHORT, "short" }, |
| 105 |
{ C_TOKEN_INT, "int" }, |
| 106 |
{ C_TOKEN_LONG, "long" }, |
| 107 |
{ C_TOKEN_SIGNED, "signed" }, |
| 108 |
{ C_TOKEN_UNSIGNED, "unsigned" }, |
| 109 |
{ C_TOKEN_BOOL, "_Bool" }, |
| 110 |
{ C_TOKEN_FLOAT, "float" }, |
| 111 |
{ C_TOKEN_DOUBLE, "double" }, |
| 112 |
{ C_TOKEN_CONST, "const" }, |
| 113 |
{ C_TOKEN_RESTRICT, "restrict" }, |
| 114 |
{ C_TOKEN_VOLATILE, "volatile" }, |
| 115 |
{ C_TOKEN_ATOMIC, "_Atomic" }, |
| 116 |
{ C_TOKEN_STRUCT, "struct" }, |
| 117 |
{ C_TOKEN_UNION, "union" }, |
| 118 |
{ C_TOKEN_CLASS, "class" }, |
| 119 |
{ C_TOKEN_ENUM, "enum" }, |
| 120 |
{ C_TOKEN_EOF, "" }, |
| 121 |
}; |
| 122 |
static_assert(array_size(tokens) |
| 123 |
== MAX_KEYWORD_TOKEN - MIN_KEYWORD_TOKEN + 2, |
| 124 |
"keyword tokens changed"); |
| 125 |
assert_lexes_cpp(c_keywords, tokens); |
| 126 |
} |
| 127 |
|
| 128 |
#test lexer_identifiers |
| 129 |
{ |
| 130 |
#define concat_space(_, x) x " " |
| 131 |
#define identifier_token(_, x) { C_TOKEN_IDENTIFIER, x }, |
| 132 |
|
| 133 |
#define identifiers "_", "x", "foo", "_bar", "baz1" |
| 134 |
static const char s[] = PP_MAP(concat_space,, identifiers); |
| 135 |
static const struct test_token tokens[] = { |
| 136 |
PP_MAP(identifier_token,, identifiers) |
| 137 |
{ C_TOKEN_EOF, "" }, |
| 138 |
}; |
| 139 |
#undef identifiers |
| 140 |
assert_lexes_c(s, tokens); |
| 141 |
} |
| 142 |
|
| 143 |
#test lexer_almost_keywords |
| 144 |
{ |
| 145 |
#define almost_keywords \ |
| 146 |
"voi", "cha", "shor", "in", "lon", "signe", "unsigne", "_Boo", \ |
| 147 |
"floa", "doubl", "_Comple", "cons", "restric", "volatil", \ |
| 148 |
"_Atomi", "struc", "unio", "clas", "enu" |
| 149 |
static const char s[] = PP_MAP(concat_space,, almost_keywords); |
| 150 |
static const struct test_token tokens[] = { |
| 151 |
PP_MAP(identifier_token,, almost_keywords) |
| 152 |
{ C_TOKEN_EOF, "" }, |
| 153 |
}; |
| 154 |
#undef almost_keywords |
| 155 |
assert_lexes_c(s, tokens); |
| 156 |
} |
| 157 |
|
| 158 |
#test lexer_identifiers_with_template_arguments |
| 159 |
{ |
| 160 |
#define concat2_with_spaces(_, x) concat2_with_spaces_impl x |
| 161 |
#define concat2_with_spaces_impl(a, b) a " " b " " |
| 162 |
#define identifier_and_template_arguments_tokens(_, x) \ |
| 163 |
identifier_and_template_arguments_tokens_impl x |
| 164 |
#define identifier_and_template_arguments_tokens_impl(a, b) \ |
| 165 |
{ C_TOKEN_IDENTIFIER, a }, { C_TOKEN_TEMPLATE_ARGUMENTS, b }, |
| 166 |
|
| 167 |
#define pairs \ |
| 168 |
("vector", "<int>"), \ |
| 169 |
("pair", "<int, double>"), \ |
| 170 |
("unordered_map", "<std::string, std::vector<pair<short, bool>>>"), \ |
| 171 |
("IntLiteral", "<123>"), \ |
| 172 |
("IntLiteralWithSuffix", "<123UL>"), \ |
| 173 |
("FloatLiteral", "<1.987>"), \ |
| 174 |
("FloatLiteralWithExponent", "<1.23423e+1f>"), \ |
| 175 |
("PointerLiteral", "<&asdf>"), \ |
| 176 |
("ParenthesizedPointerLiteral", "<(&asdf)>"), \ |
| 177 |
("CharLiteral", "<'a'>"), \ |
| 178 |
("CharLiteralEdgeCase1", "<'<'>"), \ |
| 179 |
("CharLiteralEdgeCase2", "<'>'>"), \ |
| 180 |
("CharLiteralEdgeCase3", "<'\\''>") |
| 181 |
static const char s[] = PP_MAP(concat2_with_spaces,, pairs); |
| 182 |
static const struct test_token tokens[] = { |
| 183 |
PP_MAP(identifier_and_template_arguments_tokens,, pairs) |
| 184 |
{ C_TOKEN_EOF, "" }, |
| 185 |
}; |
| 186 |
#undef pairs |
| 187 |
assert_lexes_cpp(s, tokens); |
| 188 |
} |
| 189 |
|
| 190 |
#test lexer_identifiers_with_invalid_template_arguments |
| 191 |
{ |
| 192 |
static const char * const s[] = { |
| 193 |
"vector<int", |
| 194 |
"pair<<int, double>", |
| 195 |
"unordered_map<string, vector<pair<short, bool>>", |
| 196 |
}; |
| 197 |
for (size_t i = 0; i < array_size(s); i++) { |
| 198 |
DRGN_C_FAMILY_LEXER(lexer, s[i], true); |
| 199 |
struct drgn_token token; |
| 200 |
struct drgn_error *err; |
| 201 |
do { |
| 202 |
err = drgn_lexer_pop(&lexer.lexer, &token); |
| 203 |
} while (!err && token.kind != C_TOKEN_EOF); |
| 204 |
ck_assert_ptr_nonnull(err); |
| 205 |
ck_assert_int_eq(err->code, DRGN_ERROR_SYNTAX); |
| 206 |
ck_assert(strstr(err->message, "invalid template arguments")); |
| 207 |
drgn_error_destroy(err); |
| 208 |
} |
| 209 |
} |
| 210 |
|
| 211 |
#test lexer_template_arguments_with_extra_right_brackets |
| 212 |
{ |
| 213 |
static const char * const s[] = { |
| 214 |
"vectorint>", |
| 215 |
"pair<int, double>>", |
| 216 |
}; |
| 217 |
for (size_t i = 0; i < array_size(s); i++) { |
| 218 |
DRGN_C_FAMILY_LEXER(lexer, s[i], true); |
| 219 |
struct drgn_token token; |
| 220 |
struct drgn_error *err; |
| 221 |
do { |
| 222 |
err = drgn_lexer_pop(&lexer.lexer, &token); |
| 223 |
} while (!err && token.kind != C_TOKEN_EOF); |
| 224 |
ck_assert_ptr_nonnull(err); |
| 225 |
ck_assert_int_eq(err->code, DRGN_ERROR_SYNTAX); |
| 226 |
ck_assert(strstr(err->message, "invalid character")); |
| 227 |
drgn_error_destroy(err); |
| 228 |
} |
| 229 |
} |
| 230 |
|
| 231 |
#test lexer_numbers |
| 232 |
{ |
| 233 |
static const char s[] = "0 1234 0xdeadbeef"; |
| 234 |
static const struct test_token tokens[] = { |
| 235 |
{ C_TOKEN_NUMBER, "0" }, |
| 236 |
{ C_TOKEN_NUMBER, "1234" }, |
| 237 |
{ C_TOKEN_NUMBER, "0xdeadbeef" }, |
| 238 |
{ C_TOKEN_EOF, "" }, |
| 239 |
}; |
| 240 |
assert_lexes_c(s, tokens); |
| 241 |
} |
| 242 |
|
| 243 |
#test lexer_invalid_numbers |
| 244 |
{ |
| 245 |
static const char * const s[] = { "0x", "1234y" }; |
| 246 |
for (size_t i = 0; i < array_size(s); i++) { |
| 247 |
DRGN_C_FAMILY_LEXER(lexer, s[i], false); |
| 248 |
struct drgn_token token; |
| 249 |
struct drgn_error *err = drgn_lexer_pop(&lexer.lexer, &token); |
| 250 |
ck_assert_ptr_nonnull(err); |
| 251 |
ck_assert_int_eq(err->code, DRGN_ERROR_SYNTAX); |
| 252 |
ck_assert(strstr(err->message, "invalid number")); |
| 253 |
drgn_error_destroy(err); |
| 254 |
} |
| 255 |
} |
| 256 |
|
| 257 |
#test lexer_invalid_characters |
| 258 |
{ |
| 259 |
static const char * const s[] = { "@", "$" }; |
| 260 |
for (size_t i = 0; i < array_size(s); i++) { |
| 261 |
DRGN_C_FAMILY_LEXER(lexer, s[i], false); |
| 262 |
struct drgn_token token; |
| 263 |
struct drgn_error *err = drgn_lexer_pop(&lexer.lexer, &token); |
| 264 |
ck_assert_ptr_nonnull(err); |
| 265 |
ck_assert_int_eq(err->code, DRGN_ERROR_SYNTAX); |
| 266 |
ck_assert(strstr(err->message, "invalid character")); |
| 267 |
drgn_error_destroy(err); |
| 268 |
} |
| 269 |
} |