Debian Codegraph @

llvm-project/clang/unittests/Format/TokenAnnotatorTest.cpp

raw
1 //===- unittest/Format/TokenAnnotatorTest.cpp - Formatting unit tests -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "clang/Format/Format.h"
10
11 #include "FormatTestUtils.h"
12 #include "TestLexer.h"
13 #include "gtest/gtest.h"
14
15 namespace clang {
16 namespace format {
17
18 // Not really the equality, but everything we need.
19 static bool operator==(const FormatToken &LHS,
20 const FormatToken &RHS) noexcept {
21 return LHS.Tok.getKind() == RHS.Tok.getKind() &&
22 LHS.getType() == RHS.getType();
23 }
24
25 namespace {
26
27 class TokenAnnotatorTest : public ::testing::Test {
28 protected:
29 TokenList annotate(llvm::StringRef Code,
30 const FormatStyle &Style = getLLVMStyle()) {
31 return TestLexer(Allocator, Buffers, Style).annotate(Code);
32 }
33 llvm::SpecificBumpPtrAllocator<FormatToken> Allocator;
34 std::vector<std::unique_ptr<llvm::MemoryBuffer>> Buffers;
35 };
36
37 #define EXPECT_TOKEN_KIND(FormatTok, Kind) \
38 EXPECT_EQ((FormatTok)->Tok.getKind(), Kind) << *(FormatTok)
39 #define EXPECT_TOKEN_TYPE(FormatTok, Type) \
40 EXPECT_EQ((FormatTok)->getType(), Type) << *(FormatTok)
41 #define EXPECT_TOKEN_PRECEDENCE(FormatTok, Prec) \
42 EXPECT_EQ((FormatTok)->getPrecedence(), Prec) << *(FormatTok)
43 #define EXPECT_TOKEN(FormatTok, Kind, Type) \
44 do { \
45 EXPECT_TOKEN_KIND(FormatTok, Kind); \
46 EXPECT_TOKEN_TYPE(FormatTok, Type); \
47 } while (false)
48
49 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
50 auto Tokens = annotate("auto x = [](const decltype(x) &ptr) {};");
51 EXPECT_EQ(Tokens.size(), 18u) << Tokens;
52 EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
53 EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
54 EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
55 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
56 EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
57
58 Tokens = annotate("auto x = [](const decltype(x) *ptr) {};");
59 EXPECT_EQ(Tokens.size(), 18u) << Tokens;
60 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
61 EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
62
63 Tokens = annotate("#define lambda [](const decltype(x) &ptr) {}");
64 EXPECT_EQ(Tokens.size(), 17u) << Tokens;
65 EXPECT_TOKEN(Tokens[7], tok::kw_decltype, TT_Unknown);
66 EXPECT_TOKEN(Tokens[8], tok::l_paren, TT_TypeDeclarationParen);
67 EXPECT_TOKEN(Tokens[9], tok::identifier, TT_Unknown);
68 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
69 EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
70
71 Tokens = annotate("#define lambda [](const decltype(x) *ptr) {}");
72 EXPECT_EQ(Tokens.size(), 17u) << Tokens;
73 EXPECT_TOKEN(Tokens[10], tok::r_paren, TT_TypeDeclarationParen);
74 EXPECT_TOKEN(Tokens[11], tok::star, TT_PointerOrReference);
75
76 Tokens = annotate("void f() {\n"
77 " while (p < a && *p == 'a')\n"
78 " p++;\n"
79 "}");
80 EXPECT_EQ(Tokens.size(), 21u) << Tokens;
81 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
82 EXPECT_TOKEN(Tokens[11], tok::star, TT_UnaryOperator);
83
84 Tokens = annotate("case *x:");
85 EXPECT_EQ(Tokens.size(), 5u) << Tokens;
86 EXPECT_TOKEN(Tokens[1], tok::star, TT_UnaryOperator);
87 Tokens = annotate("case &x:");
88 EXPECT_EQ(Tokens.size(), 5u) << Tokens;
89 EXPECT_TOKEN(Tokens[1], tok::amp, TT_UnaryOperator);
90
91 Tokens = annotate("bool b = 3 == int{3} && true;\n");
92 EXPECT_EQ(Tokens.size(), 13u) << Tokens;
93 EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
94
95 Tokens = annotate("struct {\n"
96 "} *ptr;");
97 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
98 EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
99 Tokens = annotate("union {\n"
100 "} *ptr;");
101 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
102 EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
103 Tokens = annotate("class {\n"
104 "} *ptr;");
105 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
106 EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
107
108 Tokens = annotate("struct {\n"
109 "} &&ptr = {};");
110 EXPECT_EQ(Tokens.size(), 10u) << Tokens;
111 EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
112 Tokens = annotate("union {\n"
113 "} &&ptr = {};");
114 EXPECT_EQ(Tokens.size(), 10u) << Tokens;
115 EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
116 Tokens = annotate("class {\n"
117 "} &&ptr = {};");
118 EXPECT_EQ(Tokens.size(), 10u) << Tokens;
119 EXPECT_TOKEN(Tokens[3], tok::ampamp, TT_PointerOrReference);
120 Tokens = annotate("int i = int{42} * 2;");
121 EXPECT_EQ(Tokens.size(), 11u) << Tokens;
122 EXPECT_TOKEN(Tokens[7], tok::star, TT_BinaryOperator);
123
124 Tokens = annotate("delete[] *ptr;");
125 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
126 EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
127 Tokens = annotate("delete[] **ptr;");
128 EXPECT_EQ(Tokens.size(), 8u) << Tokens;
129 EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
130 EXPECT_TOKEN(Tokens[4], tok::star, TT_UnaryOperator);
131 Tokens = annotate("delete[] *(ptr);");
132 EXPECT_EQ(Tokens.size(), 9u) << Tokens;
133 EXPECT_TOKEN(Tokens[3], tok::star, TT_UnaryOperator);
134
135 Tokens = annotate("void f() { void (*fnptr)(char* foo); }");
136 EXPECT_EQ(Tokens.size(), 18u) << Tokens;
137 EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
138 // FIXME: The star of a function pointer probably makes more sense as
139 // TT_PointerOrReference.
140 EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
141 EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
142
143 Tokens = annotate("void f() { void (*fnptr)(t* foo); }");
144 EXPECT_EQ(Tokens.size(), 18u) << Tokens;
145 EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_FunctionTypeLParen);
146 EXPECT_TOKEN(Tokens[7], tok::star, TT_UnaryOperator);
147 EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
148
149 Tokens = annotate("int f3() { return sizeof(Foo&); }");
150 ASSERT_EQ(Tokens.size(), 14u) << Tokens;
151 EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
152
153 Tokens = annotate("int f4() { return sizeof(Foo&&); }");
154 ASSERT_EQ(Tokens.size(), 14u) << Tokens;
155 EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
156
157 Tokens = annotate("void f5() { int f6(Foo&, Bar&); }");
158 ASSERT_EQ(Tokens.size(), 17u) << Tokens;
159 EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
160 EXPECT_TOKEN(Tokens[12], tok::amp, TT_PointerOrReference);
161
162 Tokens = annotate("void f7() { int f8(Foo&&, Bar&&); }");
163 ASSERT_EQ(Tokens.size(), 17u) << Tokens;
164 EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_PointerOrReference);
165 EXPECT_TOKEN(Tokens[12], tok::ampamp, TT_PointerOrReference);
166 }
167
168 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
169 auto Tokens = annotate("x - 0");
170 ASSERT_EQ(Tokens.size(), 4u) << Tokens;
171 EXPECT_TOKEN(Tokens[1], tok::minus, TT_BinaryOperator);
172 Tokens = annotate("0 + 0");
173 ASSERT_EQ(Tokens.size(), 4u) << Tokens;
174 EXPECT_TOKEN(Tokens[1], tok::plus, TT_BinaryOperator);
175 Tokens = annotate("x + +0");
176 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
177 EXPECT_TOKEN(Tokens[2], tok::plus, TT_UnaryOperator);
178 Tokens = annotate("x ? -0 : +0");
179 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
180 EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
181 EXPECT_TOKEN(Tokens[5], tok::plus, TT_UnaryOperator);
182 Tokens = annotate("(-0)");
183 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
184 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
185 Tokens = annotate("0, -0");
186 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
187 EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
188 Tokens = annotate("for (; -1;) {\n}");
189 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
190 EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
191 Tokens = annotate("x = -1;");
192 ASSERT_EQ(Tokens.size(), 6u) << Tokens;
193 EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
194 Tokens = annotate("x[-1]");
195 ASSERT_EQ(Tokens.size(), 6u) << Tokens;
196 EXPECT_TOKEN(Tokens[2], tok::minus, TT_UnaryOperator);
197 Tokens = annotate("x = {-1};");
198 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
199 EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
200 Tokens = annotate("case -x:");
201 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
202 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
203 Tokens = annotate("co_await -x;");
204 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
205 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
206 Tokens = annotate("co_return -x;");
207 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
208 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
209 Tokens = annotate("co_yield -x;");
210 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
211 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
212 Tokens = annotate("delete -x;");
213 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
214 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
215 Tokens = annotate("return -x;");
216 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
217 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
218 Tokens = annotate("throw -x;");
219 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
220 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
221 Tokens = annotate("sizeof -x");
222 ASSERT_EQ(Tokens.size(), 4u) << Tokens;
223 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
224 Tokens = annotate("co_await +x;");
225 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
226 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
227 Tokens = annotate("co_return +x;");
228 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
229 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
230 Tokens = annotate("co_yield +x;");
231 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
232 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
233 Tokens = annotate("delete +x;");
234 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
235 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
236 Tokens = annotate("return +x;");
237 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
238 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
239 Tokens = annotate("throw +x;");
240 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
241 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
242 Tokens = annotate("sizeof +x");
243 ASSERT_EQ(Tokens.size(), 4u) << Tokens;
244 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
245 Tokens = annotate("(int)-x");
246 ASSERT_EQ(Tokens.size(), 6u) << Tokens;
247 EXPECT_TOKEN(Tokens[3], tok::minus, TT_UnaryOperator);
248 Tokens = annotate("(-x)");
249 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
250 EXPECT_TOKEN(Tokens[1], tok::minus, TT_UnaryOperator);
251 Tokens = annotate("!+x");
252 ASSERT_EQ(Tokens.size(), 4u) << Tokens;
253 EXPECT_TOKEN(Tokens[0], tok::exclaim, TT_UnaryOperator);
254 EXPECT_TOKEN(Tokens[1], tok::plus, TT_UnaryOperator);
255 }
256
257 TEST_F(TokenAnnotatorTest, UnderstandsClasses) {
258 auto Tokens = annotate("class C {};");
259 EXPECT_EQ(Tokens.size(), 6u) << Tokens;
260 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
261
262 Tokens = annotate("const class C {} c;");
263 EXPECT_EQ(Tokens.size(), 8u) << Tokens;
264 EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_ClassLBrace);
265
266 Tokens = annotate("const class {} c;");
267 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
268 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_ClassLBrace);
269
270 Tokens = annotate("class [[deprecated(\"\")]] C { int i; };");
271 EXPECT_EQ(Tokens.size(), 17u) << Tokens;
272 EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_ClassLBrace);
273 }
274
275 TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
276 auto Tokens = annotate("struct S {};");
277 EXPECT_EQ(Tokens.size(), 6u) << Tokens;
278 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_StructLBrace);
279
280 Tokens = annotate("struct EXPORT_MACRO [[nodiscard]] C { int i; };");
281 EXPECT_EQ(Tokens.size(), 15u) << Tokens;
282 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_StructLBrace);
283
284 Tokens = annotate("struct [[deprecated]] [[nodiscard]] C { int i; };");
285 EXPECT_EQ(Tokens.size(), 19u) << Tokens;
286 EXPECT_TOKEN(Tokens[12], tok::l_brace, TT_StructLBrace);
287 }
288
289 TEST_F(TokenAnnotatorTest, UnderstandsUnions) {
290 auto Tokens = annotate("union U {};");
291 EXPECT_EQ(Tokens.size(), 6u) << Tokens;
292 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
293
294 Tokens = annotate("union U { void f() { return; } };");
295 EXPECT_EQ(Tokens.size(), 14u) << Tokens;
296 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_UnionLBrace);
297 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_FunctionLBrace);
298 }
299
300 TEST_F(TokenAnnotatorTest, UnderstandsEnums) {
301 auto Tokens = annotate("enum E {};");
302 EXPECT_EQ(Tokens.size(), 6u) << Tokens;
303 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_EnumLBrace);
304 }
305
306 TEST_F(TokenAnnotatorTest, UnderstandsDefaultedAndDeletedFunctions) {
307 auto Tokens = annotate("auto operator<=>(const T &) const & = default;");
308 EXPECT_EQ(Tokens.size(), 14u) << Tokens;
309 EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
310
311 Tokens = annotate("template <typename T> void F(T) && = delete;");
312 EXPECT_EQ(Tokens.size(), 15u) << Tokens;
313 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_PointerOrReference);
314 }
315
316 TEST_F(TokenAnnotatorTest, UnderstandsVariables) {
317 auto Tokens =
318 annotate("inline bool var = is_integral_v<int> && is_signed_v<int>;");
319 EXPECT_EQ(Tokens.size(), 15u) << Tokens;
320 EXPECT_TOKEN(Tokens[8], tok::ampamp, TT_BinaryOperator);
321 }
322
323 TEST_F(TokenAnnotatorTest, UnderstandsVariableTemplates) {
324 auto Tokens =
325 annotate("template <typename T> "
326 "inline bool var = is_integral_v<int> && is_signed_v<int>;");
327 EXPECT_EQ(Tokens.size(), 20u) << Tokens;
328 EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
329 }
330
331 TEST_F(TokenAnnotatorTest, UnderstandsTemplatesInMacros) {
332 auto Tokens =
333 annotate("#define FOO(typeName) \\\n"
334 " { #typeName, foo<FooType>(new foo<realClass>(#typeName)) }");
335 ASSERT_EQ(Tokens.size(), 27u) << Tokens;
336 EXPECT_TOKEN(Tokens[11], tok::less, TT_TemplateOpener);
337 EXPECT_TOKEN(Tokens[13], tok::greater, TT_TemplateCloser);
338 EXPECT_TOKEN(Tokens[17], tok::less, TT_TemplateOpener);
339 EXPECT_TOKEN(Tokens[19], tok::greater, TT_TemplateCloser);
340 }
341
342 TEST_F(TokenAnnotatorTest, UnderstandsGreaterAfterTemplateCloser) {
343 auto Tokens = annotate("if (std::tuple_size_v<T> > 0)");
344 ASSERT_EQ(Tokens.size(), 12u) << Tokens;
345 EXPECT_TOKEN(Tokens[5], tok::less, TT_TemplateOpener);
346 EXPECT_TOKEN(Tokens[7], tok::greater, TT_TemplateCloser);
347 EXPECT_TOKEN(Tokens[8], tok::greater, TT_BinaryOperator);
348 }
349
350 TEST_F(TokenAnnotatorTest, UnderstandsWhitespaceSensitiveMacros) {
351 FormatStyle Style = getLLVMStyle();
352 Style.WhitespaceSensitiveMacros.push_back("FOO");
353
354 auto Tokens = annotate("FOO(1+2 )\n", Style);
355 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
356 EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
357
358 Tokens = annotate("FOO(a:b:c)\n", Style);
359 EXPECT_EQ(Tokens.size(), 9u) << Tokens;
360 EXPECT_TOKEN(Tokens[0], tok::identifier, TT_UntouchableMacroFunc);
361 }
362
363 TEST_F(TokenAnnotatorTest, UnderstandsDelete) {
364 auto Tokens = annotate("delete (void *)p;");
365 EXPECT_EQ(Tokens.size(), 8u) << Tokens;
366 EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_CastRParen);
367
368 Tokens = annotate("delete[] (void *)p;");
369 EXPECT_EQ(Tokens.size(), 10u) << Tokens;
370 EXPECT_TOKEN(Tokens[6], tok::r_paren, TT_CastRParen);
371
372 Tokens = annotate("delete[] /*comment*/ (void *)p;");
373 EXPECT_EQ(Tokens.size(), 11u) << Tokens;
374 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
375
376 Tokens = annotate("delete[/*comment*/] (void *)p;");
377 EXPECT_EQ(Tokens.size(), 11u) << Tokens;
378 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
379
380 Tokens = annotate("delete/*comment*/[] (void *)p;");
381 EXPECT_EQ(Tokens.size(), 11u) << Tokens;
382 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
383 }
384
385 TEST_F(TokenAnnotatorTest, UnderstandsCasts) {
386 auto Tokens = annotate("(void)p;");
387 EXPECT_EQ(Tokens.size(), 6u) << Tokens;
388 EXPECT_TOKEN(Tokens[2], tok::r_paren, TT_CastRParen);
389
390 Tokens = annotate("auto x = (Foo)p;");
391 EXPECT_EQ(Tokens.size(), 9u) << Tokens;
392 EXPECT_TOKEN(Tokens[5], tok::r_paren, TT_CastRParen);
393
394 Tokens = annotate("(std::vector<int>)p;");
395 EXPECT_EQ(Tokens.size(), 11u) << Tokens;
396 EXPECT_TOKEN(Tokens[7], tok::r_paren, TT_CastRParen);
397
398 Tokens = annotate("return (Foo)p;");
399 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
400 EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
401
402 Tokens = annotate("throw (Foo)p;");
403 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
404 EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_CastRParen);
405 }
406
407 TEST_F(TokenAnnotatorTest, UnderstandsDynamicExceptionSpecifier) {
408 auto Tokens = annotate("void f() throw(int);");
409 EXPECT_EQ(Tokens.size(), 10u) << Tokens;
410 EXPECT_TOKEN(Tokens[4], tok::kw_throw, TT_Unknown);
411 }
412
413 TEST_F(TokenAnnotatorTest, UnderstandsFunctionRefQualifiers) {
414 auto Tokens = annotate("void f() &;");
415 EXPECT_EQ(Tokens.size(), 7u) << Tokens;
416 EXPECT_TOKEN(Tokens[4], tok::amp, TT_PointerOrReference);
417
418 Tokens = annotate("void operator=(T) &&;");
419 EXPECT_EQ(Tokens.size(), 9u) << Tokens;
420 EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
421
422 Tokens = annotate("template <typename T> void f() &;");
423 EXPECT_EQ(Tokens.size(), 12u) << Tokens;
424 EXPECT_TOKEN(Tokens[9], tok::amp, TT_PointerOrReference);
425
426 Tokens = annotate("template <typename T> void operator=(T) &;");
427 EXPECT_EQ(Tokens.size(), 14u) << Tokens;
428 EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
429 }
430
431 TEST_F(TokenAnnotatorTest, UnderstandsOverloadedOperators) {
432 auto Tokens = annotate("x.operator+()");
433 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
434 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
435 EXPECT_TOKEN(Tokens[3], tok::plus, TT_OverloadedOperator);
436 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
437 Tokens = annotate("x.operator=()");
438 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
439 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
440 EXPECT_TOKEN(Tokens[3], tok::equal, TT_OverloadedOperator);
441 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
442 Tokens = annotate("x.operator+=()");
443 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
444 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
445 EXPECT_TOKEN(Tokens[3], tok::plusequal, TT_OverloadedOperator);
446 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
447 Tokens = annotate("x.operator,()");
448 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
449 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
450 EXPECT_TOKEN(Tokens[3], tok::comma, TT_OverloadedOperator);
451 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
452 Tokens = annotate("x.operator()()");
453 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
454 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
455 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperator);
456 EXPECT_TOKEN(Tokens[4], tok::r_paren, TT_OverloadedOperator);
457 EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
458 Tokens = annotate("x.operator[]()");
459 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
460 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
461 // EXPECT_TOKEN(Tokens[3], tok::l_square, TT_OverloadedOperator);
462 // EXPECT_TOKEN(Tokens[4], tok::r_square, TT_OverloadedOperator);
463 EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
464 Tokens = annotate("x.operator\"\"_a()");
465 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
466 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
467 EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
468 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
469 Tokens = annotate("x.operator\"\" _a()");
470 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
471 // FIXME
472 // EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
473 EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
474 EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
475 Tokens = annotate("x.operator\"\"if()");
476 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
477 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
478 EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
479 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
480 Tokens = annotate("x.operator\"\"s()");
481 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
482 EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
483 EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
484 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
485 Tokens = annotate("x.operator\"\" s()");
486 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
487 // FIXME
488 // EXPECT_TOKEN(Tokens[2], tok::kw_operator, TT_FunctionDeclarationName);
489 EXPECT_TOKEN(Tokens[3], tok::string_literal, TT_OverloadedOperator);
490 EXPECT_TOKEN(Tokens[5], tok::l_paren, TT_OverloadedOperatorLParen);
491
492 Tokens = annotate("int operator+(int);");
493 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
494 EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
495 EXPECT_TOKEN(Tokens[2], tok::plus, TT_OverloadedOperator);
496 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
497 Tokens = annotate("auto operator=(T&) {}");
498 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
499 EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
500 EXPECT_TOKEN(Tokens[2], tok::equal, TT_OverloadedOperator);
501 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_OverloadedOperatorLParen);
502 Tokens = annotate("auto operator()() {}");
503 ASSERT_EQ(Tokens.size(), 9u) << Tokens;
504 EXPECT_TOKEN(Tokens[1], tok::kw_operator, TT_FunctionDeclarationName);
505 EXPECT_TOKEN(Tokens[2], tok::l_paren, TT_OverloadedOperator);
506 EXPECT_TOKEN(Tokens[3], tok::r_paren, TT_OverloadedOperator);
507 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_OverloadedOperatorLParen);
508 }
509
510 TEST_F(TokenAnnotatorTest, UnderstandsRequiresClausesAndConcepts) {
511 auto Tokens = annotate("template <typename T>\n"
512 "concept C = (Foo && Bar) && (Bar && Baz);");
513
514 ASSERT_EQ(Tokens.size(), 21u) << Tokens;
515 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
516 EXPECT_TOKEN(Tokens[13], tok::ampamp, TT_BinaryOperator);
517 EXPECT_TOKEN(Tokens[16], tok::ampamp, TT_BinaryOperator);
518
519 Tokens = annotate("template <typename T>\n"
520 "concept C = Foo && !Bar;");
521
522 ASSERT_EQ(Tokens.size(), 14u) << Tokens;
523 EXPECT_TOKEN(Tokens[9], tok::ampamp, TT_BinaryOperator);
524 EXPECT_TOKEN(Tokens[10], tok::exclaim, TT_UnaryOperator);
525
526 Tokens = annotate("template <typename T>\n"
527 "concept C = requires(T t) {\n"
528 " { t.foo() };\n"
529 "} && Bar<T> && Baz<T>;");
530 ASSERT_EQ(Tokens.size(), 35u) << Tokens;
531 EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
532 EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
533 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
534 EXPECT_TOKEN(Tokens[23], tok::ampamp, TT_BinaryOperator);
535 EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
536
537 Tokens = annotate("template<typename T>\n"
538 "requires C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>\n"
539 "struct Foo;");
540 ASSERT_EQ(Tokens.size(), 36u) << Tokens;
541 EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
542 EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
543 EXPECT_EQ(Tokens[6]->FakeLParens.size(), 1u);
544 EXPECT_TOKEN(Tokens[10], tok::ampamp, TT_BinaryOperator);
545 EXPECT_TOKEN(Tokens[16], tok::pipepipe, TT_BinaryOperator);
546 EXPECT_TOKEN(Tokens[21], tok::ampamp, TT_BinaryOperator);
547 EXPECT_TOKEN(Tokens[27], tok::ampamp, TT_BinaryOperator);
548 EXPECT_TOKEN(Tokens[31], tok::greater, TT_TemplateCloser);
549 EXPECT_EQ(Tokens[31]->FakeRParens, 1u);
550 EXPECT_TRUE(Tokens[31]->ClosesRequiresClause);
551
552 Tokens =
553 annotate("template<typename T>\n"
554 "requires (C1<T> && (C21<T> || C22<T> && C2e<T>) && C3<T>)\n"
555 "struct Foo;");
556 ASSERT_EQ(Tokens.size(), 38u) << Tokens;
557 EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
558 EXPECT_TOKEN(Tokens[7], tok::identifier, TT_Unknown);
559 EXPECT_EQ(Tokens[7]->FakeLParens.size(), 1u);
560 EXPECT_TOKEN(Tokens[11], tok::ampamp, TT_BinaryOperator);
561 EXPECT_TOKEN(Tokens[17], tok::pipepipe, TT_BinaryOperator);
562 EXPECT_TOKEN(Tokens[22], tok::ampamp, TT_BinaryOperator);
563 EXPECT_TOKEN(Tokens[28], tok::ampamp, TT_BinaryOperator);
564 EXPECT_TOKEN(Tokens[32], tok::greater, TT_TemplateCloser);
565 EXPECT_EQ(Tokens[32]->FakeRParens, 1u);
566 EXPECT_TOKEN(Tokens[33], tok::r_paren, TT_Unknown);
567 EXPECT_TRUE(Tokens[33]->ClosesRequiresClause);
568
569 Tokens = annotate("template <typename T>\n"
570 "void foo(T) noexcept requires Bar<T>;");
571 ASSERT_EQ(Tokens.size(), 18u) << Tokens;
572 EXPECT_TOKEN(Tokens[11], tok::kw_requires, TT_RequiresClause);
573
574 Tokens = annotate("template <typename T>\n"
575 "requires Bar<T> || Baz<T>\n"
576 "auto foo(T) -> int;");
577 ASSERT_EQ(Tokens.size(), 24u) << Tokens;
578 EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
579 EXPECT_EQ(Tokens[11]->FakeLParens.size(), 0u);
580 EXPECT_TRUE(Tokens[14]->ClosesRequiresClause);
581 EXPECT_TOKEN(Tokens[20], tok::arrow, TT_TrailingReturnArrow);
582
583 Tokens = annotate("template <typename T>\n"
584 "requires Bar<T>\n"
585 "bool foo(T) { return false; }");
586 ASSERT_EQ(Tokens.size(), 21u) << Tokens;
587 EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
588 EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
589 EXPECT_TOKEN(Tokens[11], tok::identifier, TT_FunctionDeclarationName);
590
591 Tokens = annotate("template <typename T>\n"
592 "requires Bar<T>\n"
593 "decltype(auto) foo(T) { return false; }");
594 ASSERT_EQ(Tokens.size(), 24u) << Tokens;
595 EXPECT_TOKEN(Tokens[5], tok::kw_requires, TT_RequiresClause);
596 EXPECT_TRUE(Tokens[9]->ClosesRequiresClause);
597 EXPECT_TOKEN(Tokens[14], tok::identifier, TT_FunctionDeclarationName);
598
599 Tokens = annotate("template <typename T>\n"
600 "struct S {\n"
601 " void foo() const requires Bar<T>;\n"
602 " void bar() const & requires Baz<T>;\n"
603 " void bar() && requires Baz2<T>;\n"
604 " void baz() const & noexcept requires Baz<T>;\n"
605 " void baz() && noexcept requires Baz2<T>;\n"
606 "};\n"
607 "\n"
608 "void S::bar() const & requires Baz<T> { }");
609 ASSERT_EQ(Tokens.size(), 85u) << Tokens;
610 EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
611 EXPECT_TOKEN(Tokens[25], tok::kw_requires, TT_RequiresClause);
612 EXPECT_TOKEN(Tokens[36], tok::kw_requires, TT_RequiresClause);
613 EXPECT_TOKEN(Tokens[49], tok::kw_requires, TT_RequiresClause);
614 EXPECT_TOKEN(Tokens[61], tok::kw_requires, TT_RequiresClause);
615 EXPECT_TOKEN(Tokens[77], tok::kw_requires, TT_RequiresClause);
616
617 Tokens = annotate("void Class::member() && requires(Constant) {}");
618 ASSERT_EQ(Tokens.size(), 14u) << Tokens;
619 EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
620
621 Tokens = annotate("void Class::member() && requires(Constant<T>) {}");
622 ASSERT_EQ(Tokens.size(), 17u) << Tokens;
623 EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
624
625 Tokens =
626 annotate("void Class::member() && requires(Namespace::Constant<T>) {}");
627 ASSERT_EQ(Tokens.size(), 19u) << Tokens;
628 EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
629
630 Tokens = annotate("void Class::member() && requires(typename "
631 "Namespace::Outer<T>::Inner::Constant) {}");
632 ASSERT_EQ(Tokens.size(), 24u) << Tokens;
633 EXPECT_TOKEN(Tokens[7], tok::kw_requires, TT_RequiresClause);
634
635 Tokens = annotate("struct [[nodiscard]] zero_t {\n"
636 " template<class T>\n"
637 " requires requires { number_zero_v<T>; }\n"
638 " [[nodiscard]] constexpr operator T() const { "
639 "return number_zero_v<T>; }\n"
640 "};");
641 ASSERT_EQ(Tokens.size(), 44u);
642 EXPECT_TOKEN(Tokens[13], tok::kw_requires, TT_RequiresClause);
643 EXPECT_TOKEN(Tokens[14], tok::kw_requires, TT_RequiresExpression);
644 EXPECT_TOKEN(Tokens[15], tok::l_brace, TT_RequiresExpressionLBrace);
645 EXPECT_TOKEN(Tokens[21], tok::r_brace, TT_Unknown);
646 EXPECT_EQ(Tokens[21]->MatchingParen, Tokens[15]);
647 EXPECT_TRUE(Tokens[21]->ClosesRequiresClause);
648
649 Tokens =
650 annotate("template <class A, class B> concept C ="
651 "std::same_as<std::iter_value_t<A>, std::iter_value_t<B>>;");
652 ASSERT_EQ(Tokens.size(), 31u) << Tokens;
653 EXPECT_TOKEN(Tokens[8], tok::kw_concept, TT_Unknown);
654 EXPECT_TOKEN(Tokens[14], tok::less, TT_TemplateOpener);
655 EXPECT_TOKEN(Tokens[18], tok::less, TT_TemplateOpener);
656 EXPECT_TOKEN(Tokens[20], tok::greater, TT_TemplateCloser);
657 EXPECT_TOKEN(Tokens[25], tok::less, TT_TemplateOpener);
658 EXPECT_TOKEN(Tokens[27], tok::greater, TT_TemplateCloser);
659 EXPECT_TOKEN(Tokens[28], tok::greater, TT_TemplateCloser);
660
661 Tokens = annotate("auto bar() -> int requires(is_integral_v<T>) {}");
662 ASSERT_EQ(Tokens.size(), 16u) << Tokens;
663 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
664
665 Tokens = annotate("auto bar() -> void requires(is_integral_v<T>) {}");
666 ASSERT_EQ(Tokens.size(), 16u) << Tokens;
667 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
668
669 Tokens = annotate("auto bar() -> MyType requires(is_integral_v<T>) {}");
670 ASSERT_EQ(Tokens.size(), 16u) << Tokens;
671 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
672
673 Tokens =
674 annotate("auto bar() -> SOME_MACRO_TYPE requires(is_integral_v<T>) {}");
675 ASSERT_EQ(Tokens.size(), 16u) << Tokens;
676 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresClause);
677
678 Tokens =
679 annotate("auto bar() -> qualified::type requires(is_integral_v<T>) {}");
680 ASSERT_EQ(Tokens.size(), 18u) << Tokens;
681 EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresClause);
682
683 Tokens =
684 annotate("auto bar() -> Template<type> requires(is_integral_v<T>) {}");
685 ASSERT_EQ(Tokens.size(), 19u) << Tokens;
686 EXPECT_TOKEN(Tokens[9], tok::kw_requires, TT_RequiresClause);
687 }
688
689 TEST_F(TokenAnnotatorTest, UnderstandsRequiresExpressions) {
690 auto Tokens = annotate("bool b = requires(int i) { i + 5; };");
691 ASSERT_EQ(Tokens.size(), 16u) << Tokens;
692 EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
693 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
694 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
695
696 Tokens = annotate("if (requires(int i) { i + 5; }) return;");
697 ASSERT_EQ(Tokens.size(), 17u) << Tokens;
698 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
699 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
700 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_RequiresExpressionLBrace);
701
702 Tokens = annotate("if (func() && requires(int i) { i + 5; }) return;");
703 ASSERT_EQ(Tokens.size(), 21u) << Tokens;
704 EXPECT_TOKEN(Tokens[6], tok::kw_requires, TT_RequiresExpression);
705 EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_RequiresExpressionLParen);
706 EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_RequiresExpressionLBrace);
707
708 Tokens = annotate("foo(requires(const T t) {});");
709 ASSERT_EQ(Tokens.size(), 13u) << Tokens;
710 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
711 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
712 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
713
714 Tokens = annotate("foo(requires(const int t) {});");
715 ASSERT_EQ(Tokens.size(), 13u) << Tokens;
716 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
717 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
718 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
719
720 Tokens = annotate("foo(requires(const T t) {});");
721 ASSERT_EQ(Tokens.size(), 13u) << Tokens;
722 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
723 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
724 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
725
726 Tokens = annotate("foo(requires(int const* volatile t) {});");
727 ASSERT_EQ(Tokens.size(), 15u) << Tokens;
728 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
729 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
730 EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
731 EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
732
733 Tokens = annotate("foo(requires(T const* volatile t) {});");
734 ASSERT_EQ(Tokens.size(), 15u) << Tokens;
735 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
736 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
737 EXPECT_TOKEN(Tokens[6], tok::star, TT_PointerOrReference);
738 EXPECT_TOKEN(Tokens[10], tok::l_brace, TT_RequiresExpressionLBrace);
739
740 Tokens = annotate("foo(requires(T& t) {});");
741 ASSERT_EQ(Tokens.size(), 13u) << Tokens;
742 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
743 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
744 EXPECT_TOKEN(Tokens[5], tok::amp, TT_PointerOrReference);
745 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
746
747 Tokens = annotate("foo(requires(T&& t) {});");
748 ASSERT_EQ(Tokens.size(), 13u) << Tokens;
749 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
750 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
751 EXPECT_TOKEN(Tokens[5], tok::ampamp, TT_PointerOrReference);
752 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_RequiresExpressionLBrace);
753
754 Tokens = annotate("bool foo = requires(T& t) {};");
755 ASSERT_EQ(Tokens.size(), 13u) << Tokens;
756 EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
757 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
758 EXPECT_TOKEN(Tokens[6], tok::amp, TT_PointerOrReference);
759 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
760
761 Tokens = annotate("bool foo = requires(T&& t) {};");
762 ASSERT_EQ(Tokens.size(), 13u) << Tokens;
763 EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
764 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
765 EXPECT_TOKEN(Tokens[6], tok::ampamp, TT_PointerOrReference);
766 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_RequiresExpressionLBrace);
767
768 Tokens =
769 annotate("foo(requires(const typename Outer<T>::Inner * const t) {});");
770 ASSERT_EQ(Tokens.size(), 21u) << Tokens;
771 EXPECT_TOKEN(Tokens[2], tok::kw_requires, TT_RequiresExpression);
772 EXPECT_TOKEN(Tokens[3], tok::l_paren, TT_RequiresExpressionLParen);
773 EXPECT_TOKEN(Tokens[12], tok::star, TT_PointerOrReference);
774 EXPECT_TOKEN(Tokens[16], tok::l_brace, TT_RequiresExpressionLBrace);
775
776 Tokens = annotate("template <typename T>\n"
777 "concept C = requires(T T) {\n"
778 " requires Bar<T> && Foo<T>;\n"
779 "};");
780 ASSERT_EQ(Tokens.size(), 28u) << Tokens;
781 EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
782 EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
783 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
784 EXPECT_TOKEN(Tokens[14], tok::kw_requires,
785 TT_RequiresClauseInARequiresExpression);
786
787 Tokens = annotate("template <typename T>\n"
788 "concept C = requires(T T) {\n"
789 " { t.func() } -> std::same_as<int>;"
790 " requires Bar<T> && Foo<T>;\n"
791 "};");
792 ASSERT_EQ(Tokens.size(), 43u) << Tokens;
793 EXPECT_TOKEN(Tokens[8], tok::kw_requires, TT_RequiresExpression);
794 EXPECT_TOKEN(Tokens[9], tok::l_paren, TT_RequiresExpressionLParen);
795 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_RequiresExpressionLBrace);
796 EXPECT_TOKEN(Tokens[29], tok::kw_requires,
797 TT_RequiresClauseInARequiresExpression);
798
799 // Invalid Code, but we don't want to crash. See http://llvm.org/PR54350.
800 Tokens = annotate("bool r10 = requires (struct new_struct { int x; } s) { "
801 "requires true; };");
802 ASSERT_EQ(Tokens.size(), 21u) << Tokens;
803 EXPECT_TOKEN(Tokens[3], tok::kw_requires, TT_RequiresExpression);
804 EXPECT_TOKEN(Tokens[4], tok::l_paren, TT_RequiresExpressionLParen);
805 EXPECT_TOKEN(Tokens[14], tok::l_brace, TT_RequiresExpressionLBrace);
806 }
807
808 TEST_F(TokenAnnotatorTest, UnderstandsPragmaRegion) {
809 // Everything after #pragma region should be ImplicitStringLiteral
810 auto Tokens = annotate("#pragma region Foo(Bar: Hello)");
811 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
812 EXPECT_TOKEN(Tokens[5], tok::identifier, TT_ImplicitStringLiteral);
813 EXPECT_TOKEN(Tokens[6], tok::colon, TT_ImplicitStringLiteral);
814 EXPECT_TOKEN(Tokens[7], tok::identifier, TT_ImplicitStringLiteral);
815
816 // Make sure it's annotated correctly inside a function as well
817 Tokens = annotate("void test(){\n#pragma region Foo(Bar: Hello)\n}");
818 ASSERT_EQ(Tokens.size(), 16u) << Tokens;
819 EXPECT_TOKEN(Tokens[10], tok::identifier, TT_ImplicitStringLiteral);
820 EXPECT_TOKEN(Tokens[11], tok::colon, TT_ImplicitStringLiteral);
821 EXPECT_TOKEN(Tokens[12], tok::identifier, TT_ImplicitStringLiteral);
822 }
823
824 TEST_F(TokenAnnotatorTest, RequiresDoesNotChangeParsingOfTheRest) {
825 const char *BaseCode = nullptr;
826 const char *ConstrainedCode = nullptr;
827 auto BaseTokenCount = 0u;
828 auto RequiresTokenCount = 0u;
829 auto PrefixTokenCount = 0u;
830
831 auto TestRequires = [&](int Line) {
832 const auto BaseTokens = annotate(BaseCode);
833 const auto ConstrainedTokens = annotate(ConstrainedCode);
834
835 #define LINE " (Line " << Line << ')'
836
837 ASSERT_EQ(BaseTokens.size(), BaseTokenCount) << BaseTokens << LINE;
838 ASSERT_EQ(ConstrainedTokens.size(), BaseTokenCount + RequiresTokenCount)
839 << LINE;
840
841 for (auto I = 0u; I < BaseTokenCount; ++I) {
842 EXPECT_EQ(
843 *BaseTokens[I],
844 *ConstrainedTokens[I < PrefixTokenCount ? I : I + RequiresTokenCount])
845 << I << LINE;
846 }
847
848 #undef LINE
849 };
850
851 BaseCode = "template<typename T>\n"
852 "T Pi = 3.14;";
853 ConstrainedCode = "template<typename T>\n"
854 " requires Foo<T>\n"
855 "T Pi = 3.14;";
856 BaseTokenCount = 11;
857 RequiresTokenCount = 5;
858 PrefixTokenCount = 5;
859 TestRequires(__LINE__);
860
861 BaseCode = "template<typename T>\n"
862 "struct Bar;";
863 ConstrainedCode = "template<typename T>\n"
864 " requires Foo<T>\n"
865 "struct Bar;";
866 BaseTokenCount = 9;
867 TestRequires(__LINE__);
868
869 BaseCode = "template<typename T>\n"
870 "struct Bar {\n"
871 " T foo();\n"
872 " T bar();\n"
873 "};";
874 ConstrainedCode = "template<typename T>\n"
875 " requires Foo<T>\n"
876 "struct Bar {\n"
877 " T foo();\n"
878 " T bar();\n"
879 "};";
880 BaseTokenCount = 21;
881 TestRequires(__LINE__);
882
883 BaseCode = "template<typename T>\n"
884 "Bar(T) -> Bar<T>;";
885 ConstrainedCode = "template<typename T>\n"
886 " requires Foo<T>\n"
887 "Bar(T) -> Bar<T>;";
888 BaseTokenCount = 16;
889 TestRequires(__LINE__);
890
891 BaseCode = "template<typename T>\n"
892 "T foo();";
893 ConstrainedCode = "template<typename T>\n"
894 " requires Foo<T>\n"
895 "T foo();";
896 BaseTokenCount = 11;
897 TestRequires(__LINE__);
898
899 BaseCode = "template<typename T>\n"
900 "T foo() {\n"
901 " auto bar = baz();\n"
902 " return bar + T{};\n"
903 "}";
904 ConstrainedCode = "template<typename T>\n"
905 " requires Foo<T>\n"
906 "T foo() {\n"
907 " auto bar = baz();\n"
908 " return bar + T{};\n"
909 "}";
910 BaseTokenCount = 26;
911 TestRequires(__LINE__);
912
913 BaseCode = "template<typename T>\n"
914 "T foo();";
915 ConstrainedCode = "template<typename T>\n"
916 "T foo() requires Foo<T>;";
917 BaseTokenCount = 11;
918 PrefixTokenCount = 9;
919 TestRequires(__LINE__);
920
921 BaseCode = "template<typename T>\n"
922 "T foo() {\n"
923 " auto bar = baz();\n"
924 " return bar + T{};\n"
925 "}";
926 ConstrainedCode = "template<typename T>\n"
927 "T foo() requires Foo<T> {\n"
928 " auto bar = baz();\n"
929 " return bar + T{};\n"
930 "}";
931 BaseTokenCount = 26;
932 TestRequires(__LINE__);
933
934 BaseCode = "template<typename T>\n"
935 "T foo();";
936 ConstrainedCode = "template<typename T>\n"
937 " requires(Foo<T>)\n"
938 "T foo();";
939 BaseTokenCount = 11;
940 RequiresTokenCount = 7;
941 PrefixTokenCount = 5;
942 TestRequires(__LINE__);
943
944 BaseCode = "template<typename T>\n"
945 "Bar(T) -> Bar<typename T::I>;";
946 ConstrainedCode = "template<typename T>\n"
947 " requires requires(T &&t) {\n"
948 " typename T::I;\n"
949 " }\n"
950 "Bar(T) -> Bar<typename T::I>;";
951 BaseTokenCount = 19;
952 RequiresTokenCount = 14;
953 PrefixTokenCount = 5;
954 TestRequires(__LINE__);
955
956 BaseCode = "struct [[nodiscard]] zero_t {\n"
957 " template<class T>\n"
958 " [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
959 "};";
960 ConstrainedCode =
961 "struct [[nodiscard]] zero_t {\n"
962 " template<class T>\n"
963 " requires requires { v<T>; }\n"
964 " [[nodiscard]] constexpr operator T() const { return v<T>; }\n"
965 "};";
966 BaseTokenCount = 35;
967 RequiresTokenCount = 9;
968 PrefixTokenCount = 13;
969 TestRequires(__LINE__);
970
971 BaseCode = "constexpr Foo(Foo const &other)\n"
972 " : value{other.value} {\n"
973 " do_magic();\n"
974 " do_more_magic();\n"
975 "}";
976 ConstrainedCode = "constexpr Foo(Foo const &other)\n"
977 " requires std::is_copy_constructible<T>\n"
978 " : value{other.value} {\n"
979 " do_magic();\n"
980 " do_more_magic();\n"
981 "}";
982 BaseTokenCount = 26;
983 RequiresTokenCount = 7;
984 PrefixTokenCount = 8;
985 TestRequires(__LINE__);
986
987 BaseCode = "constexpr Foo(Foo const &other)\n"
988 " : value{other.value} {\n"
989 " do_magic();\n"
990 " do_more_magic();\n"
991 "}";
992 ConstrainedCode = "constexpr Foo(Foo const &other)\n"
993 " requires (std::is_copy_constructible<T>)\n"
994 " : value{other.value} {\n"
995 " do_magic();\n"
996 " do_more_magic();\n"
997 "}";
998 RequiresTokenCount = 9;
999 TestRequires(__LINE__);
1000 }
1001
1002 TEST_F(TokenAnnotatorTest, UnderstandsAsm) {
1003 auto Tokens = annotate("__asm{\n"
1004 "a:\n"
1005 "};");
1006 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1007 EXPECT_TOKEN(Tokens[0], tok::kw_asm, TT_Unknown);
1008 EXPECT_TOKEN(Tokens[1], tok::l_brace, TT_InlineASMBrace);
1009 EXPECT_TOKEN(Tokens[4], tok::r_brace, TT_InlineASMBrace);
1010 }
1011
1012 TEST_F(TokenAnnotatorTest, UnderstandsObjCBlock) {
1013 auto Tokens = annotate("int (^)() = ^ ()\n"
1014 " external_source_symbol() { //\n"
1015 " return 1;\n"
1016 "};");
1017 ASSERT_EQ(Tokens.size(), 21u) << Tokens;
1018 EXPECT_TOKEN(Tokens[1], tok::l_paren, TT_ObjCBlockLParen);
1019 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_ObjCBlockLBrace);
1020
1021 Tokens = annotate("int *p = ^int*(){ //\n"
1022 " return nullptr;\n"
1023 "}();");
1024 ASSERT_EQ(Tokens.size(), 19u) << Tokens;
1025 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_ObjCBlockLBrace);
1026 }
1027
1028 TEST_F(TokenAnnotatorTest, UnderstandsLambdas) {
1029 auto Tokens = annotate("[]() constexpr {}");
1030 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1031 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1032 EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1033
1034 Tokens = annotate("[]() consteval {}");
1035 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1036 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1037 EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1038
1039 Tokens = annotate("[]() mutable {}");
1040 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1041 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1042 EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1043
1044 Tokens = annotate("[]() static {}");
1045 ASSERT_EQ(Tokens.size(), 8u) << Tokens;
1046 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1047 EXPECT_TOKEN(Tokens[5], tok::l_brace, TT_LambdaLBrace);
1048
1049 Tokens = annotate("[]() -> auto {}");
1050 ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1051 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1052 EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1053 EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1054
1055 Tokens = annotate("[]() -> auto & {}");
1056 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1057 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1058 EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1059 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1060
1061 Tokens = annotate("[]() -> auto * {}");
1062 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1063 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1064 EXPECT_TOKEN(Tokens[4], tok::arrow, TT_LambdaArrow);
1065 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1066
1067 Tokens = annotate("[] {}");
1068 ASSERT_EQ(Tokens.size(), 5u) << Tokens;
1069 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1070 EXPECT_TOKEN(Tokens[2], tok::l_brace, TT_LambdaLBrace);
1071
1072 Tokens = annotate("[] noexcept {}");
1073 ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1074 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1075 EXPECT_TOKEN(Tokens[3], tok::l_brace, TT_LambdaLBrace);
1076
1077 Tokens = annotate("[] -> auto {}");
1078 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1079 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1080 EXPECT_TOKEN(Tokens[2], tok::arrow, TT_LambdaArrow);
1081 EXPECT_TOKEN(Tokens[4], tok::l_brace, TT_LambdaLBrace);
1082
1083 Tokens = annotate("[] <typename T> () {}");
1084 ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1085 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1086 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1087 EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_LambdaLBrace);
1088
1089 Tokens = annotate("[] <typename T> {}");
1090 ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1091 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1092 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1093 EXPECT_TOKEN(Tokens[6], tok::l_brace, TT_LambdaLBrace);
1094
1095 Tokens = annotate("[] <typename... T> () {}");
1096 ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1097 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1098 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1099 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1100
1101 Tokens = annotate("[] <typename... T> {}");
1102 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1103 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1104 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1105 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1106
1107 Tokens = annotate("[] <int... T> () {}");
1108 ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1109 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1110 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1111 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1112
1113 Tokens = annotate("[] <int... T> {}");
1114 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1115 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1116 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1117 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1118
1119 Tokens = annotate("[] <Foo... T> () {}");
1120 ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1121 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1122 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1123 EXPECT_TOKEN(Tokens[9], tok::l_brace, TT_LambdaLBrace);
1124
1125 Tokens = annotate("[] <Foo... T> {}");
1126 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1127 EXPECT_TOKEN(Tokens[0], tok::l_square, TT_LambdaLSquare);
1128 EXPECT_TOKEN(Tokens[2], tok::less, TT_TemplateOpener);
1129 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_LambdaLBrace);
1130 }
1131
1132 TEST_F(TokenAnnotatorTest, UnderstandsFunctionAnnotations) {
1133 auto Tokens = annotate("template <typename T>\n"
1134 "DEPRECATED(\"Use NewClass::NewFunction instead.\")\n"
1135 "string OldFunction(const string &parameter) {}");
1136 ASSERT_EQ(Tokens.size(), 20u) << Tokens;
1137 EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_FunctionAnnotationRParen);
1138
1139 Tokens = annotate("template <typename T>\n"
1140 "A(T) noexcept;");
1141 ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1142 EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
1143 }
1144
1145 TEST_F(TokenAnnotatorTest, UnderstandsFunctionDeclarationNames) {
1146 auto Tokens = annotate("void f [[noreturn]] ();");
1147 ASSERT_EQ(Tokens.size(), 11u) << Tokens;
1148 EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1149
1150 Tokens = annotate("void f [[noreturn]] () {}");
1151 ASSERT_EQ(Tokens.size(), 12u) << Tokens;
1152 EXPECT_TOKEN(Tokens[1], tok::identifier, TT_FunctionDeclarationName);
1153 }
1154
1155 TEST_F(TokenAnnotatorTest, UnderstandsC11GenericSelection) {
1156 auto Tokens = annotate("_Generic(x, int: 1, default: 0)");
1157 ASSERT_EQ(Tokens.size(), 13u) << Tokens;
1158 EXPECT_TOKEN(Tokens[0], tok::kw__Generic, TT_Unknown);
1159 EXPECT_TOKEN(Tokens[5], tok::colon, TT_GenericSelectionColon);
1160 EXPECT_TOKEN(Tokens[9], tok::colon, TT_GenericSelectionColon);
1161 }
1162
1163 TEST_F(TokenAnnotatorTest, UnderstandsVerilogOperators) {
1164 auto Annotate = [this](llvm::StringRef Code) {
1165 return annotate(Code, getLLVMStyle(FormatStyle::LK_Verilog));
1166 };
1167 // Test that unary operators get labeled as such and that operators like '++'
1168 // don't get split.
1169 tok::TokenKind Unary[] = {tok::plus, tok::minus, tok::exclaim,
1170 tok::tilde, tok::amp, tok::pipe,
1171 tok::caret, tok::plusplus, tok::minusminus};
1172 for (auto Kind : Unary) {
1173 auto Tokens =
1174 Annotate(std::string("x = ") + tok::getPunctuatorSpelling(Kind) + "x;");
1175 ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1176 EXPECT_TOKEN(Tokens[2], Kind, TT_UnaryOperator);
1177 }
1178 // Operators formed by joining two operators like '^~'. For some of these
1179 // joined operators, we don't have a separate type, so we only test for their
1180 // precedence.
1181 std::pair<prec::Level, std::string> JoinedBinary[] = {
1182 {prec::Comma, "<->"}, {prec::Assignment, "+="},
1183 {prec::Assignment, "-="}, {prec::Assignment, "*="},
1184 {prec::Assignment, "/="}, {prec::Assignment, "%="},
1185 {prec::Assignment, "&="}, {prec::Assignment, "^="},
1186 {prec::Assignment, "<<="}, {prec::Assignment, ">>="},
1187 {prec::Assignment, "<<<="}, {prec::Assignment, ">>>="},
1188 {prec::LogicalOr, "||"}, {prec::LogicalAnd, "&&"},
1189 {prec::Equality, "=="}, {prec::Equality, "!="},
1190 {prec::Equality, "==="}, {prec::Equality, "!=="},
1191 {prec::Equality, "==?"}, {prec::Equality, "!=?"},
1192 {prec::ExclusiveOr, "~^"}, {prec::ExclusiveOr, "^~"},
1193 };
1194 for (auto Operator : JoinedBinary) {
1195 auto Tokens = Annotate(std::string("x = x ") + Operator.second + " x;");
1196 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1197 EXPECT_TOKEN_TYPE(Tokens[3], TT_BinaryOperator);
1198 EXPECT_TOKEN_PRECEDENCE(Tokens[3], Operator.first);
1199 }
1200 // '~^' and '^~' can be unary as well as binary operators.
1201 auto Tokens = Annotate("x = ~^x;");
1202 ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1203 EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
1204 Tokens = Annotate("x = ^~x;");
1205 ASSERT_EQ(Tokens.size(), 6u) << Tokens;
1206 EXPECT_TOKEN_TYPE(Tokens[2], TT_UnaryOperator);
1207 // The unary operators '~&' and '~|' can only be unary operators. The current
1208 // implementation treats each of them as separate unary '~' and '&' or '|'
1209 // operators, which is enough for formatting purposes. In FormatTestVerilog,
1210 // there is a test that there is no space in between. And even if a new line
1211 // is inserted between the '~' and '|', the semantic meaning is the same as
1212 // the joined operator, so the CanBreakBefore property doesn't need to be
1213 // false for the second operator.
1214 Tokens = Annotate("x = ~&x;");
1215 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1216 EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
1217 EXPECT_TOKEN(Tokens[3], tok::amp, TT_UnaryOperator);
1218 Tokens = Annotate("x = ~|x;");
1219 ASSERT_EQ(Tokens.size(), 7u) << Tokens;
1220 EXPECT_TOKEN(Tokens[2], tok::tilde, TT_UnaryOperator);
1221 EXPECT_TOKEN(Tokens[3], tok::pipe, TT_UnaryOperator);
1222 // Test for block label colons.
1223 Tokens = Annotate("begin : x\n"
1224 "end : x");
1225 ASSERT_EQ(Tokens.size(), 7u);
1226 EXPECT_TOKEN(Tokens[1], tok::colon, TT_VerilogBlockLabelColon);
1227 EXPECT_TOKEN(Tokens[4], tok::colon, TT_VerilogBlockLabelColon);
1228 // Test that the dimension colon is annotated correctly.
1229 Tokens = Annotate("var [1 : 0] x;");
1230 ASSERT_EQ(Tokens.size(), 9u) << Tokens;
1231 EXPECT_TOKEN(Tokens[3], tok::colon, TT_BitFieldColon);
1232 Tokens = Annotate("extern function [1 : 0] x;");
1233 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1234 EXPECT_TOKEN(Tokens[4], tok::colon, TT_BitFieldColon);
1235 // Test case labels and ternary operators.
1236 Tokens = Annotate("case (x)\n"
1237 " x:\n"
1238 " x;\n"
1239 "endcase\n");
1240 ASSERT_EQ(Tokens.size(), 10u) << Tokens;
1241 EXPECT_TOKEN(Tokens[5], tok::colon, TT_GotoLabelColon);
1242 Tokens = Annotate("case (x)\n"
1243 " x ? x : x:\n"
1244 " x;\n"
1245 "endcase\n");
1246 ASSERT_EQ(Tokens.size(), 14u) << Tokens;
1247 EXPECT_TOKEN(Tokens[5], tok::question, TT_ConditionalExpr);
1248 EXPECT_TOKEN(Tokens[7], tok::colon, TT_ConditionalExpr);
1249 EXPECT_TOKEN(Tokens[9], tok::colon, TT_GotoLabelColon);
1250 }
1251
1252 TEST_F(TokenAnnotatorTest, UnderstandConstructors) {
1253 auto Tokens = annotate("Class::Class() : BaseClass(), Member() {}");
1254
1255 // The TT_Unknown is clearly not binding for the future, please adapt if those
1256 // tokens get annotated.
1257 ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1258 EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
1259 EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1260 EXPECT_TOKEN(Tokens[7], tok::l_paren, TT_Unknown);
1261 EXPECT_TOKEN(Tokens[8], tok::r_paren, TT_Unknown);
1262 EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
1263 EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
1264 EXPECT_TOKEN(Tokens[11], tok::l_paren, TT_Unknown);
1265 EXPECT_TOKEN(Tokens[12], tok::r_paren, TT_Unknown);
1266 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
1267
1268 Tokens = annotate("Class::Class() : BaseClass{}, Member{} {}");
1269 ASSERT_EQ(Tokens.size(), 16u) << Tokens;
1270 EXPECT_TOKEN(Tokens[5], tok::colon, TT_CtorInitializerColon);
1271 EXPECT_TOKEN(Tokens[6], tok::identifier, TT_Unknown);
1272 EXPECT_TOKEN(Tokens[7], tok::l_brace, TT_Unknown);
1273 EXPECT_TOKEN(Tokens[8], tok::r_brace, TT_Unknown);
1274 EXPECT_TOKEN(Tokens[9], tok::comma, TT_CtorInitializerComma);
1275 EXPECT_TOKEN(Tokens[10], tok::identifier, TT_Unknown);
1276 EXPECT_TOKEN(Tokens[11], tok::l_brace, TT_Unknown);
1277 EXPECT_TOKEN(Tokens[12], tok::r_brace, TT_Unknown);
1278 EXPECT_TOKEN(Tokens[13], tok::l_brace, TT_FunctionLBrace);
1279 }
1280
1281 } // namespace
1282 } // namespace format
1283 } // namespace clang