Debian Codegraph @

llvm-project/clang/unittests/Tooling/Syntax/TokensTest.cpp

raw
1 //===- TokensTest.cpp -----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "clang/Tooling/Syntax/Tokens.h"
10 #include "clang/AST/ASTConsumer.h"
11 #include "clang/AST/Expr.h"
12 #include "clang/Basic/Diagnostic.h"
13 #include "clang/Basic/DiagnosticIDs.h"
14 #include "clang/Basic/DiagnosticOptions.h"
15 #include "clang/Basic/FileManager.h"
16 #include "clang/Basic/FileSystemOptions.h"
17 #include "clang/Basic/LLVM.h"
18 #include "clang/Basic/LangOptions.h"
19 #include "clang/Basic/SourceLocation.h"
20 #include "clang/Basic/SourceManager.h"
21 #include "clang/Basic/TokenKinds.def"
22 #include "clang/Basic/TokenKinds.h"
23 #include "clang/Frontend/CompilerInstance.h"
24 #include "clang/Frontend/FrontendAction.h"
25 #include "clang/Frontend/Utils.h"
26 #include "clang/Lex/Lexer.h"
27 #include "clang/Lex/PreprocessorOptions.h"
28 #include "clang/Lex/Token.h"
29 #include "clang/Tooling/Tooling.h"
30 #include "llvm/ADT/ArrayRef.h"
31 #include "llvm/ADT/IntrusiveRefCntPtr.h"
32 #include "llvm/ADT/STLExtras.h"
33 #include "llvm/ADT/StringRef.h"
34 #include "llvm/Support/FormatVariadic.h"
35 #include "llvm/Support/MemoryBuffer.h"
36 #include "llvm/Support/VirtualFileSystem.h"
37 #include "llvm/Support/raw_os_ostream.h"
38 #include "llvm/Support/raw_ostream.h"
39 #include "llvm/Testing/Annotations/Annotations.h"
40 #include "llvm/Testing/Support/SupportHelpers.h"
41 #include <cassert>
42 #include <cstdlib>
43 #include <gmock/gmock.h>
44 #include <gtest/gtest.h>
45 #include <memory>
46 #include <optional>
47 #include <ostream>
48 #include <string>
49
50 using namespace clang;
51 using namespace clang::syntax;
52
53 using llvm::ValueIs;
54 using ::testing::_;
55 using ::testing::AllOf;
56 using ::testing::Contains;
57 using ::testing::ElementsAre;
58 using ::testing::Field;
59 using ::testing::IsEmpty;
60 using ::testing::Matcher;
61 using ::testing::Not;
62 using ::testing::Pointee;
63 using ::testing::StartsWith;
64
65 namespace {
66 // Checks the passed ArrayRef<T> has the same begin() and end() iterators as the
67 // argument.
68 MATCHER_P(SameRange, A, "") {
69 return A.begin() == arg.begin() && A.end() == arg.end();
70 }
71
72 Matcher<TokenBuffer::Expansion>
73 IsExpansion(Matcher<llvm::ArrayRef<syntax::Token>> Spelled,
74 Matcher<llvm::ArrayRef<syntax::Token>> Expanded) {
75 return AllOf(Field(&TokenBuffer::Expansion::Spelled, Spelled),
76 Field(&TokenBuffer::Expansion::Expanded, Expanded));
77 }
78 // Matchers for syntax::Token.
79 MATCHER_P(Kind, K, "") { return arg.kind() == K; }
80 MATCHER_P2(HasText, Text, SourceMgr, "") {
81 return arg.text(*SourceMgr) == Text;
82 }
83 /// Checks the start and end location of a token are equal to SourceRng.
84 MATCHER_P(RangeIs, SourceRng, "") {
85 return arg.location() == SourceRng.first &&
86 arg.endLocation() == SourceRng.second;
87 }
88
89 class TokenCollectorTest : public ::testing::Test {
90 public:
91 /// Run the clang frontend, collect the preprocessed tokens from the frontend
92 /// invocation and store them in this->Buffer.
93 /// This also clears SourceManager before running the compiler.
94 void recordTokens(llvm::StringRef Code) {
95 class RecordTokens : public ASTFrontendAction {
96 public:
97 explicit RecordTokens(TokenBuffer &Result) : Result(Result) {}
98
99 bool BeginSourceFileAction(CompilerInstance &CI) override {
100 assert(!Collector && "expected only a single call to BeginSourceFile");
101 Collector.emplace(CI.getPreprocessor());
102 return true;
103 }
104 void EndSourceFileAction() override {
105 assert(Collector && "BeginSourceFileAction was never called");
106 Result = std::move(*Collector).consume();
107 Result.indexExpandedTokens();
108 }
109
110 std::unique_ptr<ASTConsumer>
111 CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override {
112 return std::make_unique<ASTConsumer>();
113 }
114
115 private:
116 TokenBuffer &Result;
117 std::optional<TokenCollector> Collector;
118 };
119
120 constexpr const char *FileName = "./input.cpp";
121 FS->addFile(FileName, time_t(), llvm::MemoryBuffer::getMemBufferCopy(""));
122 // Prepare to run a compiler.
123 if (!Diags->getClient())
124 Diags->setClient(new IgnoringDiagConsumer);
125 std::vector<const char *> Args = {"tok-test", "-std=c++03", "-fsyntax-only",
126 FileName};
127 CreateInvocationOptions CIOpts;
128 CIOpts.Diags = Diags;
129 CIOpts.VFS = FS;
130 auto CI = createInvocation(Args, std::move(CIOpts));
131 assert(CI);
132 CI->getFrontendOpts().DisableFree = false;
133 CI->getPreprocessorOpts().addRemappedFile(
134 FileName, llvm::MemoryBuffer::getMemBufferCopy(Code).release());
135 CompilerInstance Compiler;
136 Compiler.setInvocation(std::move(CI));
137 Compiler.setDiagnostics(Diags.get());
138 Compiler.setFileManager(FileMgr.get());
139 Compiler.setSourceManager(SourceMgr.get());
140
141 this->Buffer = TokenBuffer(*SourceMgr);
142 RecordTokens Recorder(this->Buffer);
143 ASSERT_TRUE(Compiler.ExecuteAction(Recorder))
144 << "failed to run the frontend";
145 }
146
147 /// Record the tokens and return a test dump of the resulting buffer.
148 std::string collectAndDump(llvm::StringRef Code) {
149 recordTokens(Code);
150 return Buffer.dumpForTests();
151 }
152
153 // Adds a file to the test VFS.
154 void addFile(llvm::StringRef Path, llvm::StringRef Contents) {
155 if (!FS->addFile(Path, time_t(),
156 llvm::MemoryBuffer::getMemBufferCopy(Contents))) {
157 ADD_FAILURE() << "could not add a file to VFS: " << Path;
158 }
159 }
160
161 /// Add a new file, run syntax::tokenize() on the range if any, run it on the
162 /// whole file otherwise and return the results.
163 std::vector<syntax::Token> tokenize(llvm::StringRef Text) {
164 llvm::Annotations Annot(Text);
165 auto FID = SourceMgr->createFileID(
166 llvm::MemoryBuffer::getMemBufferCopy(Annot.code()));
167 // FIXME: pass proper LangOptions.
168 if (Annot.ranges().empty())
169 return syntax::tokenize(FID, *SourceMgr, LangOptions());
170 return syntax::tokenize(
171 syntax::FileRange(FID, Annot.range().Begin, Annot.range().End),
172 *SourceMgr, LangOptions());
173 }
174
175 // Specialized versions of matchers that hide the SourceManager from clients.
176 Matcher<syntax::Token> HasText(std::string Text) const {
177 return ::HasText(Text, SourceMgr.get());
178 }
179 Matcher<syntax::Token> RangeIs(llvm::Annotations::Range R) const {
180 std::pair<SourceLocation, SourceLocation> Ls;
181 Ls.first = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID())
182 .getLocWithOffset(R.Begin);
183 Ls.second = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID())
184 .getLocWithOffset(R.End);
185 return ::RangeIs(Ls);
186 }
187
188 /// Finds a subrange in O(n * m).
189 template <class T, class U, class Eq>
190 llvm::ArrayRef<T> findSubrange(llvm::ArrayRef<U> Subrange,
191 llvm::ArrayRef<T> Range, Eq F) {
192 assert(Subrange.size() >= 1);
193 if (Range.size() < Subrange.size())
194 return llvm::ArrayRef(Range.end(), Range.end());
195 for (auto Begin = Range.begin(), Last = Range.end() - Subrange.size();
196 Begin <= Last; ++Begin) {
197 auto It = Begin;
198 for (auto ItSub = Subrange.begin(); ItSub != Subrange.end();
199 ++ItSub, ++It) {
200 if (!F(*ItSub, *It))
201 goto continue_outer;
202 }
203 return llvm::ArrayRef(Begin, It);
204 continue_outer:;
205 }
206 return llvm::ArrayRef(Range.end(), Range.end());
207 }
208
209 /// Finds a subrange in \p Tokens that match the tokens specified in \p Query.
210 /// The match should be unique. \p Query is a whitespace-separated list of
211 /// tokens to search for.
212 llvm::ArrayRef<syntax::Token>
213 findTokenRange(llvm::StringRef Query, llvm::ArrayRef<syntax::Token> Tokens) {
214 llvm::SmallVector<llvm::StringRef, 8> QueryTokens;
215 Query.split(QueryTokens, ' ', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
216 if (QueryTokens.empty()) {
217 ADD_FAILURE() << "will not look for an empty list of tokens";
218 std::abort();
219 }
220 // An equality test for search.
221 auto TextMatches = [this](llvm::StringRef Q, const syntax::Token &T) {
222 return Q == T.text(*SourceMgr);
223 };
224 // Find a match.
225 auto Found = findSubrange(llvm::ArrayRef(QueryTokens), Tokens, TextMatches);
226 if (Found.begin() == Tokens.end()) {
227 ADD_FAILURE() << "could not find the subrange for " << Query;
228 std::abort();
229 }
230 // Check that the match is unique.
231 if (findSubrange(llvm::ArrayRef(QueryTokens),
232 llvm::ArrayRef(Found.end(), Tokens.end()), TextMatches)
233 .begin() != Tokens.end()) {
234 ADD_FAILURE() << "match is not unique for " << Query;
235 std::abort();
236 }
237 return Found;
238 };
239
240 // Specialized versions of findTokenRange for expanded and spelled tokens.
241 llvm::ArrayRef<syntax::Token> findExpanded(llvm::StringRef Query) {
242 return findTokenRange(Query, Buffer.expandedTokens());
243 }
244 llvm::ArrayRef<syntax::Token> findSpelled(llvm::StringRef Query,
245 FileID File = FileID()) {
246 if (!File.isValid())
247 File = SourceMgr->getMainFileID();
248 return findTokenRange(Query, Buffer.spelledTokens(File));
249 }
250
251 // Data fields.
252 llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
253 new DiagnosticsEngine(new DiagnosticIDs, new DiagnosticOptions);
254 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS =
255 new llvm::vfs::InMemoryFileSystem;
256 llvm::IntrusiveRefCntPtr<FileManager> FileMgr =
257 new FileManager(FileSystemOptions(), FS);
258 llvm::IntrusiveRefCntPtr<SourceManager> SourceMgr =
259 new SourceManager(*Diags, *FileMgr);
260 /// Contains last result of calling recordTokens().
261 TokenBuffer Buffer = TokenBuffer(*SourceMgr);
262 };
263
264 TEST_F(TokenCollectorTest, RawMode) {
265 EXPECT_THAT(tokenize("int main() {}"),
266 ElementsAre(Kind(tok::kw_int),
267 AllOf(HasText("main"), Kind(tok::identifier)),
268 Kind(tok::l_paren), Kind(tok::r_paren),
269 Kind(tok::l_brace), Kind(tok::r_brace)));
270 // Comments are ignored for now.
271 EXPECT_THAT(tokenize("/* foo */int a; // more comments"),
272 ElementsAre(Kind(tok::kw_int),
273 AllOf(HasText("a"), Kind(tok::identifier)),
274 Kind(tok::semi)));
275 EXPECT_THAT(tokenize("int [[main() {]]}"),
276 ElementsAre(AllOf(HasText("main"), Kind(tok::identifier)),
277 Kind(tok::l_paren), Kind(tok::r_paren),
278 Kind(tok::l_brace)));
279 EXPECT_THAT(tokenize("int [[main() { ]]}"),
280 ElementsAre(AllOf(HasText("main"), Kind(tok::identifier)),
281 Kind(tok::l_paren), Kind(tok::r_paren),
282 Kind(tok::l_brace)));
283 // First token is partially parsed, last token is fully included even though
284 // only a part of it is contained in the range.
285 EXPECT_THAT(tokenize("int m[[ain() {ret]]urn 0;}"),
286 ElementsAre(AllOf(HasText("ain"), Kind(tok::identifier)),
287 Kind(tok::l_paren), Kind(tok::r_paren),
288 Kind(tok::l_brace), Kind(tok::kw_return)));
289 }
290
291 TEST_F(TokenCollectorTest, Basic) {
292 std::pair</*Input*/ std::string, /*Expected*/ std::string> TestCases[] = {
293 {"int main() {}",
294 R"(expanded tokens:
295 int main ( ) { }
296 file './input.cpp'
297 spelled tokens:
298 int main ( ) { }
299 no mappings.
300 )"},
301 // All kinds of whitespace are ignored.
302 {"\t\n int\t\n main\t\n (\t\n )\t\n{\t\n }\t\n",
303 R"(expanded tokens:
304 int main ( ) { }
305 file './input.cpp'
306 spelled tokens:
307 int main ( ) { }
308 no mappings.
309 )"},
310 // Annotation tokens are ignored.
311 {R"cpp(
312 #pragma GCC visibility push (public)
313 #pragma GCC visibility pop
314 )cpp",
315 R"(expanded tokens:
316 <empty>
317 file './input.cpp'
318 spelled tokens:
319 # pragma GCC visibility push ( public ) # pragma GCC visibility pop
320 mappings:
321 ['#'_0, '<eof>'_13) => ['<eof>'_0, '<eof>'_0)
322 )"},
323 // Empty files should not crash.
324 {R"cpp()cpp", R"(expanded tokens:
325 <empty>
326 file './input.cpp'
327 spelled tokens:
328 <empty>
329 no mappings.
330 )"},
331 // Should not crash on errors inside '#define' directives. Error is that
332 // stringification (#B) does not refer to a macro parameter.
333 {
334 R"cpp(
335 a
336 #define MACRO() A #B
337 )cpp",
338 R"(expanded tokens:
339 a
340 file './input.cpp'
341 spelled tokens:
342 a # define MACRO ( ) A # B
343 mappings:
344 ['#'_1, '<eof>'_9) => ['<eof>'_1, '<eof>'_1)
345 )"}};
346 for (auto &Test : TestCases)
347 EXPECT_EQ(collectAndDump(Test.first), Test.second)
348 << collectAndDump(Test.first);
349 }
350
351 TEST_F(TokenCollectorTest, Locations) {
352 // Check locations of the tokens.
353 llvm::Annotations Code(R"cpp(
354 $r1[[int]] $r2[[a]] $r3[[=]] $r4[["foo bar baz"]] $r5[[;]]
355 )cpp");
356 recordTokens(Code.code());
357 // Check expanded tokens.
358 EXPECT_THAT(
359 Buffer.expandedTokens(),
360 ElementsAre(AllOf(Kind(tok::kw_int), RangeIs(Code.range("r1"))),
361 AllOf(Kind(tok::identifier), RangeIs(Code.range("r2"))),
362 AllOf(Kind(tok::equal), RangeIs(Code.range("r3"))),
363 AllOf(Kind(tok::string_literal), RangeIs(Code.range("r4"))),
364 AllOf(Kind(tok::semi), RangeIs(Code.range("r5"))),
365 Kind(tok::eof)));
366 // Check spelled tokens.
367 EXPECT_THAT(
368 Buffer.spelledTokens(SourceMgr->getMainFileID()),
369 ElementsAre(AllOf(Kind(tok::kw_int), RangeIs(Code.range("r1"))),
370 AllOf(Kind(tok::identifier), RangeIs(Code.range("r2"))),
371 AllOf(Kind(tok::equal), RangeIs(Code.range("r3"))),
372 AllOf(Kind(tok::string_literal), RangeIs(Code.range("r4"))),
373 AllOf(Kind(tok::semi), RangeIs(Code.range("r5")))));
374
375 auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());
376 for (auto &R : Code.ranges()) {
377 EXPECT_THAT(Buffer.spelledTokenAt(StartLoc.getLocWithOffset(R.Begin)),
378 Pointee(RangeIs(R)));
379 }
380 }
381
382 TEST_F(TokenCollectorTest, MacroDirectives) {
383 // Macro directives are not stored anywhere at the moment.
384 std::string Code = R"cpp(
385 #define FOO a
386 #include "unresolved_file.h"
387 #undef FOO
388 #ifdef X
389 #else
390 #endif
391 #ifndef Y
392 #endif
393 #if 1
394 #elif 2
395 #else
396 #endif
397 #pragma once
398 #pragma something lalala
399
400 int a;
401 )cpp";
402 std::string Expected =
403 "expanded tokens:\n"
404 " int a ;\n"
405 "file './input.cpp'\n"
406 " spelled tokens:\n"
407 " # define FOO a # include \"unresolved_file.h\" # undef FOO "
408 "# ifdef X # else # endif # ifndef Y # endif # if 1 # elif 2 # else "
409 "# endif # pragma once # pragma something lalala int a ;\n"
410 " mappings:\n"
411 " ['#'_0, 'int'_39) => ['int'_0, 'int'_0)\n";
412 EXPECT_EQ(collectAndDump(Code), Expected);
413 }
414
415 TEST_F(TokenCollectorTest, MacroReplacements) {
416 std::pair</*Input*/ std::string, /*Expected*/ std::string> TestCases[] = {
417 // A simple object-like macro.
418 {R"cpp(
419 #define INT int const
420 INT a;
421 )cpp",
422 R"(expanded tokens:
423 int const a ;
424 file './input.cpp'
425 spelled tokens:
426 # define INT int const INT a ;
427 mappings:
428 ['#'_0, 'INT'_5) => ['int'_0, 'int'_0)
429 ['INT'_5, 'a'_6) => ['int'_0, 'a'_2)
430 )"},
431 // A simple function-like macro.
432 {R"cpp(
433 #define INT(a) const int
434 INT(10+10) a;
435 )cpp",
436 R"(expanded tokens:
437 const int a ;
438 file './input.cpp'
439 spelled tokens:
440 # define INT ( a ) const int INT ( 10 + 10 ) a ;
441 mappings:
442 ['#'_0, 'INT'_8) => ['const'_0, 'const'_0)
443 ['INT'_8, 'a'_14) => ['const'_0, 'a'_2)
444 )"},
445 // Recursive macro replacements.
446 {R"cpp(
447 #define ID(X) X
448 #define INT int const
449 ID(ID(INT)) a;
450 )cpp",
451 R"(expanded tokens:
452 int const a ;
453 file './input.cpp'
454 spelled tokens:
455 # define ID ( X ) X # define INT int const ID ( ID ( INT ) ) a ;
456 mappings:
457 ['#'_0, 'ID'_12) => ['int'_0, 'int'_0)
458 ['ID'_12, 'a'_19) => ['int'_0, 'a'_2)
459 )"},
460 // A little more complicated recursive macro replacements.
461 {R"cpp(
462 #define ADD(X, Y) X+Y
463 #define MULT(X, Y) X*Y
464
465 int a = ADD(MULT(1,2), MULT(3,ADD(4,5)));
466 )cpp",
467 "expanded tokens:\n"
468 " int a = 1 * 2 + 3 * 4 + 5 ;\n"
469 "file './input.cpp'\n"
470 " spelled tokens:\n"
471 " # define ADD ( X , Y ) X + Y # define MULT ( X , Y ) X * Y int "
472 "a = ADD ( MULT ( 1 , 2 ) , MULT ( 3 , ADD ( 4 , 5 ) ) ) ;\n"
473 " mappings:\n"
474 " ['#'_0, 'int'_22) => ['int'_0, 'int'_0)\n"
475 " ['ADD'_25, ';'_46) => ['1'_3, ';'_12)\n"},
476 // Empty macro replacement.
477 // FIXME: the #define directives should not be glued together.
478 {R"cpp(
479 #define EMPTY
480 #define EMPTY_FUNC(X)
481 EMPTY
482 EMPTY_FUNC(1+2+3)
483 )cpp",
484 R"(expanded tokens:
485 <empty>
486 file './input.cpp'
487 spelled tokens:
488 # define EMPTY # define EMPTY_FUNC ( X ) EMPTY EMPTY_FUNC ( 1 + 2 + 3 )
489 mappings:
490 ['#'_0, 'EMPTY'_9) => ['<eof>'_0, '<eof>'_0)
491 ['EMPTY'_9, 'EMPTY_FUNC'_10) => ['<eof>'_0, '<eof>'_0)
492 ['EMPTY_FUNC'_10, '<eof>'_18) => ['<eof>'_0, '<eof>'_0)
493 )"},
494 // File ends with a macro replacement.
495 {R"cpp(
496 #define FOO 10+10;
497 int a = FOO
498 )cpp",
499 R"(expanded tokens:
500 int a = 10 + 10 ;
501 file './input.cpp'
502 spelled tokens:
503 # define FOO 10 + 10 ; int a = FOO
504 mappings:
505 ['#'_0, 'int'_7) => ['int'_0, 'int'_0)
506 ['FOO'_10, '<eof>'_11) => ['10'_3, '<eof>'_7)
507 )"},
508 {R"cpp(
509 #define NUM 42
510 #define ID(a) a
511 #define M 1 + ID
512 M(NUM)
513 )cpp",
514 R"(expanded tokens:
515 1 + 42
516 file './input.cpp'
517 spelled tokens:
518 # define NUM 42 # define ID ( a ) a # define M 1 + ID M ( NUM )
519 mappings:
520 ['#'_0, 'M'_17) => ['1'_0, '1'_0)
521 ['M'_17, '<eof>'_21) => ['1'_0, '<eof>'_3)
522 )"},
523 };
524
525 for (auto &Test : TestCases) {
526 std::string Dump = collectAndDump(Test.first);
527 EXPECT_EQ(Test.second, Dump) << Dump;
528 }
529 }
530
531 TEST_F(TokenCollectorTest, SpecialTokens) {
532 // Tokens coming from concatenations.
533 recordTokens(R"cpp(
534 #define CONCAT(a, b) a ## b
535 int a = CONCAT(1, 2);
536 )cpp");
537 EXPECT_THAT(std::vector<syntax::Token>(Buffer.expandedTokens()),
538 Contains(HasText("12")));
539 // Multi-line tokens with slashes at the end.
540 recordTokens("i\\\nn\\\nt");
541 EXPECT_THAT(Buffer.expandedTokens(),
542 ElementsAre(AllOf(Kind(tok::kw_int), HasText("i\\\nn\\\nt")),
543 Kind(tok::eof)));
544 // FIXME: test tokens with digraphs and UCN identifiers.
545 }
546
547 TEST_F(TokenCollectorTest, LateBoundTokens) {
548 // The parser eventually breaks the first '>>' into two tokens ('>' and '>'),
549 // but we choose to record them as a single token (for now).
550 llvm::Annotations Code(R"cpp(
551 template <class T>
552 struct foo { int a; };
553 int bar = foo<foo<int$br[[>>]]().a;
554 int baz = 10 $op[[>>]] 2;
555 )cpp");
556 recordTokens(Code.code());
557 EXPECT_THAT(std::vector<syntax::Token>(Buffer.expandedTokens()),
558 AllOf(Contains(AllOf(Kind(tok::greatergreater),
559 RangeIs(Code.range("br")))),
560 Contains(AllOf(Kind(tok::greatergreater),
561 RangeIs(Code.range("op"))))));
562 }
563
564 TEST_F(TokenCollectorTest, DelayedParsing) {
565 llvm::StringLiteral Code = R"cpp(
566 struct Foo {
567 int method() {
568 // Parser will visit method bodies and initializers multiple times, but
569 // TokenBuffer should only record the first walk over the tokens;
570 return 100;
571 }
572 int a = 10;
573
574 struct Subclass {
575 void foo() {
576 Foo().method();
577 }
578 };
579 };
580 )cpp";
581 std::string ExpectedTokens =
582 "expanded tokens:\n"
583 " struct Foo { int method ( ) { return 100 ; } int a = 10 ; struct "
584 "Subclass { void foo ( ) { Foo ( ) . method ( ) ; } } ; } ;\n";
585 EXPECT_THAT(collectAndDump(Code), StartsWith(ExpectedTokens));
586 }
587
588 TEST_F(TokenCollectorTest, MultiFile) {
589 addFile("./foo.h", R"cpp(
590 #define ADD(X, Y) X+Y
591 int a = 100;
592 #include "bar.h"
593 )cpp");
594 addFile("./bar.h", R"cpp(
595 int b = ADD(1, 2);
596 #define MULT(X, Y) X*Y
597 )cpp");
598 llvm::StringLiteral Code = R"cpp(
599 #include "foo.h"
600 int c = ADD(1, MULT(2,3));
601 )cpp";
602
603 std::string Expected = R"(expanded tokens:
604 int a = 100 ; int b = 1 + 2 ; int c = 1 + 2 * 3 ;
605 file './input.cpp'
606 spelled tokens:
607 # include "foo.h" int c = ADD ( 1 , MULT ( 2 , 3 ) ) ;
608 mappings:
609 ['#'_0, 'int'_3) => ['int'_12, 'int'_12)
610 ['ADD'_6, ';'_17) => ['1'_15, ';'_20)
611 file './foo.h'
612 spelled tokens:
613 # define ADD ( X , Y ) X + Y int a = 100 ; # include "bar.h"
614 mappings:
615 ['#'_0, 'int'_11) => ['int'_0, 'int'_0)
616 ['#'_16, '<eof>'_19) => ['int'_5, 'int'_5)
617 file './bar.h'
618 spelled tokens:
619 int b = ADD ( 1 , 2 ) ; # define MULT ( X , Y ) X * Y
620 mappings:
621 ['ADD'_3, ';'_9) => ['1'_8, ';'_11)
622 ['#'_10, '<eof>'_21) => ['int'_12, 'int'_12)
623 )";
624
625 EXPECT_EQ(Expected, collectAndDump(Code))
626 << "input: " << Code << "\nresults: " << collectAndDump(Code);
627 }
628
629 class TokenBufferTest : public TokenCollectorTest {};
630
631 TEST_F(TokenBufferTest, SpelledByExpanded) {
632 recordTokens(R"cpp(
633 a1 a2 a3 b1 b2
634 )cpp");
635
636 // Expanded and spelled tokens are stored separately.
637 EXPECT_THAT(findExpanded("a1 a2"), Not(SameRange(findSpelled("a1 a2"))));
638 // Searching for subranges of expanded tokens should give the corresponding
639 // spelled ones.
640 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 b1 b2")),
641 ValueIs(SameRange(findSpelled("a1 a2 a3 b1 b2"))));
642 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
643 ValueIs(SameRange(findSpelled("a1 a2 a3"))));
644 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),
645 ValueIs(SameRange(findSpelled("b1 b2"))));
646
647 // Test search on simple macro expansions.
648 recordTokens(R"cpp(
649 #define A a1 a2 a3
650 #define B b1 b2
651
652 A split B
653 )cpp");
654 // Ranges going across expansion boundaries.
655 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1 b2")),
656 ValueIs(SameRange(findSpelled("A split B"))));
657 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
658 ValueIs(SameRange(findSpelled("A split").drop_back())));
659 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),
660 ValueIs(SameRange(findSpelled("split B").drop_front())));
661 // Ranges not fully covering macro invocations should fail.
662 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2")), std::nullopt);
663 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("b2")), std::nullopt);
664 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a2 a3 split b1 b2")),
665 std::nullopt);
666
667 // Recursive macro invocations.
668 recordTokens(R"cpp(
669 #define ID(x) x
670 #define B b1 b2
671
672 ID(ID(ID(a1) a2 a3)) split ID(B)
673 )cpp");
674
675 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),
676 ValueIs(SameRange(findSpelled("( B").drop_front())));
677 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1 b2")),
678 ValueIs(SameRange(findSpelled(
679 "ID ( ID ( ID ( a1 ) a2 a3 ) ) split ID ( B )"))));
680 // Mixed ranges with expanded and spelled tokens.
681 EXPECT_THAT(
682 Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split")),
683 ValueIs(SameRange(findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) ) split"))));
684 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("split b1 b2")),
685 ValueIs(SameRange(findSpelled("split ID ( B )"))));
686 // Macro arguments
687 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1")),
688 ValueIs(SameRange(findSpelled("a1"))));
689 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a2")),
690 ValueIs(SameRange(findSpelled("a2"))));
691 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a3")),
692 ValueIs(SameRange(findSpelled("a3"))));
693 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2")),
694 ValueIs(SameRange(findSpelled("ID ( a1 ) a2"))));
695 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
696 ValueIs(SameRange(findSpelled("ID ( a1 ) a2 a3"))));
697
698 // Empty macro expansions.
699 recordTokens(R"cpp(
700 #define EMPTY
701 #define ID(X) X
702
703 EMPTY EMPTY ID(1 2 3) EMPTY EMPTY split1
704 EMPTY EMPTY ID(4 5 6) split2
705 ID(7 8 9) EMPTY EMPTY
706 )cpp");
707 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("1 2 3")),
708 ValueIs(SameRange(findSpelled("1 2 3"))));
709 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("4 5 6")),
710 ValueIs(SameRange(findSpelled("4 5 6"))));
711 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("7 8 9")),
712 ValueIs(SameRange(findSpelled("7 8 9"))));
713
714 // Empty mappings coming from various directives.
715 recordTokens(R"cpp(
716 #define ID(X) X
717 ID(1)
718 #pragma lalala
719 not_mapped
720 )cpp");
721 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("not_mapped")),
722 ValueIs(SameRange(findSpelled("not_mapped"))));
723
724 // Multiple macro arguments
725 recordTokens(R"cpp(
726 #define ID(X) X
727 #define ID2(X, Y) X Y
728
729 ID2(ID(a1), ID(a2) a3) ID2(a4, a5 a6 a7)
730 )cpp");
731 // Should fail, spans multiple arguments.
732 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2")), std::nullopt);
733 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a2 a3")),
734 ValueIs(SameRange(findSpelled("ID ( a2 ) a3"))));
735 EXPECT_THAT(
736 Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
737 ValueIs(SameRange(findSpelled("ID2 ( ID ( a1 ) , ID ( a2 ) a3 )"))));
738 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a5 a6")),
739 ValueIs(SameRange(findSpelled("a5 a6"))));
740 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a4 a5 a6 a7")),
741 ValueIs(SameRange(findSpelled("ID2 ( a4 , a5 a6 a7 )"))));
742 // Should fail, spans multiple invocations.
743 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 a4")),
744 std::nullopt);
745
746 // https://github.com/clangd/clangd/issues/1289
747 recordTokens(R"cpp(
748 #define FOO(X) foo(X)
749 #define INDIRECT FOO(y)
750 INDIRECT // expands to foo(y)
751 )cpp");
752 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("y")), std::nullopt);
753
754 recordTokens(R"cpp(
755 #define FOO(X) a X b
756 FOO(y)
757 )cpp");
758 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("y")),
759 ValueIs(SameRange(findSpelled("y"))));
760
761 recordTokens(R"cpp(
762 #define ID(X) X
763 #define BAR ID(1)
764 BAR
765 )cpp");
766 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("1")),
767 ValueIs(SameRange(findSpelled(") BAR").drop_front())));
768
769 // Critical cases for mapping of Prev/Next in spelledForExpandedSlow.
770 recordTokens(R"cpp(
771 #define ID(X) X
772 ID(prev ID(good))
773 #define LARGE ID(prev ID(bad))
774 LARGE
775 )cpp");
776 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),
777 ValueIs(SameRange(findSpelled("good"))));
778 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), std::nullopt);
779
780 recordTokens(R"cpp(
781 #define PREV prev
782 #define ID(X) X
783 PREV ID(good)
784 #define LARGE PREV ID(bad)
785 LARGE
786 )cpp");
787 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),
788 ValueIs(SameRange(findSpelled("good"))));
789 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), std::nullopt);
790
791 recordTokens(R"cpp(
792 #define ID(X) X
793 #define ID2(X, Y) X Y
794 ID2(prev, ID(good))
795 #define LARGE ID2(prev, bad)
796 LARGE
797 )cpp");
798 EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("good")),
799 ValueIs(SameRange(findSpelled("good"))));
800 EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("bad")), std::nullopt);
801 }
802
803 TEST_F(TokenBufferTest, ExpandedTokensForRange) {
804 recordTokens(R"cpp(
805 #define SIGN(X) X##_washere
806 A SIGN(B) C SIGN(D) E SIGN(F) G
807 )cpp");
808
809 SourceRange R(findExpanded("C").front().location(),
810 findExpanded("F_washere").front().location());
811 // Expanded and spelled tokens are stored separately.
812 EXPECT_THAT(Buffer.expandedTokens(R),
813 SameRange(findExpanded("C D_washere E F_washere")));
814 EXPECT_THAT(Buffer.expandedTokens(SourceRange()), testing::IsEmpty());
815 }
816
817 TEST_F(TokenBufferTest, ExpansionsOverlapping) {
818 // Object-like macro expansions.
819 recordTokens(R"cpp(
820 #define FOO 3+4
821 int a = FOO 1;
822 int b = FOO 2;
823 )cpp");
824
825 llvm::ArrayRef<syntax::Token> Foo1 = findSpelled("FOO 1");
826 EXPECT_THAT(
827 Buffer.expansionStartingAt(Foo1.data()),
828 ValueIs(IsExpansion(SameRange(Foo1.drop_back()),
829 SameRange(findExpanded("3 + 4 1").drop_back()))));
830 EXPECT_THAT(
831 Buffer.expansionsOverlapping(Foo1),
832 ElementsAre(IsExpansion(SameRange(Foo1.drop_back()),
833 SameRange(findExpanded("3 + 4 1").drop_back()))));
834
835 llvm::ArrayRef<syntax::Token> Foo2 = findSpelled("FOO 2");
836 EXPECT_THAT(
837 Buffer.expansionStartingAt(Foo2.data()),
838 ValueIs(IsExpansion(SameRange(Foo2.drop_back()),
839 SameRange(findExpanded("3 + 4 2").drop_back()))));
840 EXPECT_THAT(
841 Buffer.expansionsOverlapping(llvm::ArrayRef(Foo1.begin(), Foo2.end())),
842 ElementsAre(IsExpansion(SameRange(Foo1.drop_back()), _),
843 IsExpansion(SameRange(Foo2.drop_back()), _)));
844
845 // Function-like macro expansions.
846 recordTokens(R"cpp(
847 #define ID(X) X
848 int a = ID(1+2+3);
849 int b = ID(ID(2+3+4));
850 )cpp");
851
852 llvm::ArrayRef<syntax::Token> ID1 = findSpelled("ID ( 1 + 2 + 3 )");
853 EXPECT_THAT(Buffer.expansionStartingAt(&ID1.front()),
854 ValueIs(IsExpansion(SameRange(ID1),
855 SameRange(findExpanded("1 + 2 + 3")))));
856 // Only the first spelled token should be found.
857 for (const auto &T : ID1.drop_front())
858 EXPECT_EQ(Buffer.expansionStartingAt(&T), std::nullopt);
859
860 llvm::ArrayRef<syntax::Token> ID2 = findSpelled("ID ( ID ( 2 + 3 + 4 ) )");
861 EXPECT_THAT(Buffer.expansionStartingAt(&ID2.front()),
862 ValueIs(IsExpansion(SameRange(ID2),
863 SameRange(findExpanded("2 + 3 + 4")))));
864 // Only the first spelled token should be found.
865 for (const auto &T : ID2.drop_front())
866 EXPECT_EQ(Buffer.expansionStartingAt(&T), std::nullopt);
867
868 EXPECT_THAT(Buffer.expansionsOverlapping(llvm::ArrayRef(
869 findSpelled("1 + 2").data(), findSpelled("4").data())),
870 ElementsAre(IsExpansion(SameRange(ID1), _),
871 IsExpansion(SameRange(ID2), _)));
872
873 // PP directives.
874 recordTokens(R"cpp(
875 #define FOO 1
876 int a = FOO;
877 #pragma once
878 int b = 1;
879 )cpp");
880
881 llvm::ArrayRef<syntax::Token> DefineFoo = findSpelled("# define FOO 1");
882 EXPECT_THAT(
883 Buffer.expansionStartingAt(&DefineFoo.front()),
884 ValueIs(IsExpansion(SameRange(DefineFoo),
885 SameRange(findExpanded("int a").take_front(0)))));
886 // Only the first spelled token should be found.
887 for (const auto &T : DefineFoo.drop_front())
888 EXPECT_EQ(Buffer.expansionStartingAt(&T), std::nullopt);
889
890 llvm::ArrayRef<syntax::Token> PragmaOnce = findSpelled("# pragma once");
891 EXPECT_THAT(
892 Buffer.expansionStartingAt(&PragmaOnce.front()),
893 ValueIs(IsExpansion(SameRange(PragmaOnce),
894 SameRange(findExpanded("int b").take_front(0)))));
895 // Only the first spelled token should be found.
896 for (const auto &T : PragmaOnce.drop_front())
897 EXPECT_EQ(Buffer.expansionStartingAt(&T), std::nullopt);
898
899 EXPECT_THAT(
900 Buffer.expansionsOverlapping(findSpelled("FOO ; # pragma")),
901 ElementsAre(IsExpansion(SameRange(findSpelled("FOO ;").drop_back()), _),
902 IsExpansion(SameRange(PragmaOnce), _)));
903 }
904
905 TEST_F(TokenBufferTest, TokensToFileRange) {
906 addFile("./foo.h", "token_from_header");
907 llvm::Annotations Code(R"cpp(
908 #define FOO token_from_expansion
909 #include "./foo.h"
910 $all[[$i[[int]] a = FOO;]]
911 )cpp");
912 recordTokens(Code.code());
913
914 auto &SM = *SourceMgr;
915
916 // Two simple examples.
917 auto Int = findExpanded("int").front();
918 auto Semi = findExpanded(";").front();
919 EXPECT_EQ(Int.range(SM), FileRange(SM.getMainFileID(), Code.range("i").Begin,
920 Code.range("i").End));
921 EXPECT_EQ(syntax::Token::range(SM, Int, Semi),
922 FileRange(SM.getMainFileID(), Code.range("all").Begin,
923 Code.range("all").End));
924 // We don't test assertion failures because death tests are slow.
925 }
926
927 TEST_F(TokenBufferTest, MacroExpansions) {
928 llvm::Annotations Code(R"cpp(
929 #define FOO B
930 #define FOO2 BA
931 #define CALL(X) int X
932 #define G CALL(FOO2)
933 int B;
934 $macro[[FOO]];
935 $macro[[CALL]](A);
936 $macro[[G]];
937 )cpp");
938 recordTokens(Code.code());
939 auto &SM = *SourceMgr;
940 auto Expansions = Buffer.macroExpansions(SM.getMainFileID());
941 std::vector<FileRange> ExpectedMacroRanges;
942 for (auto Range : Code.ranges("macro"))
943 ExpectedMacroRanges.push_back(
944 FileRange(SM.getMainFileID(), Range.Begin, Range.End));
945 std::vector<FileRange> ActualMacroRanges;
946 for (auto Expansion : Expansions)
947 ActualMacroRanges.push_back(Expansion->range(SM));
948 EXPECT_EQ(ExpectedMacroRanges, ActualMacroRanges);
949 }
950
951 TEST_F(TokenBufferTest, Touching) {
952 llvm::Annotations Code("^i^nt^ ^a^b^=^1;^");
953 recordTokens(Code.code());
954
955 auto Touching = [&](int Index) {
956 SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),
957 Code.points()[Index]);
958 return spelledTokensTouching(Loc, Buffer);
959 };
960 auto Identifier = [&](int Index) {
961 SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),
962 Code.points()[Index]);
963 const syntax::Token *Tok = spelledIdentifierTouching(Loc, Buffer);
964 return Tok ? Tok->text(*SourceMgr) : "";
965 };
966
967 EXPECT_THAT(Touching(0), SameRange(findSpelled("int")));
968 EXPECT_EQ(Identifier(0), "");
969 EXPECT_THAT(Touching(1), SameRange(findSpelled("int")));
970 EXPECT_EQ(Identifier(1), "");
971 EXPECT_THAT(Touching(2), SameRange(findSpelled("int")));
972 EXPECT_EQ(Identifier(2), "");
973
974 EXPECT_THAT(Touching(3), SameRange(findSpelled("ab")));
975 EXPECT_EQ(Identifier(3), "ab");
976 EXPECT_THAT(Touching(4), SameRange(findSpelled("ab")));
977 EXPECT_EQ(Identifier(4), "ab");
978
979 EXPECT_THAT(Touching(5), SameRange(findSpelled("ab =")));
980 EXPECT_EQ(Identifier(5), "ab");
981
982 EXPECT_THAT(Touching(6), SameRange(findSpelled("= 1")));
983 EXPECT_EQ(Identifier(6), "");
984
985 EXPECT_THAT(Touching(7), SameRange(findSpelled(";")));
986 EXPECT_EQ(Identifier(7), "");
987
988 ASSERT_EQ(Code.points().size(), 8u);
989 }
990
991 TEST_F(TokenBufferTest, ExpandedBySpelled) {
992 recordTokens(R"cpp(
993 a1 a2 a3 b1 b2
994 )cpp");
995 // Expanded and spelled tokens are stored separately.
996 EXPECT_THAT(findExpanded("a1 a2"), Not(SameRange(findSpelled("a1 a2"))));
997 // Searching for subranges of expanded tokens should give the corresponding
998 // spelled ones.
999 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1 a2 a3 b1 b2")),
1000 ElementsAre(SameRange(findExpanded("a1 a2 a3 b1 b2"))));
1001 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1 a2 a3")),
1002 ElementsAre(SameRange(findExpanded("a1 a2 a3"))));
1003 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("b1 b2")),
1004 ElementsAre(SameRange(findExpanded("b1 b2"))));
1005
1006 // Test search on simple macro expansions.
1007 recordTokens(R"cpp(
1008 #define A a1 a2 a3
1009 #define B b1 b2
1010
1011 A split B
1012 )cpp");
1013 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("A split B")),
1014 ElementsAre(SameRange(findExpanded("a1 a2 a3 split b1 b2"))));
1015 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("A split").drop_back()),
1016 ElementsAre(SameRange(findExpanded("a1 a2 a3"))));
1017 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("split B").drop_front()),
1018 ElementsAre(SameRange(findExpanded("b1 b2"))));
1019
1020 // Ranges not fully covering macro expansions should fail.
1021 recordTokens(R"cpp(
1022 #define ID(x) x
1023
1024 ID(a)
1025 )cpp");
1026 // Spelled don't cover entire mapping (missing ID token) -> empty result
1027 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("( a )")), IsEmpty());
1028 // Spelled don't cover entire mapping (missing ) token) -> empty result
1029 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( a")), IsEmpty());
1030
1031 // Recursive macro invocations.
1032 recordTokens(R"cpp(
1033 #define ID(x) x
1034 #define B b1 b2
1035
1036 ID(ID(ID(a1) a2 a3)) split ID(B)
1037 )cpp");
1038
1039 EXPECT_THAT(
1040 Buffer.expandedForSpelled(findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) )")),
1041 ElementsAre(SameRange(findExpanded("a1 a2 a3"))));
1042 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( B )")),
1043 ElementsAre(SameRange(findExpanded("b1 b2"))));
1044 EXPECT_THAT(Buffer.expandedForSpelled(
1045 findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) ) split ID ( B )")),
1046 ElementsAre(SameRange(findExpanded("a1 a2 a3 split b1 b2"))));
1047 // FIXME: these should succeed, but we do not support macro arguments yet.
1048 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1")), IsEmpty());
1049 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( a1 ) a2")),
1050 IsEmpty());
1051
1052 // Empty macro expansions.
1053 recordTokens(R"cpp(
1054 #define EMPTY
1055 #define ID(X) X
1056
1057 EMPTY EMPTY ID(1 2 3) EMPTY EMPTY split1
1058 EMPTY EMPTY ID(4 5 6) split2
1059 ID(7 8 9) EMPTY EMPTY
1060 )cpp");
1061 // Covered by empty expansions on one of both of the sides.
1062 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 1 2 3 )")),
1063 ElementsAre(SameRange(findExpanded("1 2 3"))));
1064 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 4 5 6 )")),
1065 ElementsAre(SameRange(findExpanded("4 5 6"))));
1066 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 7 8 9 )")),
1067 ElementsAre(SameRange(findExpanded("7 8 9"))));
1068 // Including the empty macro expansions on the side.
1069 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("EMPTY ID ( 1 2 3 )")),
1070 ElementsAre(SameRange(findExpanded("1 2 3"))));
1071 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 1 2 3 ) EMPTY")),
1072 ElementsAre(SameRange(findExpanded("1 2 3"))));
1073 EXPECT_THAT(
1074 Buffer.expandedForSpelled(findSpelled("EMPTY ID ( 1 2 3 ) EMPTY")),
1075 ElementsAre(SameRange(findExpanded("1 2 3"))));
1076
1077 // Empty mappings coming from various directives.
1078 recordTokens(R"cpp(
1079 #define ID(X) X
1080 ID(1)
1081 #pragma lalala
1082 not_mapped
1083 )cpp");
1084 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("# define ID ( X ) X")),
1085 IsEmpty());
1086 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("# pragma lalala")),
1087 IsEmpty());
1088
1089 // Empty macro expansion.
1090 recordTokens(R"cpp(
1091 #define EMPTY
1092 EMPTY int a = 100;
1093 )cpp");
1094 EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("EMPTY int").drop_back()),
1095 IsEmpty());
1096 }
1097
1098 TEST_F(TokenCollectorTest, Pragmas) {
1099 // Tokens coming from concatenations.
1100 recordTokens(R"cpp(
1101 void foo() {
1102 #pragma unroll 4
1103 for(int i=0;i<4;++i);
1104 }
1105 )cpp");
1106 }
1107 } // namespace