Debian Codegraph @

llvm-project/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp

raw
1 //===-- TUSchedulerTests.cpp ------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "Annotations.h"
10 #include "ClangdServer.h"
11 #include "Diagnostics.h"
12 #include "GlobalCompilationDatabase.h"
13 #include "Matchers.h"
14 #include "ParsedAST.h"
15 #include "Preamble.h"
16 #include "TUScheduler.h"
17 #include "TestFS.h"
18 #include "TestIndex.h"
19 #include "support/Cancellation.h"
20 #include "support/Context.h"
21 #include "support/Path.h"
22 #include "support/TestTracer.h"
23 #include "support/Threading.h"
24 #include "support/ThreadsafeFS.h"
25 #include "clang/Basic/DiagnosticDriver.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/FunctionExtras.h"
28 #include "llvm/ADT/ScopeExit.h"
29 #include "llvm/ADT/StringExtras.h"
30 #include "llvm/ADT/StringMap.h"
31 #include "llvm/ADT/StringRef.h"
32 #include "gmock/gmock.h"
33 #include "gtest/gtest.h"
34 #include <algorithm>
35 #include <atomic>
36 #include <chrono>
37 #include <cstdint>
38 #include <memory>
39 #include <optional>
40 #include <string>
41 #include <utility>
42
43 namespace clang {
44 namespace clangd {
45 namespace {
46
47 using ::testing::AllOf;
48 using ::testing::AnyOf;
49 using ::testing::Contains;
50 using ::testing::Each;
51 using ::testing::ElementsAre;
52 using ::testing::Eq;
53 using ::testing::Field;
54 using ::testing::IsEmpty;
55 using ::testing::Not;
56 using ::testing::Pair;
57 using ::testing::Pointee;
58 using ::testing::SizeIs;
59 using ::testing::UnorderedElementsAre;
60
61 MATCHER_P2(TUState, PreambleActivity, ASTActivity, "") {
62 if (arg.PreambleActivity != PreambleActivity) {
63 *result_listener << "preamblestate is "
64 << static_cast<uint8_t>(arg.PreambleActivity);
65 return false;
66 }
67 if (arg.ASTActivity.K != ASTActivity) {
68 *result_listener << "aststate is " << arg.ASTActivity.K;
69 return false;
70 }
71 return true;
72 }
73
74 // Simple ContextProvider to verify the provider is invoked & contexts are used.
75 static Key<std::string> BoundPath;
76 Context bindPath(PathRef F) {
77 return Context::current().derive(BoundPath, F.str());
78 }
79 llvm::StringRef boundPath() {
80 const std::string *V = Context::current().get(BoundPath);
81 return V ? *V : llvm::StringRef("");
82 }
83
84 TUScheduler::Options optsForTest() {
85 TUScheduler::Options Opts(ClangdServer::optsForTest());
86 Opts.ContextProvider = bindPath;
87 return Opts;
88 }
89
90 class TUSchedulerTests : public ::testing::Test {
91 protected:
92 ParseInputs getInputs(PathRef File, std::string Contents) {
93 ParseInputs Inputs;
94 Inputs.CompileCommand = *CDB.getCompileCommand(File);
95 Inputs.TFS = &FS;
96 Inputs.Contents = std::move(Contents);
97 Inputs.Opts = ParseOptions();
98 return Inputs;
99 }
100
101 void updateWithCallback(TUScheduler &S, PathRef File,
102 llvm::StringRef Contents, WantDiagnostics WD,
103 llvm::unique_function<void()> CB) {
104 updateWithCallback(S, File, getInputs(File, std::string(Contents)), WD,
105 std::move(CB));
106 }
107
108 void updateWithCallback(TUScheduler &S, PathRef File, ParseInputs Inputs,
109 WantDiagnostics WD,
110 llvm::unique_function<void()> CB) {
111 WithContextValue Ctx(llvm::make_scope_exit(std::move(CB)));
112 S.update(File, Inputs, WD);
113 }
114
115 static Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>
116 DiagsCallbackKey;
117
118 /// A diagnostics callback that should be passed to TUScheduler when it's used
119 /// in updateWithDiags.
120 static std::unique_ptr<ParsingCallbacks> captureDiags() {
121 class CaptureDiags : public ParsingCallbacks {
122 public:
123 void onMainAST(PathRef File, ParsedAST &AST, PublishFn Publish) override {
124 reportDiagnostics(File, *AST.getDiagnostics(), Publish);
125 }
126
127 void onFailedAST(PathRef File, llvm::StringRef Version,
128 std::vector<Diag> Diags, PublishFn Publish) override {
129 reportDiagnostics(File, Diags, Publish);
130 }
131
132 private:
133 void reportDiagnostics(PathRef File, llvm::ArrayRef<Diag> Diags,
134 PublishFn Publish) {
135 auto *D = Context::current().get(DiagsCallbackKey);
136 if (!D)
137 return;
138 Publish([&]() {
139 const_cast<
140 llvm::unique_function<void(PathRef, std::vector<Diag>)> &> (*D)(
141 File, std::move(Diags));
142 });
143 }
144 };
145 return std::make_unique<CaptureDiags>();
146 }
147
148 /// Schedule an update and call \p CB with the diagnostics it produces, if
149 /// any. The TUScheduler should be created with captureDiags as a
150 /// DiagsCallback for this to work.
151 void updateWithDiags(TUScheduler &S, PathRef File, ParseInputs Inputs,
152 WantDiagnostics WD,
153 llvm::unique_function<void(std::vector<Diag>)> CB) {
154 Path OrigFile = File.str();
155 WithContextValue Ctx(DiagsCallbackKey,
156 [OrigFile, CB = std::move(CB)](
157 PathRef File, std::vector<Diag> Diags) mutable {
158 assert(File == OrigFile);
159 CB(std::move(Diags));
160 });
161 S.update(File, std::move(Inputs), WD);
162 }
163
164 void updateWithDiags(TUScheduler &S, PathRef File, llvm::StringRef Contents,
165 WantDiagnostics WD,
166 llvm::unique_function<void(std::vector<Diag>)> CB) {
167 return updateWithDiags(S, File, getInputs(File, std::string(Contents)), WD,
168 std::move(CB));
169 }
170
171 MockFS FS;
172 MockCompilationDatabase CDB;
173 };
174
175 Key<llvm::unique_function<void(PathRef File, std::vector<Diag>)>>
176 TUSchedulerTests::DiagsCallbackKey;
177
178 TEST_F(TUSchedulerTests, MissingFiles) {
179 TUScheduler S(CDB, optsForTest());
180
181 auto Added = testPath("added.cpp");
182 FS.Files[Added] = "x";
183
184 auto Missing = testPath("missing.cpp");
185 FS.Files[Missing] = "";
186
187 S.update(Added, getInputs(Added, "x"), WantDiagnostics::No);
188
189 // Assert each operation for missing file is an error (even if it's
190 // available in VFS).
191 S.runWithAST("", Missing,
192 [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });
193 S.runWithPreamble(
194 "", Missing, TUScheduler::Stale,
195 [&](Expected<InputsAndPreamble> Preamble) { EXPECT_ERROR(Preamble); });
196 // remove() shouldn't crash on missing files.
197 S.remove(Missing);
198
199 // Assert there aren't any errors for added file.
200 S.runWithAST("", Added,
201 [&](Expected<InputsAndAST> AST) { EXPECT_TRUE(bool(AST)); });
202 S.runWithPreamble("", Added, TUScheduler::Stale,
203 [&](Expected<InputsAndPreamble> Preamble) {
204 EXPECT_TRUE(bool(Preamble));
205 });
206 S.remove(Added);
207
208 // Assert that all operations fail after removing the file.
209 S.runWithAST("", Added,
210 [&](Expected<InputsAndAST> AST) { EXPECT_ERROR(AST); });
211 S.runWithPreamble("", Added, TUScheduler::Stale,
212 [&](Expected<InputsAndPreamble> Preamble) {
213 ASSERT_FALSE(bool(Preamble));
214 llvm::consumeError(Preamble.takeError());
215 });
216 // remove() shouldn't crash on missing files.
217 S.remove(Added);
218 }
219
220 TEST_F(TUSchedulerTests, WantDiagnostics) {
221 std::atomic<int> CallbackCount(0);
222 {
223 // To avoid a racy test, don't allow tasks to actually run on the worker
224 // thread until we've scheduled them all.
225 Notification Ready;
226 TUScheduler S(CDB, optsForTest(), captureDiags());
227 auto Path = testPath("foo.cpp");
228 updateWithDiags(S, Path, "", WantDiagnostics::Yes,
229 [&](std::vector<Diag>) { Ready.wait(); });
230 updateWithDiags(S, Path, "request diags", WantDiagnostics::Yes,
231 [&](std::vector<Diag>) { ++CallbackCount; });
232 updateWithDiags(S, Path, "auto (clobbered)", WantDiagnostics::Auto,
233 [&](std::vector<Diag>) {
234 ADD_FAILURE()
235 << "auto should have been cancelled by auto";
236 });
237 updateWithDiags(S, Path, "request no diags", WantDiagnostics::No,
238 [&](std::vector<Diag>) {
239 ADD_FAILURE() << "no diags should not be called back";
240 });
241 updateWithDiags(S, Path, "auto (produces)", WantDiagnostics::Auto,
242 [&](std::vector<Diag>) { ++CallbackCount; });
243 Ready.notify();
244
245 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
246 }
247 EXPECT_EQ(2, CallbackCount);
248 }
249
250 TEST_F(TUSchedulerTests, Debounce) {
251 auto Opts = optsForTest();
252 Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(500));
253 TUScheduler S(CDB, Opts, captureDiags());
254 auto Path = testPath("foo.cpp");
255 // Issue a write that's going to be debounced away.
256 updateWithDiags(S, Path, "auto (debounced)", WantDiagnostics::Auto,
257 [&](std::vector<Diag>) {
258 ADD_FAILURE()
259 << "auto should have been debounced and canceled";
260 });
261 // Sleep a bit to verify that it's really debounce that's holding diagnostics.
262 std::this_thread::sleep_for(std::chrono::milliseconds(50));
263
264 // Issue another write, this time we'll wait for its diagnostics.
265 Notification N;
266 updateWithDiags(S, Path, "auto (timed out)", WantDiagnostics::Auto,
267 [&](std::vector<Diag>) { N.notify(); });
268 EXPECT_TRUE(N.wait(timeoutSeconds(5)));
269
270 // Once we start shutting down the TUScheduler, this one becomes a dead write.
271 updateWithDiags(S, Path, "auto (discarded)", WantDiagnostics::Auto,
272 [&](std::vector<Diag>) {
273 ADD_FAILURE()
274 << "auto should have been discarded (dead write)";
275 });
276 }
277
278 TEST_F(TUSchedulerTests, Cancellation) {
279 // We have the following update/read sequence
280 // U0
281 // U1(WantDiags=Yes) <-- cancelled
282 // R1 <-- cancelled
283 // U2(WantDiags=Yes) <-- cancelled
284 // R2A <-- cancelled
285 // R2B
286 // U3(WantDiags=Yes)
287 // R3 <-- cancelled
288 std::vector<StringRef> DiagsSeen, ReadsSeen, ReadsCanceled;
289 {
290 Notification Proceed; // Ensure we schedule everything.
291 TUScheduler S(CDB, optsForTest(), captureDiags());
292 auto Path = testPath("foo.cpp");
293 // Helper to schedule a named update and return a function to cancel it.
294 auto Update = [&](StringRef ID) -> Canceler {
295 auto T = cancelableTask();
296 WithContext C(std::move(T.first));
297 updateWithDiags(
298 S, Path, ("//" + ID).str(), WantDiagnostics::Yes,
299 [&, ID](std::vector<Diag> Diags) { DiagsSeen.push_back(ID); });
300 return std::move(T.second);
301 };
302 // Helper to schedule a named read and return a function to cancel it.
303 auto Read = [&](StringRef ID) -> Canceler {
304 auto T = cancelableTask();
305 WithContext C(std::move(T.first));
306 S.runWithAST(ID, Path, [&, ID](llvm::Expected<InputsAndAST> E) {
307 if (auto Err = E.takeError()) {
308 if (Err.isA<CancelledError>()) {
309 ReadsCanceled.push_back(ID);
310 consumeError(std::move(Err));
311 } else {
312 ADD_FAILURE() << "Non-cancelled error for " << ID << ": "
313 << llvm::toString(std::move(Err));
314 }
315 } else {
316 ReadsSeen.push_back(ID);
317 }
318 });
319 return std::move(T.second);
320 };
321
322 updateWithCallback(S, Path, "", WantDiagnostics::Yes,
323 [&]() { Proceed.wait(); });
324 // The second parens indicate cancellation, where present.
325 Update("U1")();
326 Read("R1")();
327 Update("U2")();
328 Read("R2A")();
329 Read("R2B");
330 Update("U3");
331 Read("R3")();
332 Proceed.notify();
333
334 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
335 }
336 EXPECT_THAT(DiagsSeen, ElementsAre("U2", "U3"))
337 << "U1 and all dependent reads were cancelled. "
338 "U2 has a dependent read R2A. "
339 "U3 was not cancelled.";
340 EXPECT_THAT(ReadsSeen, ElementsAre("R2B"))
341 << "All reads other than R2B were cancelled";
342 EXPECT_THAT(ReadsCanceled, ElementsAre("R1", "R2A", "R3"))
343 << "All reads other than R2B were cancelled";
344 }
345
346 TEST_F(TUSchedulerTests, InvalidationNoCrash) {
347 auto Path = testPath("foo.cpp");
348 TUScheduler S(CDB, optsForTest(), captureDiags());
349
350 Notification StartedRunning;
351 Notification ScheduledChange;
352 // We expect invalidation logic to not crash by trying to invalidate a running
353 // request.
354 S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
355 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
356 S.runWithAST(
357 "invalidatable-but-running", Path,
358 [&](llvm::Expected<InputsAndAST> AST) {
359 StartedRunning.notify();
360 ScheduledChange.wait();
361 ASSERT_TRUE(bool(AST));
362 },
363 TUScheduler::InvalidateOnUpdate);
364 StartedRunning.wait();
365 S.update(Path, getInputs(Path, ""), WantDiagnostics::Auto);
366 ScheduledChange.notify();
367 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
368 }
369
370 TEST_F(TUSchedulerTests, Invalidation) {
371 auto Path = testPath("foo.cpp");
372 TUScheduler S(CDB, optsForTest(), captureDiags());
373 std::atomic<int> Builds(0), Actions(0);
374
375 Notification Start;
376 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
377 ++Builds;
378 Start.wait();
379 });
380 S.runWithAST(
381 "invalidatable", Path,
382 [&](llvm::Expected<InputsAndAST> AST) {
383 ++Actions;
384 EXPECT_FALSE(bool(AST));
385 llvm::Error E = AST.takeError();
386 EXPECT_TRUE(E.isA<CancelledError>());
387 handleAllErrors(std::move(E), [&](const CancelledError &E) {
388 EXPECT_EQ(E.Reason, static_cast<int>(ErrorCode::ContentModified));
389 });
390 },
391 TUScheduler::InvalidateOnUpdate);
392 S.runWithAST(
393 "not-invalidatable", Path,
394 [&](llvm::Expected<InputsAndAST> AST) {
395 ++Actions;
396 EXPECT_TRUE(bool(AST));
397 },
398 TUScheduler::NoInvalidation);
399 updateWithDiags(S, Path, "b", WantDiagnostics::Auto, [&](std::vector<Diag>) {
400 ++Builds;
401 ADD_FAILURE() << "Shouldn't build, all dependents invalidated";
402 });
403 S.runWithAST(
404 "invalidatable", Path,
405 [&](llvm::Expected<InputsAndAST> AST) {
406 ++Actions;
407 EXPECT_FALSE(bool(AST));
408 llvm::Error E = AST.takeError();
409 EXPECT_TRUE(E.isA<CancelledError>());
410 consumeError(std::move(E));
411 },
412 TUScheduler::InvalidateOnUpdate);
413 updateWithDiags(S, Path, "c", WantDiagnostics::Auto,
414 [&](std::vector<Diag>) { ++Builds; });
415 S.runWithAST(
416 "invalidatable", Path,
417 [&](llvm::Expected<InputsAndAST> AST) {
418 ++Actions;
419 EXPECT_TRUE(bool(AST)) << "Shouldn't be invalidated, no update follows";
420 },
421 TUScheduler::InvalidateOnUpdate);
422 Start.notify();
423 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
424
425 EXPECT_EQ(2, Builds.load()) << "Middle build should be skipped";
426 EXPECT_EQ(4, Actions.load()) << "All actions should run (some with error)";
427 }
428
429 // We don't invalidate requests for updates that don't change the file content.
430 // These are mostly "refresh this file" events synthesized inside clangd itself.
431 // (Usually the AST rebuild is elided after verifying that all inputs are
432 // unchanged, but invalidation decisions happen earlier and so independently).
433 // See https://github.com/clangd/clangd/issues/620
434 TEST_F(TUSchedulerTests, InvalidationUnchanged) {
435 auto Path = testPath("foo.cpp");
436 TUScheduler S(CDB, optsForTest(), captureDiags());
437 std::atomic<int> Actions(0);
438
439 Notification Start;
440 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
441 Start.wait();
442 });
443 S.runWithAST(
444 "invalidatable", Path,
445 [&](llvm::Expected<InputsAndAST> AST) {
446 ++Actions;
447 EXPECT_TRUE(bool(AST))
448 << "Should not invalidate based on an update with same content: "
449 << llvm::toString(AST.takeError());
450 },
451 TUScheduler::InvalidateOnUpdate);
452 updateWithDiags(S, Path, "a", WantDiagnostics::Yes, [&](std::vector<Diag>) {
453 ADD_FAILURE() << "Shouldn't build, identical to previous";
454 });
455 Start.notify();
456 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
457
458 EXPECT_EQ(1, Actions.load()) << "All actions should run";
459 }
460
461 TEST_F(TUSchedulerTests, ManyUpdates) {
462 const int FilesCount = 3;
463 const int UpdatesPerFile = 10;
464
465 std::mutex Mut;
466 int TotalASTReads = 0;
467 int TotalPreambleReads = 0;
468 int TotalUpdates = 0;
469 llvm::StringMap<int> LatestDiagVersion;
470
471 // Run TUScheduler and collect some stats.
472 {
473 auto Opts = optsForTest();
474 Opts.UpdateDebounce = DebouncePolicy::fixed(std::chrono::milliseconds(50));
475 TUScheduler S(CDB, Opts, captureDiags());
476
477 std::vector<std::string> Files;
478 for (int I = 0; I < FilesCount; ++I) {
479 std::string Name = "foo" + std::to_string(I) + ".cpp";
480 Files.push_back(testPath(Name));
481 this->FS.Files[Files.back()] = "";
482 }
483
484 StringRef Contents1 = R"cpp(int a;)cpp";
485 StringRef Contents2 = R"cpp(int main() { return 1; })cpp";
486 StringRef Contents3 = R"cpp(int a; int b; int sum() { return a + b; })cpp";
487
488 StringRef AllContents[] = {Contents1, Contents2, Contents3};
489 const int AllContentsSize = 3;
490
491 // Scheduler may run tasks asynchronously, but should propagate the
492 // context. We stash a nonce in the context, and verify it in the task.
493 static Key<int> NonceKey;
494 int Nonce = 0;
495
496 for (int FileI = 0; FileI < FilesCount; ++FileI) {
497 for (int UpdateI = 0; UpdateI < UpdatesPerFile; ++UpdateI) {
498 auto Contents = AllContents[(FileI + UpdateI) % AllContentsSize];
499
500 auto File = Files[FileI];
501 auto Inputs = getInputs(File, Contents.str());
502 {
503 WithContextValue WithNonce(NonceKey, ++Nonce);
504 Inputs.Version = std::to_string(UpdateI);
505 updateWithDiags(
506 S, File, Inputs, WantDiagnostics::Auto,
507 [File, Nonce, Version(Inputs.Version), &Mut, &TotalUpdates,
508 &LatestDiagVersion](std::vector<Diag>) {
509 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
510 EXPECT_EQ(File, boundPath());
511
512 std::lock_guard<std::mutex> Lock(Mut);
513 ++TotalUpdates;
514 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
515 // Make sure Diags are for a newer version.
516 auto It = LatestDiagVersion.try_emplace(File, -1);
517 const int PrevVersion = It.first->second;
518 int CurVersion;
519 ASSERT_TRUE(llvm::to_integer(Version, CurVersion, 10));
520 EXPECT_LT(PrevVersion, CurVersion);
521 It.first->getValue() = CurVersion;
522 });
523 }
524 {
525 WithContextValue WithNonce(NonceKey, ++Nonce);
526 S.runWithAST(
527 "CheckAST", File,
528 [File, Inputs, Nonce, &Mut,
529 &TotalASTReads](Expected<InputsAndAST> AST) {
530 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
531 EXPECT_EQ(File, boundPath());
532
533 ASSERT_TRUE((bool)AST);
534 EXPECT_EQ(AST->Inputs.Contents, Inputs.Contents);
535 EXPECT_EQ(AST->Inputs.Version, Inputs.Version);
536 EXPECT_EQ(AST->AST.version(), Inputs.Version);
537
538 std::lock_guard<std::mutex> Lock(Mut);
539 ++TotalASTReads;
540 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
541 });
542 }
543
544 {
545 WithContextValue WithNonce(NonceKey, ++Nonce);
546 S.runWithPreamble(
547 "CheckPreamble", File, TUScheduler::Stale,
548 [File, Inputs, Nonce, &Mut,
549 &TotalPreambleReads](Expected<InputsAndPreamble> Preamble) {
550 EXPECT_THAT(Context::current().get(NonceKey), Pointee(Nonce));
551 EXPECT_EQ(File, boundPath());
552
553 ASSERT_TRUE((bool)Preamble);
554 EXPECT_EQ(Preamble->Contents, Inputs.Contents);
555
556 std::lock_guard<std::mutex> Lock(Mut);
557 ++TotalPreambleReads;
558 EXPECT_EQ(File, *TUScheduler::getFileBeingProcessedInContext());
559 });
560 }
561 }
562 }
563 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
564 } // TUScheduler destructor waits for all operations to finish.
565
566 std::lock_guard<std::mutex> Lock(Mut);
567 // Updates might get coalesced in preamble thread and result in dropping
568 // diagnostics for intermediate snapshots.
569 EXPECT_GE(TotalUpdates, FilesCount);
570 EXPECT_LE(TotalUpdates, FilesCount * UpdatesPerFile);
571 // We should receive diags for last update.
572 for (const auto &Entry : LatestDiagVersion)
573 EXPECT_EQ(Entry.second, UpdatesPerFile - 1);
574 EXPECT_EQ(TotalASTReads, FilesCount * UpdatesPerFile);
575 EXPECT_EQ(TotalPreambleReads, FilesCount * UpdatesPerFile);
576 }
577
578 TEST_F(TUSchedulerTests, EvictedAST) {
579 std::atomic<int> BuiltASTCounter(0);
580 auto Opts = optsForTest();
581 Opts.AsyncThreadsCount = 1;
582 Opts.RetentionPolicy.MaxRetainedASTs = 2;
583 trace::TestTracer Tracer;
584 TUScheduler S(CDB, Opts);
585
586 llvm::StringLiteral SourceContents = R"cpp(
587 int* a;
588 double* b = a;
589 )cpp";
590 llvm::StringLiteral OtherSourceContents = R"cpp(
591 int* a;
592 double* b = a + 0;
593 )cpp";
594
595 auto Foo = testPath("foo.cpp");
596 auto Bar = testPath("bar.cpp");
597 auto Baz = testPath("baz.cpp");
598
599 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
600 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
601 // Build one file in advance. We will not access it later, so it will be the
602 // one that the cache will evict.
603 updateWithCallback(S, Foo, SourceContents, WantDiagnostics::Yes,
604 [&BuiltASTCounter]() { ++BuiltASTCounter; });
605 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
606 ASSERT_EQ(BuiltASTCounter.load(), 1);
607 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
608 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
609
610 // Build two more files. Since we can retain only 2 ASTs, these should be
611 // the ones we see in the cache later.
612 updateWithCallback(S, Bar, SourceContents, WantDiagnostics::Yes,
613 [&BuiltASTCounter]() { ++BuiltASTCounter; });
614 updateWithCallback(S, Baz, SourceContents, WantDiagnostics::Yes,
615 [&BuiltASTCounter]() { ++BuiltASTCounter; });
616 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
617 ASSERT_EQ(BuiltASTCounter.load(), 3);
618 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
619 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(2));
620
621 // Check only the last two ASTs are retained.
622 ASSERT_THAT(S.getFilesWithCachedAST(), UnorderedElementsAre(Bar, Baz));
623
624 // Access the old file again.
625 updateWithCallback(S, Foo, OtherSourceContents, WantDiagnostics::Yes,
626 [&BuiltASTCounter]() { ++BuiltASTCounter; });
627 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
628 ASSERT_EQ(BuiltASTCounter.load(), 4);
629 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
630 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(1));
631
632 // Check the AST for foo.cpp is retained now and one of the others got
633 // evicted.
634 EXPECT_THAT(S.getFilesWithCachedAST(),
635 UnorderedElementsAre(Foo, AnyOf(Bar, Baz)));
636 }
637
638 // We send "empty" changes to TUScheduler when we think some external event
639 // *might* have invalidated current state (e.g. a header was edited).
640 // Verify that this doesn't evict our cache entries.
641 TEST_F(TUSchedulerTests, NoopChangesDontThrashCache) {
642 auto Opts = optsForTest();
643 Opts.RetentionPolicy.MaxRetainedASTs = 1;
644 TUScheduler S(CDB, Opts);
645
646 auto Foo = testPath("foo.cpp");
647 auto FooInputs = getInputs(Foo, "int x=1;");
648 auto Bar = testPath("bar.cpp");
649 auto BarInputs = getInputs(Bar, "int x=2;");
650
651 // After opening Foo then Bar, AST cache contains Bar.
652 S.update(Foo, FooInputs, WantDiagnostics::Auto);
653 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
654 S.update(Bar, BarInputs, WantDiagnostics::Auto);
655 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
656 ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
657
658 // Any number of no-op updates to Foo don't dislodge Bar from the cache.
659 S.update(Foo, FooInputs, WantDiagnostics::Auto);
660 S.update(Foo, FooInputs, WantDiagnostics::Auto);
661 S.update(Foo, FooInputs, WantDiagnostics::Auto);
662 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
663 ASSERT_THAT(S.getFilesWithCachedAST(), ElementsAre(Bar));
664 // In fact each file has been built only once.
665 ASSERT_EQ(S.fileStats().lookup(Foo).ASTBuilds, 1u);
666 ASSERT_EQ(S.fileStats().lookup(Bar).ASTBuilds, 1u);
667 }
668
669 TEST_F(TUSchedulerTests, EmptyPreamble) {
670 TUScheduler S(CDB, optsForTest());
671
672 auto Foo = testPath("foo.cpp");
673 auto Header = testPath("foo.h");
674
675 FS.Files[Header] = "void foo()";
676 FS.Timestamps[Header] = time_t(0);
677 auto *WithPreamble = R"cpp(
678 #include "foo.h"
679 int main() {}
680 )cpp";
681 auto *WithEmptyPreamble = R"cpp(int main() {})cpp";
682 S.update(Foo, getInputs(Foo, WithPreamble), WantDiagnostics::Auto);
683 S.runWithPreamble(
684 "getNonEmptyPreamble", Foo, TUScheduler::Stale,
685 [&](Expected<InputsAndPreamble> Preamble) {
686 // We expect to get a non-empty preamble.
687 EXPECT_GT(
688 cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
689 0u);
690 });
691 // Wait while the preamble is being built.
692 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
693
694 // Update the file which results in an empty preamble.
695 S.update(Foo, getInputs(Foo, WithEmptyPreamble), WantDiagnostics::Auto);
696 // Wait while the preamble is being built.
697 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
698 S.runWithPreamble(
699 "getEmptyPreamble", Foo, TUScheduler::Stale,
700 [&](Expected<InputsAndPreamble> Preamble) {
701 // We expect to get an empty preamble.
702 EXPECT_EQ(
703 cantFail(std::move(Preamble)).Preamble->Preamble.getBounds().Size,
704 0u);
705 });
706 }
707
708 TEST_F(TUSchedulerTests, ASTSignalsSmokeTests) {
709 TUScheduler S(CDB, optsForTest());
710 auto Foo = testPath("foo.cpp");
711 auto Header = testPath("foo.h");
712
713 FS.Files[Header] = "namespace tar { int foo(); }";
714 const char *Contents = R"cpp(
715 #include "foo.h"
716 namespace ns {
717 int func() {
718 return tar::foo());
719 }
720 } // namespace ns
721 )cpp";
722 // Update the file which results in an empty preamble.
723 S.update(Foo, getInputs(Foo, Contents), WantDiagnostics::Yes);
724 // Wait while the preamble is being built.
725 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
726 Notification TaskRun;
727 S.runWithPreamble(
728 "ASTSignals", Foo, TUScheduler::Stale,
729 [&](Expected<InputsAndPreamble> IP) {
730 ASSERT_FALSE(!IP);
731 std::vector<std::pair<StringRef, int>> NS;
732 for (const auto &P : IP->Signals->RelatedNamespaces)
733 NS.emplace_back(P.getKey(), P.getValue());
734 EXPECT_THAT(NS,
735 UnorderedElementsAre(Pair("ns::", 1), Pair("tar::", 1)));
736
737 std::vector<std::pair<SymbolID, int>> Sym;
738 for (const auto &P : IP->Signals->ReferencedSymbols)
739 Sym.emplace_back(P.getFirst(), P.getSecond());
740 EXPECT_THAT(Sym, UnorderedElementsAre(Pair(ns("tar").ID, 1),
741 Pair(ns("ns").ID, 1),
742 Pair(func("tar::foo").ID, 1),
743 Pair(func("ns::func").ID, 1)));
744 TaskRun.notify();
745 });
746 TaskRun.wait();
747 }
748
749 TEST_F(TUSchedulerTests, RunWaitsForPreamble) {
750 // Testing strategy: we update the file and schedule a few preamble reads at
751 // the same time. All reads should get the same non-null preamble.
752 TUScheduler S(CDB, optsForTest());
753 auto Foo = testPath("foo.cpp");
754 auto *NonEmptyPreamble = R"cpp(
755 #define FOO 1
756 #define BAR 2
757
758 int main() {}
759 )cpp";
760 constexpr int ReadsToSchedule = 10;
761 std::mutex PreamblesMut;
762 std::vector<const void *> Preambles(ReadsToSchedule, nullptr);
763 S.update(Foo, getInputs(Foo, NonEmptyPreamble), WantDiagnostics::Auto);
764 for (int I = 0; I < ReadsToSchedule; ++I) {
765 S.runWithPreamble(
766 "test", Foo, TUScheduler::Stale,
767 [I, &PreamblesMut, &Preambles](Expected<InputsAndPreamble> IP) {
768 std::lock_guard<std::mutex> Lock(PreamblesMut);
769 Preambles[I] = cantFail(std::move(IP)).Preamble;
770 });
771 }
772 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
773 // Check all actions got the same non-null preamble.
774 std::lock_guard<std::mutex> Lock(PreamblesMut);
775 ASSERT_NE(Preambles[0], nullptr);
776 ASSERT_THAT(Preambles, Each(Preambles[0]));
777 }
778
779 TEST_F(TUSchedulerTests, NoopOnEmptyChanges) {
780 TUScheduler S(CDB, optsForTest(), captureDiags());
781
782 auto Source = testPath("foo.cpp");
783 auto Header = testPath("foo.h");
784
785 FS.Files[Header] = "int a;";
786 FS.Timestamps[Header] = time_t(0);
787
788 std::string SourceContents = R"cpp(
789 #include "foo.h"
790 int b = a;
791 )cpp";
792
793 // Return value indicates if the updated callback was received.
794 auto DoUpdate = [&](std::string Contents) -> bool {
795 std::atomic<bool> Updated(false);
796 Updated = false;
797 updateWithDiags(S, Source, Contents, WantDiagnostics::Yes,
798 [&Updated](std::vector<Diag>) { Updated = true; });
799 bool UpdateFinished = S.blockUntilIdle(timeoutSeconds(10));
800 if (!UpdateFinished)
801 ADD_FAILURE() << "Updated has not finished in one second. Threading bug?";
802 return Updated;
803 };
804
805 // Test that subsequent updates with the same inputs do not cause rebuilds.
806 ASSERT_TRUE(DoUpdate(SourceContents));
807 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
808 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);
809 ASSERT_FALSE(DoUpdate(SourceContents));
810 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 1u);
811 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 1u);
812
813 // Update to a header should cause a rebuild, though.
814 FS.Timestamps[Header] = time_t(1);
815 ASSERT_TRUE(DoUpdate(SourceContents));
816 ASSERT_FALSE(DoUpdate(SourceContents));
817 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 2u);
818 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);
819
820 // Update to the contents should cause a rebuild.
821 SourceContents += "\nint c = b;";
822 ASSERT_TRUE(DoUpdate(SourceContents));
823 ASSERT_FALSE(DoUpdate(SourceContents));
824 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 3u);
825 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 2u);
826
827 // Update to the compile commands should also cause a rebuild.
828 CDB.ExtraClangFlags.push_back("-DSOMETHING");
829 ASSERT_TRUE(DoUpdate(SourceContents));
830 ASSERT_FALSE(DoUpdate(SourceContents));
831 ASSERT_EQ(S.fileStats().lookup(Source).ASTBuilds, 4u);
832 ASSERT_EQ(S.fileStats().lookup(Source).PreambleBuilds, 3u);
833 }
834
835 // We rebuild if a completely missing header exists, but not if one is added
836 // on a higher-priority include path entry (for performance).
837 // (Previously we wouldn't automatically rebuild when files were added).
838 TEST_F(TUSchedulerTests, MissingHeader) {
839 CDB.ExtraClangFlags.push_back("-I" + testPath("a"));
840 CDB.ExtraClangFlags.push_back("-I" + testPath("b"));
841 // Force both directories to exist so they don't get pruned.
842 FS.Files.try_emplace("a/__unused__");
843 FS.Files.try_emplace("b/__unused__");
844 TUScheduler S(CDB, optsForTest(), captureDiags());
845
846 auto Source = testPath("foo.cpp");
847 auto HeaderA = testPath("a/foo.h");
848 auto HeaderB = testPath("b/foo.h");
849
850 auto *SourceContents = R"cpp(
851 #include "foo.h"
852 int c = b;
853 )cpp";
854
855 ParseInputs Inputs = getInputs(Source, SourceContents);
856 std::atomic<size_t> DiagCount(0);
857
858 // Update the source contents, which should trigger an initial build with
859 // the header file missing.
860 updateWithDiags(
861 S, Source, Inputs, WantDiagnostics::Yes,
862 [&DiagCount](std::vector<Diag> Diags) {
863 ++DiagCount;
864 EXPECT_THAT(Diags,
865 ElementsAre(Field(&Diag::Message, "'foo.h' file not found"),
866 Field(&Diag::Message,
867 "use of undeclared identifier 'b'")));
868 });
869 S.blockUntilIdle(timeoutSeconds(10));
870
871 FS.Files[HeaderB] = "int b;";
872 FS.Timestamps[HeaderB] = time_t(1);
873
874 // The addition of the missing header file triggers a rebuild, no errors.
875 updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
876 [&DiagCount](std::vector<Diag> Diags) {
877 ++DiagCount;
878 EXPECT_THAT(Diags, IsEmpty());
879 });
880
881 // Ensure previous assertions are done before we touch the FS again.
882 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
883 // Add the high-priority header file, which should reintroduce the error.
884 FS.Files[HeaderA] = "int a;";
885 FS.Timestamps[HeaderA] = time_t(1);
886
887 // This isn't detected: we don't stat a/foo.h to validate the preamble.
888 updateWithDiags(S, Source, Inputs, WantDiagnostics::Yes,
889 [&DiagCount](std::vector<Diag> Diags) {
890 ++DiagCount;
891 ADD_FAILURE()
892 << "Didn't expect new diagnostics when adding a/foo.h";
893 });
894
895 // Forcing the reload should cause a rebuild.
896 Inputs.ForceRebuild = true;
897 updateWithDiags(
898 S, Source, Inputs, WantDiagnostics::Yes,
899 [&DiagCount](std::vector<Diag> Diags) {
900 ++DiagCount;
901 ElementsAre(Field(&Diag::Message, "use of undeclared identifier 'b'"));
902 });
903
904 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
905 EXPECT_EQ(DiagCount, 3U);
906 }
907
908 TEST_F(TUSchedulerTests, NoChangeDiags) {
909 trace::TestTracer Tracer;
910 TUScheduler S(CDB, optsForTest(), captureDiags());
911
912 auto FooCpp = testPath("foo.cpp");
913 const auto *Contents = "int a; int b;";
914
915 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
916 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(0));
917 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(0));
918 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
919 updateWithDiags(
920 S, FooCpp, Contents, WantDiagnostics::No,
921 [](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
922 S.runWithAST("touchAST", FooCpp, [](Expected<InputsAndAST> IA) {
923 // Make sure the AST was actually built.
924 cantFail(std::move(IA));
925 });
926 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
927 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "hit"), SizeIs(0));
928 EXPECT_THAT(Tracer.takeMetric("ast_access_read", "miss"), SizeIs(1));
929
930 // Even though the inputs didn't change and AST can be reused, we need to
931 // report the diagnostics, as they were not reported previously.
932 std::atomic<bool> SeenDiags(false);
933 updateWithDiags(S, FooCpp, Contents, WantDiagnostics::Auto,
934 [&](std::vector<Diag>) { SeenDiags = true; });
935 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
936 ASSERT_TRUE(SeenDiags);
937 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "hit"), SizeIs(1));
938 EXPECT_THAT(Tracer.takeMetric("ast_access_diag", "miss"), SizeIs(0));
939
940 // Subsequent request does not get any diagnostics callback because the same
941 // diags have previously been reported and the inputs didn't change.
942 updateWithDiags(
943 S, FooCpp, Contents, WantDiagnostics::Auto,
944 [&](std::vector<Diag>) { ADD_FAILURE() << "Should not be called."; });
945 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
946 }
947
948 TEST_F(TUSchedulerTests, Run) {
949 for (bool Sync : {false, true}) {
950 auto Opts = optsForTest();
951 if (Sync)
952 Opts.AsyncThreadsCount = 0;
953 TUScheduler S(CDB, Opts);
954 std::atomic<int> Counter(0);
955 S.run("add 1", /*Path=*/"", [&] { ++Counter; });
956 S.run("add 2", /*Path=*/"", [&] { Counter += 2; });
957 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
958 EXPECT_EQ(Counter.load(), 3);
959
960 Notification TaskRun;
961 Key<int> TestKey;
962 WithContextValue CtxWithKey(TestKey, 10);
963 const char *Path = "somepath";
964 S.run("props context", Path, [&] {
965 EXPECT_EQ(Context::current().getExisting(TestKey), 10);
966 EXPECT_EQ(Path, boundPath());
967 TaskRun.notify();
968 });
969 TaskRun.wait();
970 }
971 }
972
973 TEST_F(TUSchedulerTests, TUStatus) {
974 class CaptureTUStatus : public ClangdServer::Callbacks {
975 public:
976 void onFileUpdated(PathRef File, const TUStatus &Status) override {
977 auto ASTAction = Status.ASTActivity.K;
978 auto PreambleAction = Status.PreambleActivity;
979 std::lock_guard<std::mutex> Lock(Mutex);
980 // Only push the action if it has changed. Since TUStatus can be published
981 // from either Preamble or AST thread and when one changes the other stays
982 // the same.
983 // Note that this can result in missing some updates when something other
984 // than action kind changes, e.g. when AST is built/reused the action kind
985 // stays as Building.
986 if (ASTActions.empty() || ASTActions.back() != ASTAction)
987 ASTActions.push_back(ASTAction);
988 if (PreambleActions.empty() || PreambleActions.back() != PreambleAction)
989 PreambleActions.push_back(PreambleAction);
990 }
991
992 std::vector<PreambleAction> preambleStatuses() {
993 std::lock_guard<std::mutex> Lock(Mutex);
994 return PreambleActions;
995 }
996
997 std::vector<ASTAction::Kind> astStatuses() {
998 std::lock_guard<std::mutex> Lock(Mutex);
999 return ASTActions;
1000 }
1001
1002 private:
1003 std::mutex Mutex;
1004 std::vector<ASTAction::Kind> ASTActions;
1005 std::vector<PreambleAction> PreambleActions;
1006 } CaptureTUStatus;
1007 MockFS FS;
1008 MockCompilationDatabase CDB;
1009 ClangdServer Server(CDB, FS, ClangdServer::optsForTest(), &CaptureTUStatus);
1010 Annotations Code("int m^ain () {}");
1011
1012 // We schedule the following tasks in the queue:
1013 // [Update] [GoToDefinition]
1014 Server.addDocument(testPath("foo.cpp"), Code.code(), "1",
1015 WantDiagnostics::Auto);
1016 ASSERT_TRUE(Server.blockUntilIdleForTest());
1017 Server.locateSymbolAt(testPath("foo.cpp"), Code.point(),
1018 [](Expected<std::vector<LocatedSymbol>> Result) {
1019 ASSERT_TRUE((bool)Result);
1020 });
1021 ASSERT_TRUE(Server.blockUntilIdleForTest());
1022
1023 EXPECT_THAT(CaptureTUStatus.preambleStatuses(),
1024 ElementsAre(
1025 // PreambleThread starts idle, as the update is first handled
1026 // by ASTWorker.
1027 PreambleAction::Idle,
1028 // Then it starts building first preamble and releases that to
1029 // ASTWorker.
1030 PreambleAction::Building,
1031 // Then goes idle and stays that way as we don't receive any
1032 // more update requests.
1033 PreambleAction::Idle));
1034 EXPECT_THAT(CaptureTUStatus.astStatuses(),
1035 ElementsAre(
1036 // Starts handling the update action and blocks until the
1037 // first preamble is built.
1038 ASTAction::RunningAction,
1039 // Afterwards it builds an AST for that preamble to publish
1040 // diagnostics.
1041 ASTAction::Building,
1042 // Then goes idle.
1043 ASTAction::Idle,
1044 // Afterwards we start executing go-to-def.
1045 ASTAction::RunningAction,
1046 // Then go idle.
1047 ASTAction::Idle));
1048 }
1049
1050 TEST_F(TUSchedulerTests, CommandLineErrors) {
1051 // We should see errors from command-line parsing inside the main file.
1052 CDB.ExtraClangFlags = {"-fsome-unknown-flag"};
1053
1054 // (!) 'Ready' must live longer than TUScheduler.
1055 Notification Ready;
1056
1057 TUScheduler S(CDB, optsForTest(), captureDiags());
1058 std::vector<Diag> Diagnostics;
1059 updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
1060 WantDiagnostics::Yes, [&](std::vector<Diag> D) {
1061 Diagnostics = std::move(D);
1062 Ready.notify();
1063 });
1064 Ready.wait();
1065
1066 EXPECT_THAT(
1067 Diagnostics,
1068 ElementsAre(AllOf(
1069 Field(&Diag::ID, Eq(diag::err_drv_unknown_argument)),
1070 Field(&Diag::Name, Eq("drv_unknown_argument")),
1071 Field(&Diag::Message, "unknown argument: '-fsome-unknown-flag'"))));
1072 }
1073
1074 TEST_F(TUSchedulerTests, CommandLineWarnings) {
1075 // We should not see warnings from command-line parsing.
1076 CDB.ExtraClangFlags = {"-Wsome-unknown-warning"};
1077
1078 // (!) 'Ready' must live longer than TUScheduler.
1079 Notification Ready;
1080
1081 TUScheduler S(CDB, optsForTest(), captureDiags());
1082 std::vector<Diag> Diagnostics;
1083 updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
1084 WantDiagnostics::Yes, [&](std::vector<Diag> D) {
1085 Diagnostics = std::move(D);
1086 Ready.notify();
1087 });
1088 Ready.wait();
1089
1090 EXPECT_THAT(Diagnostics, IsEmpty());
1091 }
1092
1093 TEST(DebouncePolicy, Compute) {
1094 namespace c = std::chrono;
1095 DebouncePolicy::clock::duration History[] = {
1096 c::seconds(0),
1097 c::seconds(5),
1098 c::seconds(10),
1099 c::seconds(20),
1100 };
1101 DebouncePolicy Policy;
1102 Policy.Min = c::seconds(3);
1103 Policy.Max = c::seconds(25);
1104 // Call Policy.compute(History) and return seconds as a float.
1105 auto Compute = [&](llvm::ArrayRef<DebouncePolicy::clock::duration> History) {
1106 return c::duration_cast<c::duration<float, c::seconds::period>>(
1107 Policy.compute(History))
1108 .count();
1109 };
1110 EXPECT_NEAR(10, Compute(History), 0.01) << "(upper) median = 10";
1111 Policy.RebuildRatio = 1.5;
1112 EXPECT_NEAR(15, Compute(History), 0.01) << "median = 10, ratio = 1.5";
1113 Policy.RebuildRatio = 3;
1114 EXPECT_NEAR(25, Compute(History), 0.01) << "constrained by max";
1115 Policy.RebuildRatio = 0;
1116 EXPECT_NEAR(3, Compute(History), 0.01) << "constrained by min";
1117 EXPECT_NEAR(25, Compute({}), 0.01) << "no history -> max";
1118 }
1119
1120 TEST_F(TUSchedulerTests, AsyncPreambleThread) {
1121 // Blocks preamble thread while building preamble with \p BlockVersion until
1122 // \p N is notified.
1123 class BlockPreambleThread : public ParsingCallbacks {
1124 public:
1125 BlockPreambleThread(llvm::StringRef BlockVersion, Notification &N)
1126 : BlockVersion(BlockVersion), N(N) {}
1127 void onPreambleAST(PathRef Path, llvm::StringRef Version,
1128 const CompilerInvocation &, ASTContext &Ctx,
1129 Preprocessor &, const CanonicalIncludes &) override {
1130 if (Version == BlockVersion)
1131 N.wait();
1132 }
1133
1134 private:
1135 llvm::StringRef BlockVersion;
1136 Notification &N;
1137 };
1138
1139 static constexpr llvm::StringLiteral InputsV0 = "v0";
1140 static constexpr llvm::StringLiteral InputsV1 = "v1";
1141 Notification Ready;
1142 TUScheduler S(CDB, optsForTest(),
1143 std::make_unique<BlockPreambleThread>(InputsV1, Ready));
1144
1145 Path File = testPath("foo.cpp");
1146 auto PI = getInputs(File, "");
1147 PI.Version = InputsV0.str();
1148 S.update(File, PI, WantDiagnostics::Auto);
1149 S.blockUntilIdle(timeoutSeconds(10));
1150
1151 // Block preamble builds.
1152 PI.Version = InputsV1.str();
1153 // Issue second update which will block preamble thread.
1154 S.update(File, PI, WantDiagnostics::Auto);
1155
1156 Notification RunASTAction;
1157 // Issue an AST read, which shouldn't be blocked and see latest version of the
1158 // file.
1159 S.runWithAST("test", File, [&](Expected<InputsAndAST> AST) {
1160 ASSERT_TRUE(bool(AST));
1161 // Make sure preamble is built with stale inputs, but AST was built using
1162 // new ones.
1163 EXPECT_THAT(AST->AST.preambleVersion(), InputsV0);
1164 EXPECT_THAT(AST->Inputs.Version, InputsV1.str());
1165 RunASTAction.notify();
1166 });
1167 RunASTAction.wait();
1168 Ready.notify();
1169 }
1170
1171 TEST_F(TUSchedulerTests, OnlyPublishWhenPreambleIsBuilt) {
1172 struct PreamblePublishCounter : public ParsingCallbacks {
1173 PreamblePublishCounter(int &PreamblePublishCount)
1174 : PreamblePublishCount(PreamblePublishCount) {}
1175 void onPreamblePublished(PathRef File) override { ++PreamblePublishCount; }
1176 int &PreamblePublishCount;
1177 };
1178
1179 int PreamblePublishCount = 0;
1180 TUScheduler S(CDB, optsForTest(),
1181 std::make_unique<PreamblePublishCounter>(PreamblePublishCount));
1182
1183 Path File = testPath("foo.cpp");
1184 S.update(File, getInputs(File, ""), WantDiagnostics::Auto);
1185 S.blockUntilIdle(timeoutSeconds(10));
1186 EXPECT_EQ(PreamblePublishCount, 1);
1187 // Same contents, no publish.
1188 S.update(File, getInputs(File, ""), WantDiagnostics::Auto);
1189 S.blockUntilIdle(timeoutSeconds(10));
1190 EXPECT_EQ(PreamblePublishCount, 1);
1191 // New contents, should publish.
1192 S.update(File, getInputs(File, "#define FOO"), WantDiagnostics::Auto);
1193 S.blockUntilIdle(timeoutSeconds(10));
1194 EXPECT_EQ(PreamblePublishCount, 2);
1195 }
1196
1197 // If a header file is missing from the CDB (or inferred using heuristics), and
1198 // it's included by another open file, then we parse it using that files flags.
1199 TEST_F(TUSchedulerTests, IncluderCache) {
1200 static std::string Main = testPath("main.cpp"), Main2 = testPath("main2.cpp"),
1201 Main3 = testPath("main3.cpp"),
1202 NoCmd = testPath("no_cmd.h"),
1203 Unreliable = testPath("unreliable.h"),
1204 OK = testPath("ok.h"),
1205 NotIncluded = testPath("not_included.h");
1206 struct NoHeadersCDB : public GlobalCompilationDatabase {
1207 std::optional<tooling::CompileCommand>
1208 getCompileCommand(PathRef File) const override {
1209 if (File == NoCmd || File == NotIncluded || FailAll)
1210 return std::nullopt;
1211 auto Basic = getFallbackCommand(File);
1212 Basic.Heuristic.clear();
1213 if (File == Unreliable) {
1214 Basic.Heuristic = "not reliable";
1215 } else if (File == Main) {
1216 Basic.CommandLine.push_back("-DMAIN");
1217 } else if (File == Main2) {
1218 Basic.CommandLine.push_back("-DMAIN2");
1219 } else if (File == Main3) {
1220 Basic.CommandLine.push_back("-DMAIN3");
1221 }
1222 return Basic;
1223 }
1224
1225 std::atomic<bool> FailAll{false};
1226 } CDB;
1227 TUScheduler S(CDB, optsForTest());
1228 auto GetFlags = [&](PathRef Header) {
1229 S.update(Header, getInputs(Header, ";"), WantDiagnostics::Yes);
1230 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1231 Notification CmdDone;
1232 tooling::CompileCommand Cmd;
1233 S.runWithPreamble("GetFlags", Header, TUScheduler::StaleOrAbsent,
1234 [&](llvm::Expected<InputsAndPreamble> Inputs) {
1235 ASSERT_FALSE(!Inputs) << Inputs.takeError();
1236 Cmd = std::move(Inputs->Command);
1237 CmdDone.notify();
1238 });
1239 CmdDone.wait();
1240 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1241 return Cmd.CommandLine;
1242 };
1243
1244 for (const auto &Path : {NoCmd, Unreliable, OK, NotIncluded})
1245 FS.Files[Path] = ";";
1246
1247 // Initially these files have normal commands from the CDB.
1248 EXPECT_THAT(GetFlags(Main), Contains("-DMAIN")) << "sanity check";
1249 EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN"))) << "no includes yet";
1250
1251 // Now make Main include the others, and some should pick up its flags.
1252 const char *AllIncludes = R"cpp(
1253 #include "no_cmd.h"
1254 #include "ok.h"
1255 #include "unreliable.h"
1256 )cpp";
1257 S.update(Main, getInputs(Main, AllIncludes), WantDiagnostics::Yes);
1258 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1259 EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN"))
1260 << "Included from main file, has no own command";
1261 EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
1262 << "Included from main file, own command is heuristic";
1263 EXPECT_THAT(GetFlags(OK), Not(Contains("-DMAIN")))
1264 << "Included from main file, but own command is used";
1265 EXPECT_THAT(GetFlags(NotIncluded), Not(Contains("-DMAIN")))
1266 << "Not included from main file";
1267
1268 // Open another file - it won't overwrite the associations with Main.
1269 std::string SomeIncludes = R"cpp(
1270 #include "no_cmd.h"
1271 #include "not_included.h"
1272 )cpp";
1273 S.update(Main2, getInputs(Main2, SomeIncludes), WantDiagnostics::Yes);
1274 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1275 EXPECT_THAT(GetFlags(NoCmd),
1276 AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
1277 << "mainfile association is stable";
1278 EXPECT_THAT(GetFlags(NotIncluded),
1279 AllOf(Contains("-DMAIN2"), Not(Contains("-DMAIN"))))
1280 << "new headers are associated with new mainfile";
1281
1282 // Remove includes from main - this marks the associations as invalid but
1283 // doesn't actually remove them until another preamble claims them.
1284 S.update(Main, getInputs(Main, ""), WantDiagnostics::Yes);
1285 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1286 EXPECT_THAT(GetFlags(NoCmd),
1287 AllOf(Contains("-DMAIN"), Not(Contains("-DMAIN2"))))
1288 << "mainfile association not updated yet!";
1289
1290 // Open yet another file - this time it claims the associations.
1291 S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1292 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1293 EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
1294 << "association invalidated and then claimed by main3";
1295 EXPECT_THAT(GetFlags(Unreliable), Contains("-DMAIN"))
1296 << "association invalidated but not reclaimed";
1297 EXPECT_THAT(GetFlags(NotIncluded), Contains("-DMAIN2"))
1298 << "association still valid";
1299
1300 // Delete the file from CDB, it should invalidate the associations.
1301 CDB.FailAll = true;
1302 EXPECT_THAT(GetFlags(NoCmd), Not(Contains("-DMAIN3")))
1303 << "association should've been invalidated.";
1304 // Also run update for Main3 to invalidate the preeamble to make sure next
1305 // update populates include cache associations.
1306 S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1307 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1308 // Re-add the file and make sure nothing crashes.
1309 CDB.FailAll = false;
1310 S.update(Main3, getInputs(Main3, SomeIncludes), WantDiagnostics::Yes);
1311 EXPECT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1312 EXPECT_THAT(GetFlags(NoCmd), Contains("-DMAIN3"))
1313 << "association invalidated and then claimed by main3";
1314 }
1315
1316 TEST_F(TUSchedulerTests, PreservesLastActiveFile) {
1317 for (bool Sync : {false, true}) {
1318 auto Opts = optsForTest();
1319 if (Sync)
1320 Opts.AsyncThreadsCount = 0;
1321 TUScheduler S(CDB, Opts);
1322
1323 auto CheckNoFileActionsSeesLastActiveFile =
1324 [&](llvm::StringRef LastActiveFile) {
1325 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1326 std::atomic<int> Counter(0);
1327 // We only check for run and runQuick as runWithAST and
1328 // runWithPreamble is always bound to a file.
1329 S.run("run-UsesLastActiveFile", /*Path=*/"", [&] {
1330 ++Counter;
1331 EXPECT_EQ(LastActiveFile, boundPath());
1332 });
1333 S.runQuick("runQuick-UsesLastActiveFile", /*Path=*/"", [&] {
1334 ++Counter;
1335 EXPECT_EQ(LastActiveFile, boundPath());
1336 });
1337 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1338 EXPECT_EQ(2, Counter.load());
1339 };
1340
1341 // Check that we see no file initially
1342 CheckNoFileActionsSeesLastActiveFile("");
1343
1344 // Now check that every action scheduled with a particular file changes the
1345 // LastActiveFile.
1346 auto Path = testPath("run.cc");
1347 S.run(Path, Path, [] {});
1348 CheckNoFileActionsSeesLastActiveFile(Path);
1349
1350 Path = testPath("runQuick.cc");
1351 S.runQuick(Path, Path, [] {});
1352 CheckNoFileActionsSeesLastActiveFile(Path);
1353
1354 Path = testPath("runWithAST.cc");
1355 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1356 S.runWithAST(Path, Path, [](llvm::Expected<InputsAndAST> Inp) {
1357 EXPECT_TRUE(bool(Inp));
1358 });
1359 CheckNoFileActionsSeesLastActiveFile(Path);
1360
1361 Path = testPath("runWithPreamble.cc");
1362 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1363 S.runWithPreamble(
1364 Path, Path, TUScheduler::Stale,
1365 [](llvm::Expected<InputsAndPreamble> Inp) { EXPECT_TRUE(bool(Inp)); });
1366 CheckNoFileActionsSeesLastActiveFile(Path);
1367
1368 Path = testPath("update.cc");
1369 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1370 CheckNoFileActionsSeesLastActiveFile(Path);
1371
1372 // An update with the same contents should not change LastActiveFile.
1373 auto LastActive = Path;
1374 Path = testPath("runWithAST.cc");
1375 S.update(Path, getInputs(Path, ""), WantDiagnostics::No);
1376 CheckNoFileActionsSeesLastActiveFile(LastActive);
1377 }
1378 }
1379
1380 TEST_F(TUSchedulerTests, PreambleThrottle) {
1381 const int NumRequests = 4;
1382 // Silly throttler that waits for 4 requests, and services them in reverse.
1383 // Doesn't honor cancellation but records it.
1384 struct : public PreambleThrottler {
1385 std::mutex Mu;
1386 std::vector<std::string> Acquires;
1387 std::vector<RequestID> Releases;
1388 llvm::DenseMap<RequestID, Callback> Callbacks;
1389 // If set, the notification is signalled after acquiring the specified ID.
1390 std::optional<std::pair<RequestID, Notification *>> Notify;
1391
1392 RequestID acquire(llvm::StringRef Filename, Callback CB) override {
1393 RequestID ID;
1394 Callback Invoke;
1395 {
1396 std::lock_guard<std::mutex> Lock(Mu);
1397 ID = Acquires.size();
1398 Acquires.emplace_back(Filename);
1399 // If we're full, satisfy this request immediately.
1400 if (Acquires.size() == NumRequests) {
1401 Invoke = std::move(CB);
1402 } else {
1403 Callbacks.try_emplace(ID, std::move(CB));
1404 }
1405 }
1406 if (Invoke)
1407 Invoke();
1408 {
1409 std::lock_guard<std::mutex> Lock(Mu);
1410 if (Notify && ID == Notify->first) {
1411 Notify->second->notify();
1412 Notify.reset();
1413 }
1414 }
1415 return ID;
1416 }
1417
1418 void release(RequestID ID) override {
1419 Callback SatisfyNext;
1420 {
1421 std::lock_guard<std::mutex> Lock(Mu);
1422 Releases.push_back(ID);
1423 if (ID > 0 && Acquires.size() == NumRequests)
1424 SatisfyNext = std::move(Callbacks[ID - 1]);
1425 }
1426 if (SatisfyNext)
1427 SatisfyNext();
1428 }
1429
1430 void reset() {
1431 Acquires.clear();
1432 Releases.clear();
1433 Callbacks.clear();
1434 }
1435 } Throttler;
1436
1437 struct CaptureBuiltFilenames : public ParsingCallbacks {
1438 std::vector<std::string> &Filenames;
1439 CaptureBuiltFilenames(std::vector<std::string> &Filenames)
1440 : Filenames(Filenames) {}
1441 void onPreambleAST(PathRef Path, llvm::StringRef Version,
1442 const CompilerInvocation &CI, ASTContext &Ctx,
1443 Preprocessor &PP, const CanonicalIncludes &) override {
1444 // Deliberately no synchronization.
1445 // The PreambleThrottler should serialize these calls, if not then tsan
1446 // will find a bug here.
1447 Filenames.emplace_back(Path);
1448 }
1449 };
1450
1451 auto Opts = optsForTest();
1452 Opts.AsyncThreadsCount = 2 * NumRequests; // throttler is the bottleneck
1453 Opts.PreambleThrottler = &Throttler;
1454
1455 std::vector<std::string> Filenames;
1456
1457 {
1458 std::vector<std::string> BuiltFilenames;
1459 TUScheduler S(CDB, Opts,
1460 std::make_unique<CaptureBuiltFilenames>(BuiltFilenames));
1461 for (unsigned I = 0; I < NumRequests; ++I) {
1462 auto Path = testPath(std::to_string(I) + ".cc");
1463 Filenames.push_back(Path);
1464 S.update(Path, getInputs(Path, ""), WantDiagnostics::Yes);
1465 }
1466 ASSERT_TRUE(S.blockUntilIdle(timeoutSeconds(10)));
1467
1468 // The throttler saw all files, and we built them.
1469 EXPECT_THAT(Throttler.Acquires,
1470 testing::UnorderedElementsAreArray(Filenames));
1471 EXPECT_THAT(BuiltFilenames,
1472 testing::UnorderedElementsAreArray(Filenames));
1473 // We built the files in reverse order that the throttler saw them.
1474 EXPECT_THAT(BuiltFilenames,
1475 testing::ElementsAreArray(Throttler.Acquires.rbegin(),
1476 Throttler.Acquires.rend()));
1477 // Resources for each file were correctly released.
1478 EXPECT_THAT(Throttler.Releases, ElementsAre(3, 2, 1, 0));
1479 }
1480
1481 Throttler.reset();
1482
1483 // This time, enqueue 2 files, then cancel one of them while still waiting.
1484 // Finally shut down the server. Observe that everything gets cleaned up.
1485 Notification AfterAcquire2;
1486 Notification AfterFinishA;
1487 Throttler.Notify = {1, &AfterAcquire2};
1488 std::vector<std::string> BuiltFilenames;
1489 auto A = testPath("a.cc");
1490 auto B = testPath("b.cc");
1491 Filenames = {A, B};
1492 {
1493 TUScheduler S(CDB, Opts,
1494 std::make_unique<CaptureBuiltFilenames>(BuiltFilenames));
1495 updateWithCallback(S, A, getInputs(A, ""), WantDiagnostics::Yes,
1496 [&] { AfterFinishA.notify(); });
1497 S.update(B, getInputs(B, ""), WantDiagnostics::Yes);
1498 AfterAcquire2.wait();
1499
1500 // The throttler saw all files, but we built none.
1501 EXPECT_THAT(Throttler.Acquires,
1502 testing::UnorderedElementsAreArray(Filenames));
1503 EXPECT_THAT(BuiltFilenames, testing::IsEmpty());
1504 // We haven't released anything yet, we're still waiting.
1505 EXPECT_THAT(Throttler.Releases, testing::IsEmpty());
1506
1507 // FIXME: This is flaky, becaues the request can be destroyed after shutdown
1508 // if it hasn't been dequeued yet (stop() resets NextRequest).
1509 #if 0
1510 // Now close file A, which will shut down its AST worker.
1511 S.remove(A);
1512 // Request is destroyed after the queue shutdown, so release() has happened.
1513 AfterFinishA.wait();
1514 // We still didn't build anything.
1515 EXPECT_THAT(BuiltFilenames, testing::IsEmpty());
1516 // But we've cancelled the request to build A (not sure which its ID is).
1517 EXPECT_THAT(Throttler.Releases, ElementsAre(AnyOf(1, 0)));
1518 #endif
1519
1520 // Now shut down the TU Scheduler.
1521 }
1522 // The throttler saw all files, but we built none.
1523 EXPECT_THAT(Throttler.Acquires,
1524 testing::UnorderedElementsAreArray(Filenames));
1525 EXPECT_THAT(BuiltFilenames, testing::IsEmpty());
1526 // We gave up waiting and everything got released (in some order).
1527 EXPECT_THAT(Throttler.Releases, UnorderedElementsAre(1, 0));
1528 }
1529
1530 } // namespace
1531 } // namespace clangd
1532 } // namespace clang