Debian Codegraph @

llvm-project/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp

raw
1 //===--- GlobalCompilationDatabase.cpp ---------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "GlobalCompilationDatabase.h"
10 #include "Config.h"
11 #include "FS.h"
12 #include "SourceCode.h"
13 #include "support/Logger.h"
14 #include "support/Path.h"
15 #include "support/Threading.h"
16 #include "support/ThreadsafeFS.h"
17 #include "clang/Tooling/ArgumentsAdjusters.h"
18 #include "clang/Tooling/CompilationDatabase.h"
19 #include "clang/Tooling/CompilationDatabasePluginRegistry.h"
20 #include "clang/Tooling/JSONCompilationDatabase.h"
21 #include "llvm/ADT/PointerIntPair.h"
22 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/ScopeExit.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "llvm/ADT/StringMap.h"
26 #include "llvm/Support/Path.h"
27 #include "llvm/Support/VirtualFileSystem.h"
28 #include <atomic>
29 #include <chrono>
30 #include <condition_variable>
31 #include <mutex>
32 #include <optional>
33 #include <string>
34 #include <tuple>
35 #include <vector>
36
37 namespace clang {
38 namespace clangd {
39 namespace {
40
41 // Runs the given action on all parent directories of filename, starting from
42 // deepest directory and going up to root. Stops whenever action succeeds.
43 void actOnAllParentDirectories(PathRef FileName,
44 llvm::function_ref<bool(PathRef)> Action) {
45 for (auto Path = absoluteParent(FileName); !Path.empty() && !Action(Path);
46 Path = absoluteParent(Path))
47 ;
48 }
49
50 } // namespace
51
52 tooling::CompileCommand
53 GlobalCompilationDatabase::getFallbackCommand(PathRef File) const {
54 std::vector<std::string> Argv = {"clang"};
55 // Clang treats .h files as C by default and files without extension as linker
56 // input, resulting in unhelpful diagnostics.
57 // Parsing as Objective C++ is friendly to more cases.
58 auto FileExtension = llvm::sys::path::extension(File);
59 if (FileExtension.empty() || FileExtension == ".h")
60 Argv.push_back("-xobjective-c++-header");
61 Argv.push_back(std::string(File));
62 tooling::CompileCommand Cmd(llvm::sys::path::parent_path(File),
63 llvm::sys::path::filename(File), std::move(Argv),
64 /*Output=*/"");
65 Cmd.Heuristic = "clangd fallback";
66 return Cmd;
67 }
68
69 // Loads and caches the CDB from a single directory.
70 //
71 // This class is threadsafe, which is to say we have independent locks for each
72 // directory we're searching for a CDB.
73 // Loading is deferred until first access.
74 //
75 // The DirectoryBasedCDB keeps a map from path => DirectoryCache.
76 // Typical usage is to:
77 // - 1) determine all the paths that might be searched
78 // - 2) acquire the map lock and get-or-create all the DirectoryCache entries
79 // - 3) release the map lock and query the caches as desired
80 class DirectoryBasedGlobalCompilationDatabase::DirectoryCache {
81 using stopwatch = std::chrono::steady_clock;
82
83 // CachedFile is used to read a CDB file on disk (e.g. compile_commands.json).
84 // It specializes in being able to quickly bail out if the file is unchanged,
85 // which is the common case.
86 // Internally, it stores file metadata so a stat() can verify it's unchanged.
87 // We don't actually cache the content as it's not needed - if the file is
88 // unchanged then the previous CDB is valid.
89 struct CachedFile {
90 CachedFile(llvm::StringRef Parent, llvm::StringRef Rel) {
91 llvm::SmallString<256> Path = Parent;
92 llvm::sys::path::append(Path, Rel);
93 this->Path = Path.str().str();
94 }
95 std::string Path;
96 size_t Size = NoFileCached;
97 llvm::sys::TimePoint<> ModifiedTime;
98 FileDigest ContentHash;
99
100 static constexpr size_t NoFileCached = -1;
101
102 struct LoadResult {
103 enum {
104 FileNotFound,
105 TransientError,
106 FoundSameData,
107 FoundNewData,
108 } Result;
109 std::unique_ptr<llvm::MemoryBuffer> Buffer; // Set only if FoundNewData
110 };
111
112 LoadResult load(llvm::vfs::FileSystem &FS, bool HasOldData);
113 };
114
115 // If we've looked for a CDB here and found none, the time when that happened.
116 // (Atomics make it possible for get() to return without taking a lock)
117 std::atomic<stopwatch::rep> NoCDBAt = {
118 stopwatch::time_point::min().time_since_epoch().count()};
119
120 // Guards the following cache state.
121 std::mutex Mu;
122 // When was the cache last known to be in sync with disk state?
123 stopwatch::time_point CachePopulatedAt = stopwatch::time_point::min();
124 // Whether a new CDB has been loaded but not broadcast yet.
125 bool NeedsBroadcast = false;
126 // Last loaded CDB, meaningful if CachePopulatedAt was ever set.
127 // shared_ptr so we can overwrite this when callers are still using the CDB.
128 std::shared_ptr<tooling::CompilationDatabase> CDB;
129 // File metadata for the CDB files we support tracking directly.
130 CachedFile CompileCommandsJson;
131 CachedFile BuildCompileCommandsJson;
132 CachedFile CompileFlagsTxt;
133 // CachedFile member corresponding to CDB.
134 // CDB | ACF | Scenario
135 // null | null | no CDB found, or initial empty cache
136 // set | null | CDB was loaded via generic plugin interface
137 // null | set | found known CDB file, but parsing it failed
138 // set | set | CDB was parsed from a known file
139 CachedFile *ActiveCachedFile = nullptr;
140
141 public:
142 DirectoryCache(llvm::StringRef Path)
143 : CompileCommandsJson(Path, "compile_commands.json"),
144 BuildCompileCommandsJson(Path, "build/compile_commands.json"),
145 CompileFlagsTxt(Path, "compile_flags.txt"), Path(Path) {
146 assert(llvm::sys::path::is_absolute(Path));
147 }
148
149 // Absolute canonical path that we're the cache for. (Not case-folded).
150 const std::string Path;
151
152 // Get the CDB associated with this directory.
153 // ShouldBroadcast:
154 // - as input, signals whether the caller is willing to broadcast a
155 // newly-discovered CDB. (e.g. to trigger background indexing)
156 // - as output, signals whether the caller should do so.
157 // (If a new CDB is discovered and ShouldBroadcast is false, we mark the
158 // CDB as needing broadcast, and broadcast it next time we can).
159 std::shared_ptr<const tooling::CompilationDatabase>
160 get(const ThreadsafeFS &TFS, bool &ShouldBroadcast,
161 stopwatch::time_point FreshTime, stopwatch::time_point FreshTimeMissing) {
162 // Fast path for common case without taking lock.
163 if (stopwatch::time_point(stopwatch::duration(NoCDBAt.load())) >
164 FreshTimeMissing) {
165 ShouldBroadcast = false;
166 return nullptr;
167 }
168
169 std::lock_guard<std::mutex> Lock(Mu);
170 auto RequestBroadcast = llvm::make_scope_exit([&, OldCDB(CDB.get())] {
171 // If we loaded a new CDB, it should be broadcast at some point.
172 if (CDB != nullptr && CDB.get() != OldCDB)
173 NeedsBroadcast = true;
174 else if (CDB == nullptr) // nothing to broadcast anymore!
175 NeedsBroadcast = false;
176 // If we have something to broadcast, then do so iff allowed.
177 if (!ShouldBroadcast)
178 return;
179 ShouldBroadcast = NeedsBroadcast;
180 NeedsBroadcast = false;
181 });
182
183 // If our cache is valid, serve from it.
184 if (CachePopulatedAt > FreshTime)
185 return CDB;
186
187 if (/*MayCache=*/load(*TFS.view(/*CWD=*/std::nullopt))) {
188 // Use new timestamp, as loading may be slow.
189 CachePopulatedAt = stopwatch::now();
190 NoCDBAt.store((CDB ? stopwatch::time_point::min() : CachePopulatedAt)
191 .time_since_epoch()
192 .count());
193 }
194
195 return CDB;
196 }
197
198 private:
199 // Updates `CDB` from disk state. Returns false on failure.
200 bool load(llvm::vfs::FileSystem &FS);
201 };
202
203 DirectoryBasedGlobalCompilationDatabase::DirectoryCache::CachedFile::LoadResult
204 DirectoryBasedGlobalCompilationDatabase::DirectoryCache::CachedFile::load(
205 llvm::vfs::FileSystem &FS, bool HasOldData) {
206 auto Stat = FS.status(Path);
207 if (!Stat || !Stat->isRegularFile()) {
208 Size = NoFileCached;
209 ContentHash = {};
210 return {LoadResult::FileNotFound, nullptr};
211 }
212 // If both the size and mtime match, presume unchanged without reading.
213 if (HasOldData && Stat->getLastModificationTime() == ModifiedTime &&
214 Stat->getSize() == Size)
215 return {LoadResult::FoundSameData, nullptr};
216 auto Buf = FS.getBufferForFile(Path);
217 if (!Buf || (*Buf)->getBufferSize() != Stat->getSize()) {
218 // Don't clear the cache - possible we're seeing inconsistent size as the
219 // file is being recreated. If it ends up identical later, great!
220 //
221 // This isn't a complete solution: if we see a partial file but stat/read
222 // agree on its size, we're ultimately going to have spurious CDB reloads.
223 // May be worth fixing if generators don't write atomically (CMake does).
224 elog("Failed to read {0}: {1}", Path,
225 Buf ? "size changed" : Buf.getError().message());
226 return {LoadResult::TransientError, nullptr};
227 }
228
229 FileDigest NewContentHash = digest((*Buf)->getBuffer());
230 if (HasOldData && NewContentHash == ContentHash) {
231 // mtime changed but data is the same: avoid rebuilding the CDB.
232 ModifiedTime = Stat->getLastModificationTime();
233 return {LoadResult::FoundSameData, nullptr};
234 }
235
236 Size = (*Buf)->getBufferSize();
237 ModifiedTime = Stat->getLastModificationTime();
238 ContentHash = NewContentHash;
239 return {LoadResult::FoundNewData, std::move(*Buf)};
240 }
241
242 // Adapt CDB-loading functions to a common interface for DirectoryCache::load().
243 static std::unique_ptr<tooling::CompilationDatabase>
244 parseJSON(PathRef Path, llvm::StringRef Data, std::string &Error) {
245 if (auto CDB = tooling::JSONCompilationDatabase::loadFromBuffer(
246 Data, Error, tooling::JSONCommandLineSyntax::AutoDetect)) {
247 // FS used for expanding response files.
248 // FIXME: ExpandResponseFilesDatabase appears not to provide the usual
249 // thread-safety guarantees, as the access to FS is not locked!
250 // For now, use the real FS, which is known to be threadsafe (if we don't
251 // use/change working directory, which ExpandResponseFilesDatabase doesn't).
252 auto FS = llvm::vfs::getRealFileSystem();
253 return tooling::inferTargetAndDriverMode(
254 tooling::inferMissingCompileCommands(
255 expandResponseFiles(std::move(CDB), std::move(FS))));
256 }
257 return nullptr;
258 }
259 static std::unique_ptr<tooling::CompilationDatabase>
260 parseFixed(PathRef Path, llvm::StringRef Data, std::string &Error) {
261 return tooling::FixedCompilationDatabase::loadFromBuffer(
262 llvm::sys::path::parent_path(Path), Data, Error);
263 }
264
265 bool DirectoryBasedGlobalCompilationDatabase::DirectoryCache::load(
266 llvm::vfs::FileSystem &FS) {
267 dlog("Probing directory {0}", Path);
268 std::string Error;
269
270 // Load from the specially-supported compilation databases (JSON + Fixed).
271 // For these, we know the files they read and cache their metadata so we can
272 // cheaply validate whether they've changed, and hot-reload if they have.
273 // (As a bonus, these are also VFS-clean)!
274 struct CDBFile {
275 CachedFile *File;
276 // Wrapper for {Fixed,JSON}CompilationDatabase::loadFromBuffer.
277 std::unique_ptr<tooling::CompilationDatabase> (*Parser)(
278 PathRef,
279 /*Data*/ llvm::StringRef,
280 /*ErrorMsg*/ std::string &);
281 };
282 for (const auto &Entry : {CDBFile{&CompileCommandsJson, parseJSON},
283 CDBFile{&BuildCompileCommandsJson, parseJSON},
284 CDBFile{&CompileFlagsTxt, parseFixed}}) {
285 bool Active = ActiveCachedFile == Entry.File;
286 auto Loaded = Entry.File->load(FS, Active);
287 switch (Loaded.Result) {
288 case CachedFile::LoadResult::FileNotFound:
289 if (Active) {
290 log("Unloaded compilation database from {0}", Entry.File->Path);
291 ActiveCachedFile = nullptr;
292 CDB = nullptr;
293 }
294 // Continue looking at other candidates.
295 break;
296 case CachedFile::LoadResult::TransientError:
297 // File existed but we couldn't read it. Reuse the cache, retry later.
298 return false; // Load again next time.
299 case CachedFile::LoadResult::FoundSameData:
300 assert(Active && "CachedFile may not return 'same data' if !HasOldData");
301 // This is the critical file, and it hasn't changed.
302 return true;
303 case CachedFile::LoadResult::FoundNewData:
304 // We have a new CDB!
305 CDB = Entry.Parser(Entry.File->Path, Loaded.Buffer->getBuffer(), Error);
306 if (CDB)
307 log("{0} compilation database from {1}", Active ? "Reloaded" : "Loaded",
308 Entry.File->Path);
309 else
310 elog("Failed to load compilation database from {0}: {1}",
311 Entry.File->Path, Error);
312 ActiveCachedFile = Entry.File;
313 return true;
314 }
315 }
316
317 // Fall back to generic handling of compilation databases.
318 // We don't know what files they read, so can't efficiently check whether
319 // they need to be reloaded. So we never do that.
320 // FIXME: the interface doesn't provide a way to virtualize FS access.
321
322 // Don't try these more than once. If we've scanned before, we're done.
323 if (CachePopulatedAt > stopwatch::time_point::min())
324 return true;
325 for (const auto &Entry :
326 tooling::CompilationDatabasePluginRegistry::entries()) {
327 // Avoid duplicating the special cases handled above.
328 if (Entry.getName() == "fixed-compilation-database" ||
329 Entry.getName() == "json-compilation-database")
330 continue;
331 auto Plugin = Entry.instantiate();
332 if (auto CDB = Plugin->loadFromDirectory(Path, Error)) {
333 log("Loaded compilation database from {0} with plugin {1}", Path,
334 Entry.getName());
335 this->CDB = std::move(CDB);
336 return true;
337 }
338 // Don't log Error here, it's usually just "couldn't find <file>".
339 }
340 dlog("No compilation database at {0}", Path);
341 return true;
342 }
343
344 DirectoryBasedGlobalCompilationDatabase::
345 DirectoryBasedGlobalCompilationDatabase(const Options &Opts)
346 : Opts(Opts), Broadcaster(std::make_unique<BroadcastThread>(*this)) {
347 if (!this->Opts.ContextProvider)
348 this->Opts.ContextProvider = [](llvm::StringRef) {
349 return Context::current().clone();
350 };
351 }
352
353 DirectoryBasedGlobalCompilationDatabase::
354 ~DirectoryBasedGlobalCompilationDatabase() = default;
355
356 std::optional<tooling::CompileCommand>
357 DirectoryBasedGlobalCompilationDatabase::getCompileCommand(PathRef File) const {
358 CDBLookupRequest Req;
359 Req.FileName = File;
360 Req.ShouldBroadcast = true;
361 auto Now = std::chrono::steady_clock::now();
362 Req.FreshTime = Now - Opts.RevalidateAfter;
363 Req.FreshTimeMissing = Now - Opts.RevalidateMissingAfter;
364
365 auto Res = lookupCDB(Req);
366 if (!Res) {
367 log("Failed to find compilation database for {0}", File);
368 return std::nullopt;
369 }
370
371 auto Candidates = Res->CDB->getCompileCommands(File);
372 if (!Candidates.empty())
373 return std::move(Candidates.front());
374
375 return std::nullopt;
376 }
377
378 std::vector<DirectoryBasedGlobalCompilationDatabase::DirectoryCache *>
379 DirectoryBasedGlobalCompilationDatabase::getDirectoryCaches(
380 llvm::ArrayRef<llvm::StringRef> Dirs) const {
381 std::vector<std::string> FoldedDirs;
382 FoldedDirs.reserve(Dirs.size());
383 for (const auto &Dir : Dirs) {
384 #ifndef NDEBUG
385 if (!llvm::sys::path::is_absolute(Dir))
386 elog("Trying to cache CDB for relative {0}");
387 #endif
388 FoldedDirs.push_back(maybeCaseFoldPath(Dir));
389 }
390
391 std::vector<DirectoryCache *> Ret;
392 Ret.reserve(Dirs.size());
393
394 std::lock_guard<std::mutex> Lock(DirCachesMutex);
395 for (unsigned I = 0; I < Dirs.size(); ++I)
396 Ret.push_back(&DirCaches.try_emplace(FoldedDirs[I], Dirs[I]).first->second);
397 return Ret;
398 }
399
400 std::optional<DirectoryBasedGlobalCompilationDatabase::CDBLookupResult>
401 DirectoryBasedGlobalCompilationDatabase::lookupCDB(
402 CDBLookupRequest Request) const {
403 assert(llvm::sys::path::is_absolute(Request.FileName) &&
404 "path must be absolute");
405
406 std::string Storage;
407 std::vector<llvm::StringRef> SearchDirs;
408 if (Opts.CompileCommandsDir) // FIXME: unify this case with config.
409 SearchDirs = {*Opts.CompileCommandsDir};
410 else {
411 WithContext WithProvidedContext(Opts.ContextProvider(Request.FileName));
412 const auto &Spec = Config::current().CompileFlags.CDBSearch;
413 switch (Spec.Policy) {
414 case Config::CDBSearchSpec::NoCDBSearch:
415 return std::nullopt;
416 case Config::CDBSearchSpec::FixedDir:
417 Storage = *Spec.FixedCDBPath;
418 SearchDirs = {Storage};
419 break;
420 case Config::CDBSearchSpec::Ancestors:
421 // Traverse the canonical version to prevent false positives. i.e.:
422 // src/build/../a.cc can detect a CDB in /src/build if not
423 // canonicalized.
424 Storage = removeDots(Request.FileName);
425 actOnAllParentDirectories(Storage, [&](llvm::StringRef Dir) {
426 SearchDirs.push_back(Dir);
427 return false;
428 });
429 }
430 }
431
432 std::shared_ptr<const tooling::CompilationDatabase> CDB = nullptr;
433 bool ShouldBroadcast = false;
434 DirectoryCache *DirCache = nullptr;
435 for (DirectoryCache *Candidate : getDirectoryCaches(SearchDirs)) {
436 bool CandidateShouldBroadcast = Request.ShouldBroadcast;
437 if ((CDB = Candidate->get(Opts.TFS, CandidateShouldBroadcast,
438 Request.FreshTime, Request.FreshTimeMissing))) {
439 DirCache = Candidate;
440 ShouldBroadcast = CandidateShouldBroadcast;
441 break;
442 }
443 }
444
445 if (!CDB)
446 return std::nullopt;
447
448 CDBLookupResult Result;
449 Result.CDB = std::move(CDB);
450 Result.PI.SourceRoot = DirCache->Path;
451
452 if (ShouldBroadcast)
453 broadcastCDB(Result);
454 return Result;
455 }
456
457 // The broadcast thread announces files with new compile commands to the world.
458 // Primarily this is used to enqueue them for background indexing.
459 //
460 // It's on a separate thread because:
461 // - otherwise it would block the first parse of the initial file
462 // - we need to enumerate all files in the CDB, of which there are many
463 // - we (will) have to evaluate config for every file in the CDB, which is slow
464 class DirectoryBasedGlobalCompilationDatabase::BroadcastThread {
465 class Filter;
466 DirectoryBasedGlobalCompilationDatabase &Parent;
467
468 std::mutex Mu;
469 std::condition_variable CV;
470 // Shutdown flag (CV is notified after writing).
471 // This is atomic so that broadcasts can also observe it and abort early.
472 std::atomic<bool> ShouldStop = {false};
473 struct Task {
474 CDBLookupResult Lookup;
475 Context Ctx;
476 };
477 std::deque<Task> Queue;
478 std::optional<Task> ActiveTask;
479 std::thread Thread; // Must be last member.
480
481 // Thread body: this is just the basic queue procesing boilerplate.
482 void run() {
483 std::unique_lock<std::mutex> Lock(Mu);
484 while (true) {
485 bool Stopping = false;
486 CV.wait(Lock, [&] {
487 return (Stopping = ShouldStop.load(std::memory_order_acquire)) ||
488 !Queue.empty();
489 });
490 if (Stopping) {
491 Queue.clear();
492 CV.notify_all();
493 return;
494 }
495 ActiveTask = std::move(Queue.front());
496 Queue.pop_front();
497
498 Lock.unlock();
499 {
500 WithContext WithCtx(std::move(ActiveTask->Ctx));
501 process(ActiveTask->Lookup);
502 }
503 Lock.lock();
504 ActiveTask.reset();
505 CV.notify_all();
506 }
507 }
508
509 // Inspects a new CDB and broadcasts the files it owns.
510 void process(const CDBLookupResult &T);
511
512 public:
513 BroadcastThread(DirectoryBasedGlobalCompilationDatabase &Parent)
514 : Parent(Parent), Thread([this] { run(); }) {}
515
516 void enqueue(CDBLookupResult Lookup) {
517 {
518 assert(!Lookup.PI.SourceRoot.empty());
519 std::lock_guard<std::mutex> Lock(Mu);
520 // New CDB takes precedence over any queued one for the same directory.
521 llvm::erase_if(Queue, [&](const Task &T) {
522 return T.Lookup.PI.SourceRoot == Lookup.PI.SourceRoot;
523 });
524 Queue.push_back({std::move(Lookup), Context::current().clone()});
525 }
526 CV.notify_all();
527 }
528
529 bool blockUntilIdle(Deadline Timeout) {
530 std::unique_lock<std::mutex> Lock(Mu);
531 return wait(Lock, CV, Timeout,
532 [&] { return Queue.empty() && !ActiveTask; });
533 }
534
535 ~BroadcastThread() {
536 {
537 std::lock_guard<std::mutex> Lock(Mu);
538 ShouldStop.store(true, std::memory_order_release);
539 }
540 CV.notify_all();
541 Thread.join();
542 }
543 };
544
545 // The DirBasedCDB associates each file with a specific CDB.
546 // When a CDB is discovered, it may claim to describe files that we associate
547 // with a different CDB. We do not want to broadcast discovery of these, and
548 // trigger background indexing of them.
549 //
550 // We must filter the list, and check whether they are associated with this CDB.
551 // This class attempts to do so efficiently.
552 //
553 // Roughly, it:
554 // - loads the config for each file, and determines the relevant search path
555 // - gathers all directories that are part of any search path
556 // - (lazily) checks for a CDB in each such directory at most once
557 // - walks the search path for each file and determines whether to include it.
558 class DirectoryBasedGlobalCompilationDatabase::BroadcastThread::Filter {
559 llvm::StringRef ThisDir;
560 DirectoryBasedGlobalCompilationDatabase &Parent;
561
562 // Keep track of all directories we might check for CDBs.
563 struct DirInfo {
564 DirectoryCache *Cache = nullptr;
565 enum { Unknown, Missing, TargetCDB, OtherCDB } State = Unknown;
566 DirInfo *Parent = nullptr;
567 };
568 llvm::StringMap<DirInfo> Dirs;
569
570 // A search path starts at a directory, and either includes ancestors or not.
571 using SearchPath = llvm::PointerIntPair<DirInfo *, 1>;
572
573 // Add all ancestor directories of FilePath to the tracked set.
574 // Returns the immediate parent of the file.
575 DirInfo *addParents(llvm::StringRef FilePath) {
576 DirInfo *Leaf = nullptr;
577 DirInfo *Child = nullptr;
578 actOnAllParentDirectories(FilePath, [&](llvm::StringRef Dir) {
579 auto &Info = Dirs[Dir];
580 // If this is the first iteration, then this node is the overall result.
581 if (!Leaf)
582 Leaf = &Info;
583 // Fill in the parent link from the previous iteration to this parent.
584 if (Child)
585 Child->Parent = &Info;
586 // Keep walking, whether we inserted or not, if parent link is missing.
587 // (If it's present, parent links must be present up to the root, so stop)
588 Child = &Info;
589 return Info.Parent != nullptr;
590 });
591 return Leaf;
592 }
593
594 // Populates DirInfo::Cache (and State, if it is TargetCDB).
595 void grabCaches() {
596 // Fast path out if there were no files, or CDB loading is off.
597 if (Dirs.empty())
598 return;
599
600 std::vector<llvm::StringRef> DirKeys;
601 std::vector<DirInfo *> DirValues;
602 DirKeys.reserve(Dirs.size() + 1);
603 DirValues.reserve(Dirs.size());
604 for (auto &E : Dirs) {
605 DirKeys.push_back(E.first());
606 DirValues.push_back(&E.second);
607 }
608
609 // Also look up the cache entry for the CDB we're broadcasting.
610 // Comparing DirectoryCache pointers is more robust than checking string
611 // equality, e.g. reuses the case-sensitivity handling.
612 DirKeys.push_back(ThisDir);
613 auto DirCaches = Parent.getDirectoryCaches(DirKeys);
614 const DirectoryCache *ThisCache = DirCaches.back();
615 DirCaches.pop_back();
616 DirKeys.pop_back();
617
618 for (unsigned I = 0; I < DirKeys.size(); ++I) {
619 DirValues[I]->Cache = DirCaches[I];
620 if (DirCaches[I] == ThisCache)
621 DirValues[I]->State = DirInfo::TargetCDB;
622 }
623 }
624
625 // Should we include a file from this search path?
626 bool shouldInclude(SearchPath P) {
627 DirInfo *Info = P.getPointer();
628 if (!Info)
629 return false;
630 if (Info->State == DirInfo::Unknown) {
631 assert(Info->Cache && "grabCaches() should have filled this");
632 // Given that we know that CDBs have been moved/generated, don't trust
633 // caches. (This should be rare, so it's OK to add a little latency).
634 constexpr auto IgnoreCache = std::chrono::steady_clock::time_point::max();
635 // Don't broadcast CDBs discovered while broadcasting!
636 bool ShouldBroadcast = false;
637 bool Exists =
638 nullptr != Info->Cache->get(Parent.Opts.TFS, ShouldBroadcast,
639 /*FreshTime=*/IgnoreCache,
640 /*FreshTimeMissing=*/IgnoreCache);
641 Info->State = Exists ? DirInfo::OtherCDB : DirInfo::Missing;
642 }
643 // If we have a CDB, include the file if it's the target CDB only.
644 if (Info->State != DirInfo::Missing)
645 return Info->State == DirInfo::TargetCDB;
646 // If we have no CDB and no relevant parent, don't include the file.
647 if (!P.getInt() || !Info->Parent)
648 return false;
649 // Walk up to the next parent.
650 return shouldInclude(SearchPath(Info->Parent, 1));
651 }
652
653 public:
654 Filter(llvm::StringRef ThisDir,
655 DirectoryBasedGlobalCompilationDatabase &Parent)
656 : ThisDir(ThisDir), Parent(Parent) {}
657
658 std::vector<std::string> filter(std::vector<std::string> AllFiles,
659 std::atomic<bool> &ShouldStop) {
660 std::vector<std::string> Filtered;
661 // Allow for clean early-exit of the slow parts.
662 auto ExitEarly = [&] {
663 if (ShouldStop.load(std::memory_order_acquire)) {
664 log("Giving up on broadcasting CDB, as we're shutting down");
665 Filtered.clear();
666 return true;
667 }
668 return false;
669 };
670 // Compute search path for each file.
671 std::vector<SearchPath> SearchPaths(AllFiles.size());
672 for (unsigned I = 0; I < AllFiles.size(); ++I) {
673 if (Parent.Opts.CompileCommandsDir) { // FIXME: unify with config
674 SearchPaths[I].setPointer(&Dirs[*Parent.Opts.CompileCommandsDir]);
675 continue;
676 }
677 if (ExitEarly()) // loading config may be slow
678 return Filtered;
679 WithContext WithProvidedContent(Parent.Opts.ContextProvider(AllFiles[I]));
680 const Config::CDBSearchSpec &Spec =
681 Config::current().CompileFlags.CDBSearch;
682 switch (Spec.Policy) {
683 case Config::CDBSearchSpec::NoCDBSearch:
684 break;
685 case Config::CDBSearchSpec::Ancestors:
686 SearchPaths[I].setInt(/*Recursive=*/1);
687 SearchPaths[I].setPointer(addParents(AllFiles[I]));
688 break;
689 case Config::CDBSearchSpec::FixedDir:
690 SearchPaths[I].setPointer(&Dirs[*Spec.FixedCDBPath]);
691 break;
692 }
693 }
694 // Get the CDB cache for each dir on the search path, but don't load yet.
695 grabCaches();
696 // Now work out which files we want to keep, loading CDBs where needed.
697 for (unsigned I = 0; I < AllFiles.size(); ++I) {
698 if (ExitEarly()) // loading CDBs may be slow
699 return Filtered;
700 if (shouldInclude(SearchPaths[I]))
701 Filtered.push_back(std::move(AllFiles[I]));
702 }
703 return Filtered;
704 }
705 };
706
707 void DirectoryBasedGlobalCompilationDatabase::BroadcastThread::process(
708 const CDBLookupResult &T) {
709 vlog("Broadcasting compilation database from {0}", T.PI.SourceRoot);
710 std::vector<std::string> GovernedFiles =
711 Filter(T.PI.SourceRoot, Parent).filter(T.CDB->getAllFiles(), ShouldStop);
712 if (!GovernedFiles.empty())
713 Parent.OnCommandChanged.broadcast(std::move(GovernedFiles));
714 }
715
716 void DirectoryBasedGlobalCompilationDatabase::broadcastCDB(
717 CDBLookupResult Result) const {
718 assert(Result.CDB && "Trying to broadcast an invalid CDB!");
719 Broadcaster->enqueue(Result);
720 }
721
722 bool DirectoryBasedGlobalCompilationDatabase::blockUntilIdle(
723 Deadline Timeout) const {
724 return Broadcaster->blockUntilIdle(Timeout);
725 }
726
727 std::optional<ProjectInfo>
728 DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const {
729 CDBLookupRequest Req;
730 Req.FileName = File;
731 Req.ShouldBroadcast = false;
732 Req.FreshTime = Req.FreshTimeMissing =
733 std::chrono::steady_clock::time_point::min();
734 auto Res = lookupCDB(Req);
735 if (!Res)
736 return std::nullopt;
737 return Res->PI;
738 }
739
740 OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base,
741 std::vector<std::string> FallbackFlags,
742 CommandMangler Mangler)
743 : DelegatingCDB(Base), Mangler(std::move(Mangler)),
744 FallbackFlags(std::move(FallbackFlags)) {}
745
746 std::optional<tooling::CompileCommand>
747 OverlayCDB::getCompileCommand(PathRef File) const {
748 std::optional<tooling::CompileCommand> Cmd;
749 {
750 std::lock_guard<std::mutex> Lock(Mutex);
751 auto It = Commands.find(removeDots(File));
752 if (It != Commands.end())
753 Cmd = It->second;
754 }
755 if (!Cmd)
756 Cmd = DelegatingCDB::getCompileCommand(File);
757 if (!Cmd)
758 return std::nullopt;
759 if (Mangler)
760 Mangler(*Cmd, File);
761 return Cmd;
762 }
763
764 tooling::CompileCommand OverlayCDB::getFallbackCommand(PathRef File) const {
765 auto Cmd = DelegatingCDB::getFallbackCommand(File);
766 std::lock_guard<std::mutex> Lock(Mutex);
767 Cmd.CommandLine.insert(Cmd.CommandLine.end(), FallbackFlags.begin(),
768 FallbackFlags.end());
769 if (Mangler)
770 Mangler(Cmd, File);
771 return Cmd;
772 }
773
774 void OverlayCDB::setCompileCommand(PathRef File,
775 std::optional<tooling::CompileCommand> Cmd) {
776 // We store a canonical version internally to prevent mismatches between set
777 // and get compile commands. Also it assures clients listening to broadcasts
778 // doesn't receive different names for the same file.
779 std::string CanonPath = removeDots(File);
780 {
781 std::unique_lock<std::mutex> Lock(Mutex);
782 if (Cmd)
783 Commands[CanonPath] = std::move(*Cmd);
784 else
785 Commands.erase(CanonPath);
786 }
787 OnCommandChanged.broadcast({CanonPath});
788 }
789
790 DelegatingCDB::DelegatingCDB(const GlobalCompilationDatabase *Base)
791 : Base(Base) {
792 if (Base)
793 BaseChanged = Base->watch([this](const std::vector<std::string> Changes) {
794 OnCommandChanged.broadcast(Changes);
795 });
796 }
797
798 DelegatingCDB::DelegatingCDB(std::unique_ptr<GlobalCompilationDatabase> Base)
799 : DelegatingCDB(Base.get()) {
800 BaseOwner = std::move(Base);
801 }
802
803 std::optional<tooling::CompileCommand>
804 DelegatingCDB::getCompileCommand(PathRef File) const {
805 if (!Base)
806 return std::nullopt;
807 return Base->getCompileCommand(File);
808 }
809
810 std::optional<ProjectInfo> DelegatingCDB::getProjectInfo(PathRef File) const {
811 if (!Base)
812 return std::nullopt;
813 return Base->getProjectInfo(File);
814 }
815
816 tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const {
817 if (!Base)
818 return GlobalCompilationDatabase::getFallbackCommand(File);
819 return Base->getFallbackCommand(File);
820 }
821
822 bool DelegatingCDB::blockUntilIdle(Deadline D) const {
823 if (!Base)
824 return true;
825 return Base->blockUntilIdle(D);
826 }
827
828 } // namespace clangd
829 } // namespace clang