| 1 |
//===--- GlobalCompilationDatabase.cpp ---------------------------*- C++-*-===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
|
| 9 |
#include "GlobalCompilationDatabase.h" |
| 10 |
#include "Config.h" |
| 11 |
#include "FS.h" |
| 12 |
#include "SourceCode.h" |
| 13 |
#include "support/Logger.h" |
| 14 |
#include "support/Path.h" |
| 15 |
#include "support/Threading.h" |
| 16 |
#include "support/ThreadsafeFS.h" |
| 17 |
#include "clang/Tooling/ArgumentsAdjusters.h" |
| 18 |
#include "clang/Tooling/CompilationDatabase.h" |
| 19 |
#include "clang/Tooling/CompilationDatabasePluginRegistry.h" |
| 20 |
#include "clang/Tooling/JSONCompilationDatabase.h" |
| 21 |
#include "llvm/ADT/PointerIntPair.h" |
| 22 |
#include "llvm/ADT/STLExtras.h" |
| 23 |
#include "llvm/ADT/ScopeExit.h" |
| 24 |
#include "llvm/ADT/SmallString.h" |
| 25 |
#include "llvm/ADT/StringMap.h" |
| 26 |
#include "llvm/Support/Path.h" |
| 27 |
#include "llvm/Support/VirtualFileSystem.h" |
| 28 |
#include <atomic> |
| 29 |
#include <chrono> |
| 30 |
#include <condition_variable> |
| 31 |
#include <mutex> |
| 32 |
#include <optional> |
| 33 |
#include <string> |
| 34 |
#include <tuple> |
| 35 |
#include <vector> |
| 36 |
|
| 37 |
namespace clang { |
| 38 |
namespace clangd { |
| 39 |
namespace { |
| 40 |
|
| 41 |
// Runs the given action on all parent directories of filename, starting from |
| 42 |
// deepest directory and going up to root. Stops whenever action succeeds. |
| 43 |
void actOnAllParentDirectories(PathRef FileName, |
| 44 |
llvm::function_ref<bool(PathRef)> Action) { |
| 45 |
for (auto Path = absoluteParent(FileName); !Path.empty() && !Action(Path); |
| 46 |
Path = absoluteParent(Path)) |
| 47 |
; |
| 48 |
} |
| 49 |
|
| 50 |
} // namespace |
| 51 |
|
| 52 |
tooling::CompileCommand |
| 53 |
GlobalCompilationDatabase::getFallbackCommand(PathRef File) const { |
| 54 |
std::vector<std::string> Argv = {"clang"}; |
| 55 |
// Clang treats .h files as C by default and files without extension as linker |
| 56 |
// input, resulting in unhelpful diagnostics. |
| 57 |
// Parsing as Objective C++ is friendly to more cases. |
| 58 |
auto FileExtension = llvm::sys::path::extension(File); |
| 59 |
if (FileExtension.empty() || FileExtension == ".h") |
| 60 |
Argv.push_back("-xobjective-c++-header"); |
| 61 |
Argv.push_back(std::string(File)); |
| 62 |
tooling::CompileCommand Cmd(llvm::sys::path::parent_path(File), |
| 63 |
llvm::sys::path::filename(File), std::move(Argv), |
| 64 |
/*Output=*/""); |
| 65 |
Cmd.Heuristic = "clangd fallback"; |
| 66 |
return Cmd; |
| 67 |
} |
| 68 |
|
| 69 |
// Loads and caches the CDB from a single directory. |
| 70 |
// |
| 71 |
// This class is threadsafe, which is to say we have independent locks for each |
| 72 |
// directory we're searching for a CDB. |
| 73 |
// Loading is deferred until first access. |
| 74 |
// |
| 75 |
// The DirectoryBasedCDB keeps a map from path => DirectoryCache. |
| 76 |
// Typical usage is to: |
| 77 |
// - 1) determine all the paths that might be searched |
| 78 |
// - 2) acquire the map lock and get-or-create all the DirectoryCache entries |
| 79 |
// - 3) release the map lock and query the caches as desired |
| 80 |
class DirectoryBasedGlobalCompilationDatabase::DirectoryCache { |
| 81 |
using stopwatch = std::chrono::steady_clock; |
| 82 |
|
| 83 |
// CachedFile is used to read a CDB file on disk (e.g. compile_commands.json). |
| 84 |
// It specializes in being able to quickly bail out if the file is unchanged, |
| 85 |
// which is the common case. |
| 86 |
// Internally, it stores file metadata so a stat() can verify it's unchanged. |
| 87 |
// We don't actually cache the content as it's not needed - if the file is |
| 88 |
// unchanged then the previous CDB is valid. |
| 89 |
struct CachedFile { |
| 90 |
CachedFile(llvm::StringRef Parent, llvm::StringRef Rel) { |
| 91 |
llvm::SmallString<256> Path = Parent; |
| 92 |
llvm::sys::path::append(Path, Rel); |
| 93 |
this->Path = Path.str().str(); |
| 94 |
} |
| 95 |
std::string Path; |
| 96 |
size_t Size = NoFileCached; |
| 97 |
llvm::sys::TimePoint<> ModifiedTime; |
| 98 |
FileDigest ContentHash; |
| 99 |
|
| 100 |
static constexpr size_t NoFileCached = -1; |
| 101 |
|
| 102 |
struct LoadResult { |
| 103 |
enum { |
| 104 |
FileNotFound, |
| 105 |
TransientError, |
| 106 |
FoundSameData, |
| 107 |
FoundNewData, |
| 108 |
} Result; |
| 109 |
std::unique_ptr<llvm::MemoryBuffer> Buffer; // Set only if FoundNewData |
| 110 |
}; |
| 111 |
|
| 112 |
LoadResult load(llvm::vfs::FileSystem &FS, bool HasOldData); |
| 113 |
}; |
| 114 |
|
| 115 |
// If we've looked for a CDB here and found none, the time when that happened. |
| 116 |
// (Atomics make it possible for get() to return without taking a lock) |
| 117 |
std::atomic<stopwatch::rep> NoCDBAt = { |
| 118 |
stopwatch::time_point::min().time_since_epoch().count()}; |
| 119 |
|
| 120 |
// Guards the following cache state. |
| 121 |
std::mutex Mu; |
| 122 |
// When was the cache last known to be in sync with disk state? |
| 123 |
stopwatch::time_point CachePopulatedAt = stopwatch::time_point::min(); |
| 124 |
// Whether a new CDB has been loaded but not broadcast yet. |
| 125 |
bool NeedsBroadcast = false; |
| 126 |
// Last loaded CDB, meaningful if CachePopulatedAt was ever set. |
| 127 |
// shared_ptr so we can overwrite this when callers are still using the CDB. |
| 128 |
std::shared_ptr<tooling::CompilationDatabase> CDB; |
| 129 |
// File metadata for the CDB files we support tracking directly. |
| 130 |
CachedFile CompileCommandsJson; |
| 131 |
CachedFile BuildCompileCommandsJson; |
| 132 |
CachedFile CompileFlagsTxt; |
| 133 |
// CachedFile member corresponding to CDB. |
| 134 |
// CDB | ACF | Scenario |
| 135 |
// null | null | no CDB found, or initial empty cache |
| 136 |
// set | null | CDB was loaded via generic plugin interface |
| 137 |
// null | set | found known CDB file, but parsing it failed |
| 138 |
// set | set | CDB was parsed from a known file |
| 139 |
CachedFile *ActiveCachedFile = nullptr; |
| 140 |
|
| 141 |
public: |
| 142 |
DirectoryCache(llvm::StringRef Path) |
| 143 |
: CompileCommandsJson(Path, "compile_commands.json"), |
| 144 |
BuildCompileCommandsJson(Path, "build/compile_commands.json"), |
| 145 |
CompileFlagsTxt(Path, "compile_flags.txt"), Path(Path) { |
| 146 |
assert(llvm::sys::path::is_absolute(Path)); |
| 147 |
} |
| 148 |
|
| 149 |
// Absolute canonical path that we're the cache for. (Not case-folded). |
| 150 |
const std::string Path; |
| 151 |
|
| 152 |
// Get the CDB associated with this directory. |
| 153 |
// ShouldBroadcast: |
| 154 |
// - as input, signals whether the caller is willing to broadcast a |
| 155 |
// newly-discovered CDB. (e.g. to trigger background indexing) |
| 156 |
// - as output, signals whether the caller should do so. |
| 157 |
// (If a new CDB is discovered and ShouldBroadcast is false, we mark the |
| 158 |
// CDB as needing broadcast, and broadcast it next time we can). |
| 159 |
std::shared_ptr<const tooling::CompilationDatabase> |
| 160 |
get(const ThreadsafeFS &TFS, bool &ShouldBroadcast, |
| 161 |
stopwatch::time_point FreshTime, stopwatch::time_point FreshTimeMissing) { |
| 162 |
// Fast path for common case without taking lock. |
| 163 |
if (stopwatch::time_point(stopwatch::duration(NoCDBAt.load())) > |
| 164 |
FreshTimeMissing) { |
| 165 |
ShouldBroadcast = false; |
| 166 |
return nullptr; |
| 167 |
} |
| 168 |
|
| 169 |
std::lock_guard<std::mutex> Lock(Mu); |
| 170 |
auto RequestBroadcast = llvm::make_scope_exit([&, OldCDB(CDB.get())] { |
| 171 |
// If we loaded a new CDB, it should be broadcast at some point. |
| 172 |
if (CDB != nullptr && CDB.get() != OldCDB) |
| 173 |
NeedsBroadcast = true; |
| 174 |
else if (CDB == nullptr) // nothing to broadcast anymore! |
| 175 |
NeedsBroadcast = false; |
| 176 |
// If we have something to broadcast, then do so iff allowed. |
| 177 |
if (!ShouldBroadcast) |
| 178 |
return; |
| 179 |
ShouldBroadcast = NeedsBroadcast; |
| 180 |
NeedsBroadcast = false; |
| 181 |
}); |
| 182 |
|
| 183 |
// If our cache is valid, serve from it. |
| 184 |
if (CachePopulatedAt > FreshTime) |
| 185 |
return CDB; |
| 186 |
|
| 187 |
if (/*MayCache=*/load(*TFS.view(/*CWD=*/std::nullopt))) { |
| 188 |
// Use new timestamp, as loading may be slow. |
| 189 |
CachePopulatedAt = stopwatch::now(); |
| 190 |
NoCDBAt.store((CDB ? stopwatch::time_point::min() : CachePopulatedAt) |
| 191 |
.time_since_epoch() |
| 192 |
.count()); |
| 193 |
} |
| 194 |
|
| 195 |
return CDB; |
| 196 |
} |
| 197 |
|
| 198 |
private: |
| 199 |
// Updates `CDB` from disk state. Returns false on failure. |
| 200 |
bool load(llvm::vfs::FileSystem &FS); |
| 201 |
}; |
| 202 |
|
| 203 |
DirectoryBasedGlobalCompilationDatabase::DirectoryCache::CachedFile::LoadResult |
| 204 |
DirectoryBasedGlobalCompilationDatabase::DirectoryCache::CachedFile::load( |
| 205 |
llvm::vfs::FileSystem &FS, bool HasOldData) { |
| 206 |
auto Stat = FS.status(Path); |
| 207 |
if (!Stat || !Stat->isRegularFile()) { |
| 208 |
Size = NoFileCached; |
| 209 |
ContentHash = {}; |
| 210 |
return {LoadResult::FileNotFound, nullptr}; |
| 211 |
} |
| 212 |
// If both the size and mtime match, presume unchanged without reading. |
| 213 |
if (HasOldData && Stat->getLastModificationTime() == ModifiedTime && |
| 214 |
Stat->getSize() == Size) |
| 215 |
return {LoadResult::FoundSameData, nullptr}; |
| 216 |
auto Buf = FS.getBufferForFile(Path); |
| 217 |
if (!Buf || (*Buf)->getBufferSize() != Stat->getSize()) { |
| 218 |
// Don't clear the cache - possible we're seeing inconsistent size as the |
| 219 |
// file is being recreated. If it ends up identical later, great! |
| 220 |
// |
| 221 |
// This isn't a complete solution: if we see a partial file but stat/read |
| 222 |
// agree on its size, we're ultimately going to have spurious CDB reloads. |
| 223 |
// May be worth fixing if generators don't write atomically (CMake does). |
| 224 |
elog("Failed to read {0}: {1}", Path, |
| 225 |
Buf ? "size changed" : Buf.getError().message()); |
| 226 |
return {LoadResult::TransientError, nullptr}; |
| 227 |
} |
| 228 |
|
| 229 |
FileDigest NewContentHash = digest((*Buf)->getBuffer()); |
| 230 |
if (HasOldData && NewContentHash == ContentHash) { |
| 231 |
// mtime changed but data is the same: avoid rebuilding the CDB. |
| 232 |
ModifiedTime = Stat->getLastModificationTime(); |
| 233 |
return {LoadResult::FoundSameData, nullptr}; |
| 234 |
} |
| 235 |
|
| 236 |
Size = (*Buf)->getBufferSize(); |
| 237 |
ModifiedTime = Stat->getLastModificationTime(); |
| 238 |
ContentHash = NewContentHash; |
| 239 |
return {LoadResult::FoundNewData, std::move(*Buf)}; |
| 240 |
} |
| 241 |
|
| 242 |
// Adapt CDB-loading functions to a common interface for DirectoryCache::load(). |
| 243 |
static std::unique_ptr<tooling::CompilationDatabase> |
| 244 |
parseJSON(PathRef Path, llvm::StringRef Data, std::string &Error) { |
| 245 |
if (auto CDB = tooling::JSONCompilationDatabase::loadFromBuffer( |
| 246 |
Data, Error, tooling::JSONCommandLineSyntax::AutoDetect)) { |
| 247 |
// FS used for expanding response files. |
| 248 |
// FIXME: ExpandResponseFilesDatabase appears not to provide the usual |
| 249 |
// thread-safety guarantees, as the access to FS is not locked! |
| 250 |
// For now, use the real FS, which is known to be threadsafe (if we don't |
| 251 |
// use/change working directory, which ExpandResponseFilesDatabase doesn't). |
| 252 |
auto FS = llvm::vfs::getRealFileSystem(); |
| 253 |
return tooling::inferTargetAndDriverMode( |
| 254 |
tooling::inferMissingCompileCommands( |
| 255 |
expandResponseFiles(std::move(CDB), std::move(FS)))); |
| 256 |
} |
| 257 |
return nullptr; |
| 258 |
} |
| 259 |
static std::unique_ptr<tooling::CompilationDatabase> |
| 260 |
parseFixed(PathRef Path, llvm::StringRef Data, std::string &Error) { |
| 261 |
return tooling::FixedCompilationDatabase::loadFromBuffer( |
| 262 |
llvm::sys::path::parent_path(Path), Data, Error); |
| 263 |
} |
| 264 |
|
| 265 |
bool DirectoryBasedGlobalCompilationDatabase::DirectoryCache::load( |
| 266 |
llvm::vfs::FileSystem &FS) { |
| 267 |
dlog("Probing directory {0}", Path); |
| 268 |
std::string Error; |
| 269 |
|
| 270 |
// Load from the specially-supported compilation databases (JSON + Fixed). |
| 271 |
// For these, we know the files they read and cache their metadata so we can |
| 272 |
// cheaply validate whether they've changed, and hot-reload if they have. |
| 273 |
// (As a bonus, these are also VFS-clean)! |
| 274 |
struct CDBFile { |
| 275 |
CachedFile *File; |
| 276 |
// Wrapper for {Fixed,JSON}CompilationDatabase::loadFromBuffer. |
| 277 |
std::unique_ptr<tooling::CompilationDatabase> (*Parser)( |
| 278 |
PathRef, |
| 279 |
/*Data*/ llvm::StringRef, |
| 280 |
/*ErrorMsg*/ std::string &); |
| 281 |
}; |
| 282 |
for (const auto &Entry : {CDBFile{&CompileCommandsJson, parseJSON}, |
| 283 |
CDBFile{&BuildCompileCommandsJson, parseJSON}, |
| 284 |
CDBFile{&CompileFlagsTxt, parseFixed}}) { |
| 285 |
bool Active = ActiveCachedFile == Entry.File; |
| 286 |
auto Loaded = Entry.File->load(FS, Active); |
| 287 |
switch (Loaded.Result) { |
| 288 |
case CachedFile::LoadResult::FileNotFound: |
| 289 |
if (Active) { |
| 290 |
log("Unloaded compilation database from {0}", Entry.File->Path); |
| 291 |
ActiveCachedFile = nullptr; |
| 292 |
CDB = nullptr; |
| 293 |
} |
| 294 |
// Continue looking at other candidates. |
| 295 |
break; |
| 296 |
case CachedFile::LoadResult::TransientError: |
| 297 |
// File existed but we couldn't read it. Reuse the cache, retry later. |
| 298 |
return false; // Load again next time. |
| 299 |
case CachedFile::LoadResult::FoundSameData: |
| 300 |
assert(Active && "CachedFile may not return 'same data' if !HasOldData"); |
| 301 |
// This is the critical file, and it hasn't changed. |
| 302 |
return true; |
| 303 |
case CachedFile::LoadResult::FoundNewData: |
| 304 |
// We have a new CDB! |
| 305 |
CDB = Entry.Parser(Entry.File->Path, Loaded.Buffer->getBuffer(), Error); |
| 306 |
if (CDB) |
| 307 |
log("{0} compilation database from {1}", Active ? "Reloaded" : "Loaded", |
| 308 |
Entry.File->Path); |
| 309 |
else |
| 310 |
elog("Failed to load compilation database from {0}: {1}", |
| 311 |
Entry.File->Path, Error); |
| 312 |
ActiveCachedFile = Entry.File; |
| 313 |
return true; |
| 314 |
} |
| 315 |
} |
| 316 |
|
| 317 |
// Fall back to generic handling of compilation databases. |
| 318 |
// We don't know what files they read, so can't efficiently check whether |
| 319 |
// they need to be reloaded. So we never do that. |
| 320 |
// FIXME: the interface doesn't provide a way to virtualize FS access. |
| 321 |
|
| 322 |
// Don't try these more than once. If we've scanned before, we're done. |
| 323 |
if (CachePopulatedAt > stopwatch::time_point::min()) |
| 324 |
return true; |
| 325 |
for (const auto &Entry : |
| 326 |
tooling::CompilationDatabasePluginRegistry::entries()) { |
| 327 |
// Avoid duplicating the special cases handled above. |
| 328 |
if (Entry.getName() == "fixed-compilation-database" || |
| 329 |
Entry.getName() == "json-compilation-database") |
| 330 |
continue; |
| 331 |
auto Plugin = Entry.instantiate(); |
| 332 |
if (auto CDB = Plugin->loadFromDirectory(Path, Error)) { |
| 333 |
log("Loaded compilation database from {0} with plugin {1}", Path, |
| 334 |
Entry.getName()); |
| 335 |
this->CDB = std::move(CDB); |
| 336 |
return true; |
| 337 |
} |
| 338 |
// Don't log Error here, it's usually just "couldn't find <file>". |
| 339 |
} |
| 340 |
dlog("No compilation database at {0}", Path); |
| 341 |
return true; |
| 342 |
} |
| 343 |
|
| 344 |
DirectoryBasedGlobalCompilationDatabase:: |
| 345 |
DirectoryBasedGlobalCompilationDatabase(const Options &Opts) |
| 346 |
: Opts(Opts), Broadcaster(std::make_unique<BroadcastThread>(*this)) { |
| 347 |
if (!this->Opts.ContextProvider) |
| 348 |
this->Opts.ContextProvider = [](llvm::StringRef) { |
| 349 |
return Context::current().clone(); |
| 350 |
}; |
| 351 |
} |
| 352 |
|
| 353 |
DirectoryBasedGlobalCompilationDatabase:: |
| 354 |
~DirectoryBasedGlobalCompilationDatabase() = default; |
| 355 |
|
| 356 |
std::optional<tooling::CompileCommand> |
| 357 |
DirectoryBasedGlobalCompilationDatabase::getCompileCommand(PathRef File) const { |
| 358 |
CDBLookupRequest Req; |
| 359 |
Req.FileName = File; |
| 360 |
Req.ShouldBroadcast = true; |
| 361 |
auto Now = std::chrono::steady_clock::now(); |
| 362 |
Req.FreshTime = Now - Opts.RevalidateAfter; |
| 363 |
Req.FreshTimeMissing = Now - Opts.RevalidateMissingAfter; |
| 364 |
|
| 365 |
auto Res = lookupCDB(Req); |
| 366 |
if (!Res) { |
| 367 |
log("Failed to find compilation database for {0}", File); |
| 368 |
return std::nullopt; |
| 369 |
} |
| 370 |
|
| 371 |
auto Candidates = Res->CDB->getCompileCommands(File); |
| 372 |
if (!Candidates.empty()) |
| 373 |
return std::move(Candidates.front()); |
| 374 |
|
| 375 |
return std::nullopt; |
| 376 |
} |
| 377 |
|
| 378 |
std::vector<DirectoryBasedGlobalCompilationDatabase::DirectoryCache *> |
| 379 |
DirectoryBasedGlobalCompilationDatabase::getDirectoryCaches( |
| 380 |
llvm::ArrayRef<llvm::StringRef> Dirs) const { |
| 381 |
std::vector<std::string> FoldedDirs; |
| 382 |
FoldedDirs.reserve(Dirs.size()); |
| 383 |
for (const auto &Dir : Dirs) { |
| 384 |
#ifndef NDEBUG |
| 385 |
if (!llvm::sys::path::is_absolute(Dir)) |
| 386 |
elog("Trying to cache CDB for relative {0}"); |
| 387 |
#endif |
| 388 |
FoldedDirs.push_back(maybeCaseFoldPath(Dir)); |
| 389 |
} |
| 390 |
|
| 391 |
std::vector<DirectoryCache *> Ret; |
| 392 |
Ret.reserve(Dirs.size()); |
| 393 |
|
| 394 |
std::lock_guard<std::mutex> Lock(DirCachesMutex); |
| 395 |
for (unsigned I = 0; I < Dirs.size(); ++I) |
| 396 |
Ret.push_back(&DirCaches.try_emplace(FoldedDirs[I], Dirs[I]).first->second); |
| 397 |
return Ret; |
| 398 |
} |
| 399 |
|
| 400 |
std::optional<DirectoryBasedGlobalCompilationDatabase::CDBLookupResult> |
| 401 |
DirectoryBasedGlobalCompilationDatabase::lookupCDB( |
| 402 |
CDBLookupRequest Request) const { |
| 403 |
assert(llvm::sys::path::is_absolute(Request.FileName) && |
| 404 |
"path must be absolute"); |
| 405 |
|
| 406 |
std::string Storage; |
| 407 |
std::vector<llvm::StringRef> SearchDirs; |
| 408 |
if (Opts.CompileCommandsDir) // FIXME: unify this case with config. |
| 409 |
SearchDirs = {*Opts.CompileCommandsDir}; |
| 410 |
else { |
| 411 |
WithContext WithProvidedContext(Opts.ContextProvider(Request.FileName)); |
| 412 |
const auto &Spec = Config::current().CompileFlags.CDBSearch; |
| 413 |
switch (Spec.Policy) { |
| 414 |
case Config::CDBSearchSpec::NoCDBSearch: |
| 415 |
return std::nullopt; |
| 416 |
case Config::CDBSearchSpec::FixedDir: |
| 417 |
Storage = *Spec.FixedCDBPath; |
| 418 |
SearchDirs = {Storage}; |
| 419 |
break; |
| 420 |
case Config::CDBSearchSpec::Ancestors: |
| 421 |
// Traverse the canonical version to prevent false positives. i.e.: |
| 422 |
// src/build/../a.cc can detect a CDB in /src/build if not |
| 423 |
// canonicalized. |
| 424 |
Storage = removeDots(Request.FileName); |
| 425 |
actOnAllParentDirectories(Storage, [&](llvm::StringRef Dir) { |
| 426 |
SearchDirs.push_back(Dir); |
| 427 |
return false; |
| 428 |
}); |
| 429 |
} |
| 430 |
} |
| 431 |
|
| 432 |
std::shared_ptr<const tooling::CompilationDatabase> CDB = nullptr; |
| 433 |
bool ShouldBroadcast = false; |
| 434 |
DirectoryCache *DirCache = nullptr; |
| 435 |
for (DirectoryCache *Candidate : getDirectoryCaches(SearchDirs)) { |
| 436 |
bool CandidateShouldBroadcast = Request.ShouldBroadcast; |
| 437 |
if ((CDB = Candidate->get(Opts.TFS, CandidateShouldBroadcast, |
| 438 |
Request.FreshTime, Request.FreshTimeMissing))) { |
| 439 |
DirCache = Candidate; |
| 440 |
ShouldBroadcast = CandidateShouldBroadcast; |
| 441 |
break; |
| 442 |
} |
| 443 |
} |
| 444 |
|
| 445 |
if (!CDB) |
| 446 |
return std::nullopt; |
| 447 |
|
| 448 |
CDBLookupResult Result; |
| 449 |
Result.CDB = std::move(CDB); |
| 450 |
Result.PI.SourceRoot = DirCache->Path; |
| 451 |
|
| 452 |
if (ShouldBroadcast) |
| 453 |
broadcastCDB(Result); |
| 454 |
return Result; |
| 455 |
} |
| 456 |
|
| 457 |
// The broadcast thread announces files with new compile commands to the world. |
| 458 |
// Primarily this is used to enqueue them for background indexing. |
| 459 |
// |
| 460 |
// It's on a separate thread because: |
| 461 |
// - otherwise it would block the first parse of the initial file |
| 462 |
// - we need to enumerate all files in the CDB, of which there are many |
| 463 |
// - we (will) have to evaluate config for every file in the CDB, which is slow |
| 464 |
class DirectoryBasedGlobalCompilationDatabase::BroadcastThread { |
| 465 |
class Filter; |
| 466 |
DirectoryBasedGlobalCompilationDatabase &Parent; |
| 467 |
|
| 468 |
std::mutex Mu; |
| 469 |
std::condition_variable CV; |
| 470 |
// Shutdown flag (CV is notified after writing). |
| 471 |
// This is atomic so that broadcasts can also observe it and abort early. |
| 472 |
std::atomic<bool> ShouldStop = {false}; |
| 473 |
struct Task { |
| 474 |
CDBLookupResult Lookup; |
| 475 |
Context Ctx; |
| 476 |
}; |
| 477 |
std::deque<Task> Queue; |
| 478 |
std::optional<Task> ActiveTask; |
| 479 |
std::thread Thread; // Must be last member. |
| 480 |
|
| 481 |
// Thread body: this is just the basic queue procesing boilerplate. |
| 482 |
void run() { |
| 483 |
std::unique_lock<std::mutex> Lock(Mu); |
| 484 |
while (true) { |
| 485 |
bool Stopping = false; |
| 486 |
CV.wait(Lock, [&] { |
| 487 |
return (Stopping = ShouldStop.load(std::memory_order_acquire)) || |
| 488 |
!Queue.empty(); |
| 489 |
}); |
| 490 |
if (Stopping) { |
| 491 |
Queue.clear(); |
| 492 |
CV.notify_all(); |
| 493 |
return; |
| 494 |
} |
| 495 |
ActiveTask = std::move(Queue.front()); |
| 496 |
Queue.pop_front(); |
| 497 |
|
| 498 |
Lock.unlock(); |
| 499 |
{ |
| 500 |
WithContext WithCtx(std::move(ActiveTask->Ctx)); |
| 501 |
process(ActiveTask->Lookup); |
| 502 |
} |
| 503 |
Lock.lock(); |
| 504 |
ActiveTask.reset(); |
| 505 |
CV.notify_all(); |
| 506 |
} |
| 507 |
} |
| 508 |
|
| 509 |
// Inspects a new CDB and broadcasts the files it owns. |
| 510 |
void process(const CDBLookupResult &T); |
| 511 |
|
| 512 |
public: |
| 513 |
BroadcastThread(DirectoryBasedGlobalCompilationDatabase &Parent) |
| 514 |
: Parent(Parent), Thread([this] { run(); }) {} |
| 515 |
|
| 516 |
void enqueue(CDBLookupResult Lookup) { |
| 517 |
{ |
| 518 |
assert(!Lookup.PI.SourceRoot.empty()); |
| 519 |
std::lock_guard<std::mutex> Lock(Mu); |
| 520 |
// New CDB takes precedence over any queued one for the same directory. |
| 521 |
llvm::erase_if(Queue, [&](const Task &T) { |
| 522 |
return T.Lookup.PI.SourceRoot == Lookup.PI.SourceRoot; |
| 523 |
}); |
| 524 |
Queue.push_back({std::move(Lookup), Context::current().clone()}); |
| 525 |
} |
| 526 |
CV.notify_all(); |
| 527 |
} |
| 528 |
|
| 529 |
bool blockUntilIdle(Deadline Timeout) { |
| 530 |
std::unique_lock<std::mutex> Lock(Mu); |
| 531 |
return wait(Lock, CV, Timeout, |
| 532 |
[&] { return Queue.empty() && !ActiveTask; }); |
| 533 |
} |
| 534 |
|
| 535 |
~BroadcastThread() { |
| 536 |
{ |
| 537 |
std::lock_guard<std::mutex> Lock(Mu); |
| 538 |
ShouldStop.store(true, std::memory_order_release); |
| 539 |
} |
| 540 |
CV.notify_all(); |
| 541 |
Thread.join(); |
| 542 |
} |
| 543 |
}; |
| 544 |
|
| 545 |
// The DirBasedCDB associates each file with a specific CDB. |
| 546 |
// When a CDB is discovered, it may claim to describe files that we associate |
| 547 |
// with a different CDB. We do not want to broadcast discovery of these, and |
| 548 |
// trigger background indexing of them. |
| 549 |
// |
| 550 |
// We must filter the list, and check whether they are associated with this CDB. |
| 551 |
// This class attempts to do so efficiently. |
| 552 |
// |
| 553 |
// Roughly, it: |
| 554 |
// - loads the config for each file, and determines the relevant search path |
| 555 |
// - gathers all directories that are part of any search path |
| 556 |
// - (lazily) checks for a CDB in each such directory at most once |
| 557 |
// - walks the search path for each file and determines whether to include it. |
| 558 |
class DirectoryBasedGlobalCompilationDatabase::BroadcastThread::Filter { |
| 559 |
llvm::StringRef ThisDir; |
| 560 |
DirectoryBasedGlobalCompilationDatabase &Parent; |
| 561 |
|
| 562 |
// Keep track of all directories we might check for CDBs. |
| 563 |
struct DirInfo { |
| 564 |
DirectoryCache *Cache = nullptr; |
| 565 |
enum { Unknown, Missing, TargetCDB, OtherCDB } State = Unknown; |
| 566 |
DirInfo *Parent = nullptr; |
| 567 |
}; |
| 568 |
llvm::StringMap<DirInfo> Dirs; |
| 569 |
|
| 570 |
// A search path starts at a directory, and either includes ancestors or not. |
| 571 |
using SearchPath = llvm::PointerIntPair<DirInfo *, 1>; |
| 572 |
|
| 573 |
// Add all ancestor directories of FilePath to the tracked set. |
| 574 |
// Returns the immediate parent of the file. |
| 575 |
DirInfo *addParents(llvm::StringRef FilePath) { |
| 576 |
DirInfo *Leaf = nullptr; |
| 577 |
DirInfo *Child = nullptr; |
| 578 |
actOnAllParentDirectories(FilePath, [&](llvm::StringRef Dir) { |
| 579 |
auto &Info = Dirs[Dir]; |
| 580 |
// If this is the first iteration, then this node is the overall result. |
| 581 |
if (!Leaf) |
| 582 |
Leaf = &Info; |
| 583 |
// Fill in the parent link from the previous iteration to this parent. |
| 584 |
if (Child) |
| 585 |
Child->Parent = &Info; |
| 586 |
// Keep walking, whether we inserted or not, if parent link is missing. |
| 587 |
// (If it's present, parent links must be present up to the root, so stop) |
| 588 |
Child = &Info; |
| 589 |
return Info.Parent != nullptr; |
| 590 |
}); |
| 591 |
return Leaf; |
| 592 |
} |
| 593 |
|
| 594 |
// Populates DirInfo::Cache (and State, if it is TargetCDB). |
| 595 |
void grabCaches() { |
| 596 |
// Fast path out if there were no files, or CDB loading is off. |
| 597 |
if (Dirs.empty()) |
| 598 |
return; |
| 599 |
|
| 600 |
std::vector<llvm::StringRef> DirKeys; |
| 601 |
std::vector<DirInfo *> DirValues; |
| 602 |
DirKeys.reserve(Dirs.size() + 1); |
| 603 |
DirValues.reserve(Dirs.size()); |
| 604 |
for (auto &E : Dirs) { |
| 605 |
DirKeys.push_back(E.first()); |
| 606 |
DirValues.push_back(&E.second); |
| 607 |
} |
| 608 |
|
| 609 |
// Also look up the cache entry for the CDB we're broadcasting. |
| 610 |
// Comparing DirectoryCache pointers is more robust than checking string |
| 611 |
// equality, e.g. reuses the case-sensitivity handling. |
| 612 |
DirKeys.push_back(ThisDir); |
| 613 |
auto DirCaches = Parent.getDirectoryCaches(DirKeys); |
| 614 |
const DirectoryCache *ThisCache = DirCaches.back(); |
| 615 |
DirCaches.pop_back(); |
| 616 |
DirKeys.pop_back(); |
| 617 |
|
| 618 |
for (unsigned I = 0; I < DirKeys.size(); ++I) { |
| 619 |
DirValues[I]->Cache = DirCaches[I]; |
| 620 |
if (DirCaches[I] == ThisCache) |
| 621 |
DirValues[I]->State = DirInfo::TargetCDB; |
| 622 |
} |
| 623 |
} |
| 624 |
|
| 625 |
// Should we include a file from this search path? |
| 626 |
bool shouldInclude(SearchPath P) { |
| 627 |
DirInfo *Info = P.getPointer(); |
| 628 |
if (!Info) |
| 629 |
return false; |
| 630 |
if (Info->State == DirInfo::Unknown) { |
| 631 |
assert(Info->Cache && "grabCaches() should have filled this"); |
| 632 |
// Given that we know that CDBs have been moved/generated, don't trust |
| 633 |
// caches. (This should be rare, so it's OK to add a little latency). |
| 634 |
constexpr auto IgnoreCache = std::chrono::steady_clock::time_point::max(); |
| 635 |
// Don't broadcast CDBs discovered while broadcasting! |
| 636 |
bool ShouldBroadcast = false; |
| 637 |
bool Exists = |
| 638 |
nullptr != Info->Cache->get(Parent.Opts.TFS, ShouldBroadcast, |
| 639 |
/*FreshTime=*/IgnoreCache, |
| 640 |
/*FreshTimeMissing=*/IgnoreCache); |
| 641 |
Info->State = Exists ? DirInfo::OtherCDB : DirInfo::Missing; |
| 642 |
} |
| 643 |
// If we have a CDB, include the file if it's the target CDB only. |
| 644 |
if (Info->State != DirInfo::Missing) |
| 645 |
return Info->State == DirInfo::TargetCDB; |
| 646 |
// If we have no CDB and no relevant parent, don't include the file. |
| 647 |
if (!P.getInt() || !Info->Parent) |
| 648 |
return false; |
| 649 |
// Walk up to the next parent. |
| 650 |
return shouldInclude(SearchPath(Info->Parent, 1)); |
| 651 |
} |
| 652 |
|
| 653 |
public: |
| 654 |
Filter(llvm::StringRef ThisDir, |
| 655 |
DirectoryBasedGlobalCompilationDatabase &Parent) |
| 656 |
: ThisDir(ThisDir), Parent(Parent) {} |
| 657 |
|
| 658 |
std::vector<std::string> filter(std::vector<std::string> AllFiles, |
| 659 |
std::atomic<bool> &ShouldStop) { |
| 660 |
std::vector<std::string> Filtered; |
| 661 |
// Allow for clean early-exit of the slow parts. |
| 662 |
auto ExitEarly = [&] { |
| 663 |
if (ShouldStop.load(std::memory_order_acquire)) { |
| 664 |
log("Giving up on broadcasting CDB, as we're shutting down"); |
| 665 |
Filtered.clear(); |
| 666 |
return true; |
| 667 |
} |
| 668 |
return false; |
| 669 |
}; |
| 670 |
// Compute search path for each file. |
| 671 |
std::vector<SearchPath> SearchPaths(AllFiles.size()); |
| 672 |
for (unsigned I = 0; I < AllFiles.size(); ++I) { |
| 673 |
if (Parent.Opts.CompileCommandsDir) { // FIXME: unify with config |
| 674 |
SearchPaths[I].setPointer(&Dirs[*Parent.Opts.CompileCommandsDir]); |
| 675 |
continue; |
| 676 |
} |
| 677 |
if (ExitEarly()) // loading config may be slow |
| 678 |
return Filtered; |
| 679 |
WithContext WithProvidedContent(Parent.Opts.ContextProvider(AllFiles[I])); |
| 680 |
const Config::CDBSearchSpec &Spec = |
| 681 |
Config::current().CompileFlags.CDBSearch; |
| 682 |
switch (Spec.Policy) { |
| 683 |
case Config::CDBSearchSpec::NoCDBSearch: |
| 684 |
break; |
| 685 |
case Config::CDBSearchSpec::Ancestors: |
| 686 |
SearchPaths[I].setInt(/*Recursive=*/1); |
| 687 |
SearchPaths[I].setPointer(addParents(AllFiles[I])); |
| 688 |
break; |
| 689 |
case Config::CDBSearchSpec::FixedDir: |
| 690 |
SearchPaths[I].setPointer(&Dirs[*Spec.FixedCDBPath]); |
| 691 |
break; |
| 692 |
} |
| 693 |
} |
| 694 |
// Get the CDB cache for each dir on the search path, but don't load yet. |
| 695 |
grabCaches(); |
| 696 |
// Now work out which files we want to keep, loading CDBs where needed. |
| 697 |
for (unsigned I = 0; I < AllFiles.size(); ++I) { |
| 698 |
if (ExitEarly()) // loading CDBs may be slow |
| 699 |
return Filtered; |
| 700 |
if (shouldInclude(SearchPaths[I])) |
| 701 |
Filtered.push_back(std::move(AllFiles[I])); |
| 702 |
} |
| 703 |
return Filtered; |
| 704 |
} |
| 705 |
}; |
| 706 |
|
| 707 |
void DirectoryBasedGlobalCompilationDatabase::BroadcastThread::process( |
| 708 |
const CDBLookupResult &T) { |
| 709 |
vlog("Broadcasting compilation database from {0}", T.PI.SourceRoot); |
| 710 |
std::vector<std::string> GovernedFiles = |
| 711 |
Filter(T.PI.SourceRoot, Parent).filter(T.CDB->getAllFiles(), ShouldStop); |
| 712 |
if (!GovernedFiles.empty()) |
| 713 |
Parent.OnCommandChanged.broadcast(std::move(GovernedFiles)); |
| 714 |
} |
| 715 |
|
| 716 |
void DirectoryBasedGlobalCompilationDatabase::broadcastCDB( |
| 717 |
CDBLookupResult Result) const { |
| 718 |
assert(Result.CDB && "Trying to broadcast an invalid CDB!"); |
| 719 |
Broadcaster->enqueue(Result); |
| 720 |
} |
| 721 |
|
| 722 |
bool DirectoryBasedGlobalCompilationDatabase::blockUntilIdle( |
| 723 |
Deadline Timeout) const { |
| 724 |
return Broadcaster->blockUntilIdle(Timeout); |
| 725 |
} |
| 726 |
|
| 727 |
std::optional<ProjectInfo> |
| 728 |
DirectoryBasedGlobalCompilationDatabase::getProjectInfo(PathRef File) const { |
| 729 |
CDBLookupRequest Req; |
| 730 |
Req.FileName = File; |
| 731 |
Req.ShouldBroadcast = false; |
| 732 |
Req.FreshTime = Req.FreshTimeMissing = |
| 733 |
std::chrono::steady_clock::time_point::min(); |
| 734 |
auto Res = lookupCDB(Req); |
| 735 |
if (!Res) |
| 736 |
return std::nullopt; |
| 737 |
return Res->PI; |
| 738 |
} |
| 739 |
|
| 740 |
OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base, |
| 741 |
std::vector<std::string> FallbackFlags, |
| 742 |
CommandMangler Mangler) |
| 743 |
: DelegatingCDB(Base), Mangler(std::move(Mangler)), |
| 744 |
FallbackFlags(std::move(FallbackFlags)) {} |
| 745 |
|
| 746 |
std::optional<tooling::CompileCommand> |
| 747 |
OverlayCDB::getCompileCommand(PathRef File) const { |
| 748 |
std::optional<tooling::CompileCommand> Cmd; |
| 749 |
{ |
| 750 |
std::lock_guard<std::mutex> Lock(Mutex); |
| 751 |
auto It = Commands.find(removeDots(File)); |
| 752 |
if (It != Commands.end()) |
| 753 |
Cmd = It->second; |
| 754 |
} |
| 755 |
if (!Cmd) |
| 756 |
Cmd = DelegatingCDB::getCompileCommand(File); |
| 757 |
if (!Cmd) |
| 758 |
return std::nullopt; |
| 759 |
if (Mangler) |
| 760 |
Mangler(*Cmd, File); |
| 761 |
return Cmd; |
| 762 |
} |
| 763 |
|
| 764 |
tooling::CompileCommand OverlayCDB::getFallbackCommand(PathRef File) const { |
| 765 |
auto Cmd = DelegatingCDB::getFallbackCommand(File); |
| 766 |
std::lock_guard<std::mutex> Lock(Mutex); |
| 767 |
Cmd.CommandLine.insert(Cmd.CommandLine.end(), FallbackFlags.begin(), |
| 768 |
FallbackFlags.end()); |
| 769 |
if (Mangler) |
| 770 |
Mangler(Cmd, File); |
| 771 |
return Cmd; |
| 772 |
} |
| 773 |
|
| 774 |
void OverlayCDB::setCompileCommand(PathRef File, |
| 775 |
std::optional<tooling::CompileCommand> Cmd) { |
| 776 |
// We store a canonical version internally to prevent mismatches between set |
| 777 |
// and get compile commands. Also it assures clients listening to broadcasts |
| 778 |
// doesn't receive different names for the same file. |
| 779 |
std::string CanonPath = removeDots(File); |
| 780 |
{ |
| 781 |
std::unique_lock<std::mutex> Lock(Mutex); |
| 782 |
if (Cmd) |
| 783 |
Commands[CanonPath] = std::move(*Cmd); |
| 784 |
else |
| 785 |
Commands.erase(CanonPath); |
| 786 |
} |
| 787 |
OnCommandChanged.broadcast({CanonPath}); |
| 788 |
} |
| 789 |
|
| 790 |
DelegatingCDB::DelegatingCDB(const GlobalCompilationDatabase *Base) |
| 791 |
: Base(Base) { |
| 792 |
if (Base) |
| 793 |
BaseChanged = Base->watch([this](const std::vector<std::string> Changes) { |
| 794 |
OnCommandChanged.broadcast(Changes); |
| 795 |
}); |
| 796 |
} |
| 797 |
|
| 798 |
DelegatingCDB::DelegatingCDB(std::unique_ptr<GlobalCompilationDatabase> Base) |
| 799 |
: DelegatingCDB(Base.get()) { |
| 800 |
BaseOwner = std::move(Base); |
| 801 |
} |
| 802 |
|
| 803 |
std::optional<tooling::CompileCommand> |
| 804 |
DelegatingCDB::getCompileCommand(PathRef File) const { |
| 805 |
if (!Base) |
| 806 |
return std::nullopt; |
| 807 |
return Base->getCompileCommand(File); |
| 808 |
} |
| 809 |
|
| 810 |
std::optional<ProjectInfo> DelegatingCDB::getProjectInfo(PathRef File) const { |
| 811 |
if (!Base) |
| 812 |
return std::nullopt; |
| 813 |
return Base->getProjectInfo(File); |
| 814 |
} |
| 815 |
|
| 816 |
tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const { |
| 817 |
if (!Base) |
| 818 |
return GlobalCompilationDatabase::getFallbackCommand(File); |
| 819 |
return Base->getFallbackCommand(File); |
| 820 |
} |
| 821 |
|
| 822 |
bool DelegatingCDB::blockUntilIdle(Deadline D) const { |
| 823 |
if (!Base) |
| 824 |
return true; |
| 825 |
return Base->blockUntilIdle(D); |
| 826 |
} |
| 827 |
|
| 828 |
} // namespace clangd |
| 829 |
} // namespace clang |