| 1 |
//===--- Preamble.cpp - Reusing expensive parts of the AST ----------------===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
|
| 9 |
#include "Preamble.h" |
| 10 |
#include "Compiler.h" |
| 11 |
#include "Config.h" |
| 12 |
#include "Headers.h" |
| 13 |
#include "SourceCode.h" |
| 14 |
#include "clang-include-cleaner/Record.h" |
| 15 |
#include "support/Logger.h" |
| 16 |
#include "support/ThreadsafeFS.h" |
| 17 |
#include "support/Trace.h" |
| 18 |
#include "clang/AST/DeclTemplate.h" |
| 19 |
#include "clang/Basic/Diagnostic.h" |
| 20 |
#include "clang/Basic/DiagnosticLex.h" |
| 21 |
#include "clang/Basic/LangOptions.h" |
| 22 |
#include "clang/Basic/SourceLocation.h" |
| 23 |
#include "clang/Basic/SourceManager.h" |
| 24 |
#include "clang/Basic/TokenKinds.h" |
| 25 |
#include "clang/Frontend/CompilerInvocation.h" |
| 26 |
#include "clang/Frontend/FrontendActions.h" |
| 27 |
#include "clang/Lex/HeaderSearch.h" |
| 28 |
#include "clang/Lex/Lexer.h" |
| 29 |
#include "clang/Lex/PPCallbacks.h" |
| 30 |
#include "clang/Lex/Preprocessor.h" |
| 31 |
#include "clang/Lex/PreprocessorOptions.h" |
| 32 |
#include "clang/Tooling/CompilationDatabase.h" |
| 33 |
#include "llvm/ADT/ArrayRef.h" |
| 34 |
#include "llvm/ADT/DenseMap.h" |
| 35 |
#include "llvm/ADT/IntrusiveRefCntPtr.h" |
| 36 |
#include "llvm/ADT/STLExtras.h" |
| 37 |
#include "llvm/ADT/SmallString.h" |
| 38 |
#include "llvm/ADT/StringRef.h" |
| 39 |
#include "llvm/Support/Error.h" |
| 40 |
#include "llvm/Support/ErrorHandling.h" |
| 41 |
#include "llvm/Support/FormatVariadic.h" |
| 42 |
#include "llvm/Support/MemoryBuffer.h" |
| 43 |
#include "llvm/Support/Path.h" |
| 44 |
#include "llvm/Support/VirtualFileSystem.h" |
| 45 |
#include "llvm/Support/raw_ostream.h" |
| 46 |
#include <iterator> |
| 47 |
#include <memory> |
| 48 |
#include <string> |
| 49 |
#include <system_error> |
| 50 |
#include <utility> |
| 51 |
#include <vector> |
| 52 |
|
| 53 |
namespace clang { |
| 54 |
namespace clangd { |
| 55 |
namespace { |
| 56 |
constexpr llvm::StringLiteral PreamblePatchHeaderName = "__preamble_patch__.h"; |
| 57 |
|
| 58 |
bool compileCommandsAreEqual(const tooling::CompileCommand &LHS, |
| 59 |
const tooling::CompileCommand &RHS) { |
| 60 |
// We don't check for Output, it should not matter to clangd. |
| 61 |
return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename && |
| 62 |
llvm::ArrayRef(LHS.CommandLine).equals(RHS.CommandLine); |
| 63 |
} |
| 64 |
|
| 65 |
class CppFilePreambleCallbacks : public PreambleCallbacks { |
| 66 |
public: |
| 67 |
CppFilePreambleCallbacks( |
| 68 |
PathRef File, PreambleParsedCallback ParsedCallback, |
| 69 |
PreambleBuildStats *Stats, bool ParseForwardingFunctions, |
| 70 |
std::function<void(CompilerInstance &)> BeforeExecuteCallback) |
| 71 |
: File(File), ParsedCallback(ParsedCallback), Stats(Stats), |
| 72 |
ParseForwardingFunctions(ParseForwardingFunctions), |
| 73 |
BeforeExecuteCallback(std::move(BeforeExecuteCallback)) {} |
| 74 |
|
| 75 |
IncludeStructure takeIncludes() { return std::move(Includes); } |
| 76 |
|
| 77 |
MainFileMacros takeMacros() { return std::move(Macros); } |
| 78 |
|
| 79 |
std::vector<PragmaMark> takeMarks() { return std::move(Marks); } |
| 80 |
|
| 81 |
include_cleaner::PragmaIncludes takePragmaIncludes() { |
| 82 |
return std::move(Pragmas); |
| 83 |
} |
| 84 |
CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); } |
| 85 |
|
| 86 |
bool isMainFileIncludeGuarded() const { return IsMainFileIncludeGuarded; } |
| 87 |
|
| 88 |
void AfterExecute(CompilerInstance &CI) override { |
| 89 |
if (ParsedCallback) { |
| 90 |
trace::Span Tracer("Running PreambleCallback"); |
| 91 |
ParsedCallback(CI.getASTContext(), CI.getPreprocessor(), CanonIncludes); |
| 92 |
} |
| 93 |
|
| 94 |
const SourceManager &SM = CI.getSourceManager(); |
| 95 |
const FileEntry *MainFE = SM.getFileEntryForID(SM.getMainFileID()); |
| 96 |
IsMainFileIncludeGuarded = |
| 97 |
CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded( |
| 98 |
MainFE); |
| 99 |
|
| 100 |
if (Stats) { |
| 101 |
const ASTContext &AST = CI.getASTContext(); |
| 102 |
Stats->BuildSize = AST.getASTAllocatedMemory(); |
| 103 |
Stats->BuildSize += AST.getSideTableAllocatedMemory(); |
| 104 |
Stats->BuildSize += AST.Idents.getAllocator().getTotalMemory(); |
| 105 |
Stats->BuildSize += AST.Selectors.getTotalMemory(); |
| 106 |
|
| 107 |
Stats->BuildSize += AST.getSourceManager().getContentCacheSize(); |
| 108 |
Stats->BuildSize += AST.getSourceManager().getDataStructureSizes(); |
| 109 |
Stats->BuildSize += |
| 110 |
AST.getSourceManager().getMemoryBufferSizes().malloc_bytes; |
| 111 |
|
| 112 |
const Preprocessor &PP = CI.getPreprocessor(); |
| 113 |
Stats->BuildSize += PP.getTotalMemory(); |
| 114 |
if (PreprocessingRecord *PRec = PP.getPreprocessingRecord()) |
| 115 |
Stats->BuildSize += PRec->getTotalMemory(); |
| 116 |
Stats->BuildSize += PP.getHeaderSearchInfo().getTotalMemory(); |
| 117 |
} |
| 118 |
} |
| 119 |
|
| 120 |
void BeforeExecute(CompilerInstance &CI) override { |
| 121 |
CanonIncludes.addSystemHeadersMapping(CI.getLangOpts()); |
| 122 |
LangOpts = &CI.getLangOpts(); |
| 123 |
SourceMgr = &CI.getSourceManager(); |
| 124 |
Includes.collect(CI); |
| 125 |
if (Config::current().Diagnostics.UnusedIncludes == |
| 126 |
Config::UnusedIncludesPolicy::Experiment) |
| 127 |
Pragmas.record(CI); |
| 128 |
if (BeforeExecuteCallback) |
| 129 |
BeforeExecuteCallback(CI); |
| 130 |
} |
| 131 |
|
| 132 |
std::unique_ptr<PPCallbacks> createPPCallbacks() override { |
| 133 |
assert(SourceMgr && LangOpts && |
| 134 |
"SourceMgr and LangOpts must be set at this point"); |
| 135 |
|
| 136 |
return std::make_unique<PPChainedCallbacks>( |
| 137 |
std::make_unique<CollectMainFileMacros>(*SourceMgr, Macros), |
| 138 |
collectPragmaMarksCallback(*SourceMgr, Marks)); |
| 139 |
} |
| 140 |
|
| 141 |
CommentHandler *getCommentHandler() override { |
| 142 |
IWYUHandler = collectIWYUHeaderMaps(&CanonIncludes); |
| 143 |
return IWYUHandler.get(); |
| 144 |
} |
| 145 |
|
| 146 |
static bool isLikelyForwardingFunction(FunctionTemplateDecl *FT) { |
| 147 |
const auto *FD = FT->getTemplatedDecl(); |
| 148 |
const auto NumParams = FD->getNumParams(); |
| 149 |
// Check whether its last parameter is a parameter pack... |
| 150 |
if (NumParams > 0) { |
| 151 |
const auto *LastParam = FD->getParamDecl(NumParams - 1); |
| 152 |
if (const auto *PET = dyn_cast<PackExpansionType>(LastParam->getType())) { |
| 153 |
// ... of the type T&&... or T... |
| 154 |
const auto BaseType = PET->getPattern().getNonReferenceType(); |
| 155 |
if (const auto *TTPT = |
| 156 |
dyn_cast<TemplateTypeParmType>(BaseType.getTypePtr())) { |
| 157 |
// ... whose template parameter comes from the function directly |
| 158 |
if (FT->getTemplateParameters()->getDepth() == TTPT->getDepth()) { |
| 159 |
return true; |
| 160 |
} |
| 161 |
} |
| 162 |
} |
| 163 |
} |
| 164 |
return false; |
| 165 |
} |
| 166 |
|
| 167 |
bool shouldSkipFunctionBody(Decl *D) override { |
| 168 |
// Usually we don't need to look inside the bodies of header functions |
| 169 |
// to understand the program. However when forwarding function like |
| 170 |
// emplace() forward their arguments to some other function, the |
| 171 |
// interesting overload resolution happens inside the forwarding |
| 172 |
// function's body. To provide more meaningful diagnostics, |
| 173 |
// code completion, and parameter hints we should parse (and later |
| 174 |
// instantiate) the bodies. |
| 175 |
if (auto *FT = llvm::dyn_cast<clang::FunctionTemplateDecl>(D)) { |
| 176 |
if (ParseForwardingFunctions) { |
| 177 |
// Don't skip parsing the body if it looks like a forwarding function |
| 178 |
if (isLikelyForwardingFunction(FT)) |
| 179 |
return false; |
| 180 |
} else { |
| 181 |
// By default, only take care of make_unique |
| 182 |
// std::make_unique is trivial, and we diagnose bad constructor calls. |
| 183 |
if (const auto *II = FT->getDeclName().getAsIdentifierInfo()) { |
| 184 |
if (II->isStr("make_unique") && FT->isInStdNamespace()) |
| 185 |
return false; |
| 186 |
} |
| 187 |
} |
| 188 |
} |
| 189 |
return true; |
| 190 |
} |
| 191 |
|
| 192 |
private: |
| 193 |
PathRef File; |
| 194 |
PreambleParsedCallback ParsedCallback; |
| 195 |
IncludeStructure Includes; |
| 196 |
CanonicalIncludes CanonIncludes; |
| 197 |
include_cleaner::PragmaIncludes Pragmas; |
| 198 |
MainFileMacros Macros; |
| 199 |
std::vector<PragmaMark> Marks; |
| 200 |
bool IsMainFileIncludeGuarded = false; |
| 201 |
std::unique_ptr<CommentHandler> IWYUHandler = nullptr; |
| 202 |
const clang::LangOptions *LangOpts = nullptr; |
| 203 |
const SourceManager *SourceMgr = nullptr; |
| 204 |
PreambleBuildStats *Stats; |
| 205 |
bool ParseForwardingFunctions; |
| 206 |
std::function<void(CompilerInstance &)> BeforeExecuteCallback; |
| 207 |
}; |
| 208 |
|
| 209 |
// Represents directives other than includes, where basic textual information is |
| 210 |
// enough. |
| 211 |
struct TextualPPDirective { |
| 212 |
unsigned DirectiveLine; |
| 213 |
// Full text that's representing the directive, including the `#`. |
| 214 |
std::string Text; |
| 215 |
unsigned Offset; |
| 216 |
|
| 217 |
bool operator==(const TextualPPDirective &RHS) const { |
| 218 |
return std::tie(DirectiveLine, Offset, Text) == |
| 219 |
std::tie(RHS.DirectiveLine, RHS.Offset, RHS.Text); |
| 220 |
} |
| 221 |
}; |
| 222 |
|
| 223 |
// Formats a PP directive consisting of Prefix (e.g. "#define ") and Body ("X |
| 224 |
// 10"). The formatting is copied so that the tokens in Body have PresumedLocs |
| 225 |
// with correct columns and lines. |
| 226 |
std::string spellDirective(llvm::StringRef Prefix, |
| 227 |
CharSourceRange DirectiveRange, |
| 228 |
const LangOptions &LangOpts, const SourceManager &SM, |
| 229 |
unsigned &DirectiveLine, unsigned &Offset) { |
| 230 |
std::string SpelledDirective; |
| 231 |
llvm::raw_string_ostream OS(SpelledDirective); |
| 232 |
OS << Prefix; |
| 233 |
|
| 234 |
// Make sure DirectiveRange is a char range and doesn't contain macro ids. |
| 235 |
DirectiveRange = SM.getExpansionRange(DirectiveRange); |
| 236 |
if (DirectiveRange.isTokenRange()) { |
| 237 |
DirectiveRange.setEnd( |
| 238 |
Lexer::getLocForEndOfToken(DirectiveRange.getEnd(), 0, SM, LangOpts)); |
| 239 |
} |
| 240 |
|
| 241 |
auto DecompLoc = SM.getDecomposedLoc(DirectiveRange.getBegin()); |
| 242 |
DirectiveLine = SM.getLineNumber(DecompLoc.first, DecompLoc.second); |
| 243 |
Offset = DecompLoc.second; |
| 244 |
auto TargetColumn = SM.getColumnNumber(DecompLoc.first, DecompLoc.second) - 1; |
| 245 |
|
| 246 |
// Pad with spaces before DirectiveRange to make sure it will be on right |
| 247 |
// column when patched. |
| 248 |
if (Prefix.size() <= TargetColumn) { |
| 249 |
// There is enough space for Prefix and space before directive, use it. |
| 250 |
// We try to squeeze the Prefix into the same line whenever we can, as |
| 251 |
// putting onto a separate line won't work at the beginning of the file. |
| 252 |
OS << std::string(TargetColumn - Prefix.size(), ' '); |
| 253 |
} else { |
| 254 |
// Prefix was longer than the space we had. We produce e.g.: |
| 255 |
// #line N-1 |
| 256 |
// #define \ |
| 257 |
// X 10 |
| 258 |
OS << "\\\n" << std::string(TargetColumn, ' '); |
| 259 |
// Decrement because we put an additional line break before |
| 260 |
// DirectiveRange.begin(). |
| 261 |
--DirectiveLine; |
| 262 |
} |
| 263 |
OS << toSourceCode(SM, DirectiveRange.getAsRange()); |
| 264 |
return OS.str(); |
| 265 |
} |
| 266 |
|
| 267 |
// Collects #define directives inside the main file. |
| 268 |
struct DirectiveCollector : public PPCallbacks { |
| 269 |
DirectiveCollector(const Preprocessor &PP, |
| 270 |
std::vector<TextualPPDirective> &TextualDirectives) |
| 271 |
: LangOpts(PP.getLangOpts()), SM(PP.getSourceManager()), |
| 272 |
TextualDirectives(TextualDirectives) {} |
| 273 |
|
| 274 |
void FileChanged(SourceLocation Loc, FileChangeReason Reason, |
| 275 |
SrcMgr::CharacteristicKind FileType, |
| 276 |
FileID PrevFID) override { |
| 277 |
InMainFile = SM.isWrittenInMainFile(Loc); |
| 278 |
} |
| 279 |
|
| 280 |
void MacroDefined(const Token &MacroNameTok, |
| 281 |
const MacroDirective *MD) override { |
| 282 |
if (!InMainFile) |
| 283 |
return; |
| 284 |
TextualDirectives.emplace_back(); |
| 285 |
TextualPPDirective &TD = TextualDirectives.back(); |
| 286 |
|
| 287 |
const auto *MI = MD->getMacroInfo(); |
| 288 |
TD.Text = |
| 289 |
spellDirective("#define ", |
| 290 |
CharSourceRange::getTokenRange( |
| 291 |
MI->getDefinitionLoc(), MI->getDefinitionEndLoc()), |
| 292 |
LangOpts, SM, TD.DirectiveLine, TD.Offset); |
| 293 |
} |
| 294 |
|
| 295 |
private: |
| 296 |
bool InMainFile = true; |
| 297 |
const LangOptions &LangOpts; |
| 298 |
const SourceManager &SM; |
| 299 |
std::vector<TextualPPDirective> &TextualDirectives; |
| 300 |
}; |
| 301 |
|
| 302 |
struct ScannedPreamble { |
| 303 |
std::vector<Inclusion> Includes; |
| 304 |
std::vector<TextualPPDirective> TextualDirectives; |
| 305 |
PreambleBounds Bounds = {0, false}; |
| 306 |
}; |
| 307 |
|
| 308 |
/// Scans the preprocessor directives in the preamble section of the file by |
| 309 |
/// running preprocessor over \p Contents. Returned includes do not contain |
| 310 |
/// resolved paths. \p Cmd is used to build the compiler invocation, which might |
| 311 |
/// stat/read files. |
| 312 |
llvm::Expected<ScannedPreamble> |
| 313 |
scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) { |
| 314 |
class EmptyFS : public ThreadsafeFS { |
| 315 |
private: |
| 316 |
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override { |
| 317 |
return new llvm::vfs::InMemoryFileSystem; |
| 318 |
} |
| 319 |
}; |
| 320 |
EmptyFS FS; |
| 321 |
// Build and run Preprocessor over the preamble. |
| 322 |
ParseInputs PI; |
| 323 |
PI.Contents = Contents.str(); |
| 324 |
PI.TFS = &FS; |
| 325 |
PI.CompileCommand = Cmd; |
| 326 |
IgnoringDiagConsumer IgnoreDiags; |
| 327 |
auto CI = buildCompilerInvocation(PI, IgnoreDiags); |
| 328 |
if (!CI) |
| 329 |
return error("failed to create compiler invocation"); |
| 330 |
CI->getDiagnosticOpts().IgnoreWarnings = true; |
| 331 |
auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Contents); |
| 332 |
// This means we're scanning (though not preprocessing) the preamble section |
| 333 |
// twice. However, it's important to precisely follow the preamble bounds used |
| 334 |
// elsewhere. |
| 335 |
auto Bounds = ComputePreambleBounds(*CI->getLangOpts(), *ContentsBuffer, 0); |
| 336 |
auto PreambleContents = |
| 337 |
llvm::MemoryBuffer::getMemBufferCopy(Contents.substr(0, Bounds.Size)); |
| 338 |
auto Clang = prepareCompilerInstance( |
| 339 |
std::move(CI), nullptr, std::move(PreambleContents), |
| 340 |
// Provide an empty FS to prevent preprocessor from performing IO. This |
| 341 |
// also implies missing resolved paths for includes. |
| 342 |
FS.view(std::nullopt), IgnoreDiags); |
| 343 |
if (Clang->getFrontendOpts().Inputs.empty()) |
| 344 |
return error("compiler instance had no inputs"); |
| 345 |
// We are only interested in main file includes. |
| 346 |
Clang->getPreprocessorOpts().SingleFileParseMode = true; |
| 347 |
Clang->getPreprocessorOpts().UsePredefines = false; |
| 348 |
PreprocessOnlyAction Action; |
| 349 |
if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) |
| 350 |
return error("failed BeginSourceFile"); |
| 351 |
Preprocessor &PP = Clang->getPreprocessor(); |
| 352 |
IncludeStructure Includes; |
| 353 |
Includes.collect(*Clang); |
| 354 |
ScannedPreamble SP; |
| 355 |
SP.Bounds = Bounds; |
| 356 |
PP.addPPCallbacks( |
| 357 |
std::make_unique<DirectiveCollector>(PP, SP.TextualDirectives)); |
| 358 |
if (llvm::Error Err = Action.Execute()) |
| 359 |
return std::move(Err); |
| 360 |
Action.EndSourceFile(); |
| 361 |
SP.Includes = std::move(Includes.MainFileIncludes); |
| 362 |
return SP; |
| 363 |
} |
| 364 |
|
| 365 |
const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) { |
| 366 |
switch (IncludeDirective) { |
| 367 |
case tok::pp_include: |
| 368 |
return "include"; |
| 369 |
case tok::pp_import: |
| 370 |
return "import"; |
| 371 |
case tok::pp_include_next: |
| 372 |
return "include_next"; |
| 373 |
default: |
| 374 |
break; |
| 375 |
} |
| 376 |
llvm_unreachable("not an include directive"); |
| 377 |
} |
| 378 |
|
| 379 |
// Checks whether \p FileName is a valid spelling of main file. |
| 380 |
bool isMainFile(llvm::StringRef FileName, const SourceManager &SM) { |
| 381 |
auto FE = SM.getFileManager().getFile(FileName); |
| 382 |
return FE && *FE == SM.getFileEntryForID(SM.getMainFileID()); |
| 383 |
} |
| 384 |
|
| 385 |
// Accumulating wall time timer. Similar to llvm::Timer, but much cheaper, |
| 386 |
// it only tracks wall time. |
| 387 |
// Since this is a generic timer, We may want to move this to support/ if we |
| 388 |
// find a use case outside of FS time tracking. |
| 389 |
class WallTimer { |
| 390 |
public: |
| 391 |
WallTimer() : TotalTime(std::chrono::steady_clock::duration::zero()) {} |
| 392 |
// [Re-]Start the timer. |
| 393 |
void startTimer() { StartTime = std::chrono::steady_clock::now(); } |
| 394 |
// Stop the timer and update total time. |
| 395 |
void stopTimer() { |
| 396 |
TotalTime += std::chrono::steady_clock::now() - StartTime; |
| 397 |
} |
| 398 |
// Return total time, in seconds. |
| 399 |
double getTime() { return std::chrono::duration<double>(TotalTime).count(); } |
| 400 |
|
| 401 |
private: |
| 402 |
std::chrono::steady_clock::duration TotalTime; |
| 403 |
std::chrono::steady_clock::time_point StartTime; |
| 404 |
}; |
| 405 |
|
| 406 |
class WallTimerRegion { |
| 407 |
public: |
| 408 |
WallTimerRegion(WallTimer &T) : T(T) { T.startTimer(); } |
| 409 |
~WallTimerRegion() { T.stopTimer(); } |
| 410 |
|
| 411 |
private: |
| 412 |
WallTimer &T; |
| 413 |
}; |
| 414 |
|
| 415 |
// Used by TimerFS, tracks time spent in status() and getBuffer() calls while |
| 416 |
// proxying to underlying File implementation. |
| 417 |
class TimerFile : public llvm::vfs::File { |
| 418 |
public: |
| 419 |
TimerFile(WallTimer &Timer, std::unique_ptr<File> InnerFile) |
| 420 |
: Timer(Timer), InnerFile(std::move(InnerFile)) {} |
| 421 |
|
| 422 |
llvm::ErrorOr<llvm::vfs::Status> status() override { |
| 423 |
WallTimerRegion T(Timer); |
| 424 |
return InnerFile->status(); |
| 425 |
} |
| 426 |
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> |
| 427 |
getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator, |
| 428 |
bool IsVolatile) override { |
| 429 |
WallTimerRegion T(Timer); |
| 430 |
return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator, |
| 431 |
IsVolatile); |
| 432 |
} |
| 433 |
std::error_code close() override { |
| 434 |
WallTimerRegion T(Timer); |
| 435 |
return InnerFile->close(); |
| 436 |
} |
| 437 |
|
| 438 |
private: |
| 439 |
WallTimer &Timer; |
| 440 |
std::unique_ptr<llvm::vfs::File> InnerFile; |
| 441 |
}; |
| 442 |
|
| 443 |
// A wrapper for FileSystems that tracks the amount of time spent in status() |
| 444 |
// and openFileForRead() calls. |
| 445 |
class TimerFS : public llvm::vfs::ProxyFileSystem { |
| 446 |
public: |
| 447 |
TimerFS(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) |
| 448 |
: ProxyFileSystem(std::move(FS)) {} |
| 449 |
|
| 450 |
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> |
| 451 |
openFileForRead(const llvm::Twine &Path) override { |
| 452 |
WallTimerRegion T(Timer); |
| 453 |
auto FileOr = getUnderlyingFS().openFileForRead(Path); |
| 454 |
if (!FileOr) |
| 455 |
return FileOr; |
| 456 |
return std::make_unique<TimerFile>(Timer, std::move(FileOr.get())); |
| 457 |
} |
| 458 |
|
| 459 |
llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override { |
| 460 |
WallTimerRegion T(Timer); |
| 461 |
return getUnderlyingFS().status(Path); |
| 462 |
} |
| 463 |
|
| 464 |
double getTime() { return Timer.getTime(); } |
| 465 |
|
| 466 |
private: |
| 467 |
WallTimer Timer; |
| 468 |
}; |
| 469 |
|
| 470 |
} // namespace |
| 471 |
|
| 472 |
std::shared_ptr<const PreambleData> |
| 473 |
buildPreamble(PathRef FileName, CompilerInvocation CI, |
| 474 |
const ParseInputs &Inputs, bool StoreInMemory, |
| 475 |
PreambleParsedCallback PreambleCallback, |
| 476 |
PreambleBuildStats *Stats) { |
| 477 |
// Note that we don't need to copy the input contents, preamble can live |
| 478 |
// without those. |
| 479 |
auto ContentsBuffer = |
| 480 |
llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName); |
| 481 |
auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0); |
| 482 |
|
| 483 |
trace::Span Tracer("BuildPreamble"); |
| 484 |
SPAN_ATTACH(Tracer, "File", FileName); |
| 485 |
std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners; |
| 486 |
if (Inputs.FeatureModules) { |
| 487 |
for (auto &M : *Inputs.FeatureModules) { |
| 488 |
if (auto Listener = M.astListeners()) |
| 489 |
ASTListeners.emplace_back(std::move(Listener)); |
| 490 |
} |
| 491 |
} |
| 492 |
StoreDiags PreambleDiagnostics; |
| 493 |
PreambleDiagnostics.setDiagCallback( |
| 494 |
[&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) { |
| 495 |
llvm::for_each(ASTListeners, |
| 496 |
[&](const auto &L) { L->sawDiagnostic(D, Diag); }); |
| 497 |
}); |
| 498 |
llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine = |
| 499 |
CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(), |
| 500 |
&PreambleDiagnostics, false); |
| 501 |
const Config &Cfg = Config::current(); |
| 502 |
PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel, |
| 503 |
const clang::Diagnostic &Info) { |
| 504 |
if (Cfg.Diagnostics.SuppressAll || |
| 505 |
isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress, |
| 506 |
*CI.getLangOpts())) |
| 507 |
return DiagnosticsEngine::Ignored; |
| 508 |
switch (Info.getID()) { |
| 509 |
case diag::warn_no_newline_eof: |
| 510 |
case diag::warn_cxx98_compat_no_newline_eof: |
| 511 |
case diag::ext_no_newline_eof: |
| 512 |
// If the preamble doesn't span the whole file, drop the no newline at |
| 513 |
// eof warnings. |
| 514 |
return Bounds.Size != ContentsBuffer->getBufferSize() |
| 515 |
? DiagnosticsEngine::Level::Ignored |
| 516 |
: DiagLevel; |
| 517 |
} |
| 518 |
return DiagLevel; |
| 519 |
}); |
| 520 |
|
| 521 |
// Skip function bodies when building the preamble to speed up building |
| 522 |
// the preamble and make it smaller. |
| 523 |
assert(!CI.getFrontendOpts().SkipFunctionBodies); |
| 524 |
CI.getFrontendOpts().SkipFunctionBodies = true; |
| 525 |
// We don't want to write comment locations into PCH. They are racy and slow |
| 526 |
// to read back. We rely on dynamic index for the comments instead. |
| 527 |
CI.getPreprocessorOpts().WriteCommentListToPCH = false; |
| 528 |
|
| 529 |
CppFilePreambleCallbacks CapturedInfo( |
| 530 |
FileName, PreambleCallback, Stats, |
| 531 |
Inputs.Opts.PreambleParseForwardingFunctions, |
| 532 |
[&ASTListeners](CompilerInstance &CI) { |
| 533 |
for (const auto &L : ASTListeners) |
| 534 |
L->beforeExecute(CI); |
| 535 |
}); |
| 536 |
auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory); |
| 537 |
llvm::SmallString<32> AbsFileName(FileName); |
| 538 |
VFS->makeAbsolute(AbsFileName); |
| 539 |
auto StatCache = std::make_unique<PreambleFileStatusCache>(AbsFileName); |
| 540 |
auto StatCacheFS = StatCache->getProducingFS(VFS); |
| 541 |
llvm::IntrusiveRefCntPtr<TimerFS> TimedFS(new TimerFS(StatCacheFS)); |
| 542 |
|
| 543 |
WallTimer PreambleTimer; |
| 544 |
PreambleTimer.startTimer(); |
| 545 |
auto BuiltPreamble = PrecompiledPreamble::Build( |
| 546 |
CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine, |
| 547 |
Stats ? TimedFS : StatCacheFS, std::make_shared<PCHContainerOperations>(), |
| 548 |
StoreInMemory, CapturedInfo); |
| 549 |
PreambleTimer.stopTimer(); |
| 550 |
|
| 551 |
// When building the AST for the main file, we do want the function |
| 552 |
// bodies. |
| 553 |
CI.getFrontendOpts().SkipFunctionBodies = false; |
| 554 |
|
| 555 |
if (Stats != nullptr) { |
| 556 |
Stats->TotalBuildTime = PreambleTimer.getTime(); |
| 557 |
Stats->FileSystemTime = TimedFS->getTime(); |
| 558 |
Stats->SerializedSize = BuiltPreamble ? BuiltPreamble->getSize() : 0; |
| 559 |
} |
| 560 |
|
| 561 |
if (BuiltPreamble) { |
| 562 |
log("Built preamble of size {0} for file {1} version {2} in {3} seconds", |
| 563 |
BuiltPreamble->getSize(), FileName, Inputs.Version, |
| 564 |
PreambleTimer.getTime()); |
| 565 |
std::vector<Diag> Diags = PreambleDiagnostics.take(); |
| 566 |
auto Result = std::make_shared<PreambleData>(std::move(*BuiltPreamble)); |
| 567 |
Result->Version = Inputs.Version; |
| 568 |
Result->CompileCommand = Inputs.CompileCommand; |
| 569 |
Result->Diags = std::move(Diags); |
| 570 |
Result->Includes = CapturedInfo.takeIncludes(); |
| 571 |
Result->Pragmas = CapturedInfo.takePragmaIncludes(); |
| 572 |
Result->Macros = CapturedInfo.takeMacros(); |
| 573 |
Result->Marks = CapturedInfo.takeMarks(); |
| 574 |
Result->CanonIncludes = CapturedInfo.takeCanonicalIncludes(); |
| 575 |
Result->StatCache = std::move(StatCache); |
| 576 |
Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded(); |
| 577 |
return Result; |
| 578 |
} |
| 579 |
|
| 580 |
elog("Could not build a preamble for file {0} version {1}: {2}", FileName, |
| 581 |
Inputs.Version, BuiltPreamble.getError().message()); |
| 582 |
for (const Diag &D : PreambleDiagnostics.take()) { |
| 583 |
if (D.Severity < DiagnosticsEngine::Error) |
| 584 |
continue; |
| 585 |
// Not an ideal way to show errors, but better than nothing! |
| 586 |
elog(" error: {0}", D.Message); |
| 587 |
} |
| 588 |
return nullptr; |
| 589 |
} |
| 590 |
|
| 591 |
bool isPreambleCompatible(const PreambleData &Preamble, |
| 592 |
const ParseInputs &Inputs, PathRef FileName, |
| 593 |
const CompilerInvocation &CI) { |
| 594 |
auto ContentsBuffer = |
| 595 |
llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName); |
| 596 |
auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0); |
| 597 |
auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory); |
| 598 |
return compileCommandsAreEqual(Inputs.CompileCommand, |
| 599 |
Preamble.CompileCommand) && |
| 600 |
Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS); |
| 601 |
} |
| 602 |
|
| 603 |
void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) { |
| 604 |
for (char C : Text) { |
| 605 |
switch (C) { |
| 606 |
case '\\': |
| 607 |
case '"': |
| 608 |
OS << '\\'; |
| 609 |
break; |
| 610 |
default: |
| 611 |
break; |
| 612 |
} |
| 613 |
OS << C; |
| 614 |
} |
| 615 |
} |
| 616 |
|
| 617 |
PreamblePatch PreamblePatch::create(llvm::StringRef FileName, |
| 618 |
const ParseInputs &Modified, |
| 619 |
const PreambleData &Baseline, |
| 620 |
PatchType PatchType) { |
| 621 |
trace::Span Tracer("CreatePreamblePatch"); |
| 622 |
SPAN_ATTACH(Tracer, "File", FileName); |
| 623 |
assert(llvm::sys::path::is_absolute(FileName) && "relative FileName!"); |
| 624 |
// First scan preprocessor directives in Baseline and Modified. These will be |
| 625 |
// used to figure out newly added directives in Modified. Scanning can fail, |
| 626 |
// the code just bails out and creates an empty patch in such cases, as: |
| 627 |
// - If scanning for Baseline fails, no knowledge of existing includes hence |
| 628 |
// patch will contain all the includes in Modified. Leading to rebuild of |
| 629 |
// whole preamble, which is terribly slow. |
| 630 |
// - If scanning for Modified fails, cannot figure out newly added ones so |
| 631 |
// there's nothing to do but generate an empty patch. |
| 632 |
auto BaselineScan = scanPreamble( |
| 633 |
// Contents needs to be null-terminated. |
| 634 |
Baseline.Preamble.getContents().str(), Modified.CompileCommand); |
| 635 |
if (!BaselineScan) { |
| 636 |
elog("Failed to scan baseline of {0}: {1}", FileName, |
| 637 |
BaselineScan.takeError()); |
| 638 |
return PreamblePatch::unmodified(Baseline); |
| 639 |
} |
| 640 |
auto ModifiedScan = scanPreamble(Modified.Contents, Modified.CompileCommand); |
| 641 |
if (!ModifiedScan) { |
| 642 |
elog("Failed to scan modified contents of {0}: {1}", FileName, |
| 643 |
ModifiedScan.takeError()); |
| 644 |
return PreamblePatch::unmodified(Baseline); |
| 645 |
} |
| 646 |
|
| 647 |
bool IncludesChanged = BaselineScan->Includes != ModifiedScan->Includes; |
| 648 |
bool DirectivesChanged = |
| 649 |
BaselineScan->TextualDirectives != ModifiedScan->TextualDirectives; |
| 650 |
if ((PatchType == PatchType::MacroDirectives || !IncludesChanged) && |
| 651 |
!DirectivesChanged) |
| 652 |
return PreamblePatch::unmodified(Baseline); |
| 653 |
|
| 654 |
PreamblePatch PP; |
| 655 |
// This shouldn't coincide with any real file name. |
| 656 |
llvm::SmallString<128> PatchName; |
| 657 |
llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName), |
| 658 |
PreamblePatchHeaderName); |
| 659 |
PP.PatchFileName = PatchName.str().str(); |
| 660 |
PP.ModifiedBounds = ModifiedScan->Bounds; |
| 661 |
|
| 662 |
llvm::raw_string_ostream Patch(PP.PatchContents); |
| 663 |
// Set default filename for subsequent #line directives |
| 664 |
Patch << "#line 0 \""; |
| 665 |
// FileName part of a line directive is subject to backslash escaping, which |
| 666 |
// might lead to problems on windows especially. |
| 667 |
escapeBackslashAndQuotes(FileName, Patch); |
| 668 |
Patch << "\"\n"; |
| 669 |
|
| 670 |
if (IncludesChanged && PatchType == PatchType::All) { |
| 671 |
// We are only interested in newly added includes, record the ones in |
| 672 |
// Baseline for exclusion. |
| 673 |
llvm::DenseMap<std::pair<tok::PPKeywordKind, llvm::StringRef>, |
| 674 |
/*Resolved=*/llvm::StringRef> |
| 675 |
ExistingIncludes; |
| 676 |
for (const auto &Inc : Baseline.Includes.MainFileIncludes) |
| 677 |
ExistingIncludes[{Inc.Directive, Inc.Written}] = Inc.Resolved; |
| 678 |
// There might be includes coming from disabled regions, record these for |
| 679 |
// exclusion too. note that we don't have resolved paths for those. |
| 680 |
for (const auto &Inc : BaselineScan->Includes) |
| 681 |
ExistingIncludes.try_emplace({Inc.Directive, Inc.Written}); |
| 682 |
// Calculate extra includes that needs to be inserted. |
| 683 |
for (auto &Inc : ModifiedScan->Includes) { |
| 684 |
auto It = ExistingIncludes.find({Inc.Directive, Inc.Written}); |
| 685 |
// Include already present in the baseline preamble. Set resolved path and |
| 686 |
// put into preamble includes. |
| 687 |
if (It != ExistingIncludes.end()) { |
| 688 |
Inc.Resolved = It->second.str(); |
| 689 |
PP.PreambleIncludes.push_back(Inc); |
| 690 |
continue; |
| 691 |
} |
| 692 |
// Include is new in the modified preamble. Inject it into the patch and |
| 693 |
// use #line to set the presumed location to where it is spelled. |
| 694 |
auto LineCol = offsetToClangLineColumn(Modified.Contents, Inc.HashOffset); |
| 695 |
Patch << llvm::formatv("#line {0}\n", LineCol.first); |
| 696 |
Patch << llvm::formatv( |
| 697 |
"#{0} {1}\n", spellingForIncDirective(Inc.Directive), Inc.Written); |
| 698 |
} |
| 699 |
} |
| 700 |
|
| 701 |
if (DirectivesChanged) { |
| 702 |
// We need to patch all the directives, since they are order dependent. e.g: |
| 703 |
// #define BAR(X) NEW(X) // Newly introduced in Modified |
| 704 |
// #define BAR(X) OLD(X) // Exists in the Baseline |
| 705 |
// |
| 706 |
// If we've patched only the first directive, the macro definition would've |
| 707 |
// been wrong for the rest of the file, since patch is applied after the |
| 708 |
// baseline preamble. |
| 709 |
// |
| 710 |
// Note that we deliberately ignore conditional directives and undefs to |
| 711 |
// reduce complexity. The former might cause problems because scanning is |
| 712 |
// imprecise and might pick directives from disabled regions. |
| 713 |
for (const auto &TD : ModifiedScan->TextualDirectives) { |
| 714 |
Patch << "#line " << TD.DirectiveLine << '\n'; |
| 715 |
Patch << TD.Text << '\n'; |
| 716 |
} |
| 717 |
} |
| 718 |
dlog("Created preamble patch: {0}", Patch.str()); |
| 719 |
Patch.flush(); |
| 720 |
return PP; |
| 721 |
} |
| 722 |
|
| 723 |
PreamblePatch PreamblePatch::createFullPatch(llvm::StringRef FileName, |
| 724 |
const ParseInputs &Modified, |
| 725 |
const PreambleData &Baseline) { |
| 726 |
return create(FileName, Modified, Baseline, PatchType::All); |
| 727 |
} |
| 728 |
|
| 729 |
PreamblePatch PreamblePatch::createMacroPatch(llvm::StringRef FileName, |
| 730 |
const ParseInputs &Modified, |
| 731 |
const PreambleData &Baseline) { |
| 732 |
return create(FileName, Modified, Baseline, PatchType::MacroDirectives); |
| 733 |
} |
| 734 |
|
| 735 |
void PreamblePatch::apply(CompilerInvocation &CI) const { |
| 736 |
// No need to map an empty file. |
| 737 |
if (PatchContents.empty()) |
| 738 |
return; |
| 739 |
auto &PPOpts = CI.getPreprocessorOpts(); |
| 740 |
auto PatchBuffer = |
| 741 |
// we copy here to ensure contents are still valid if CI outlives the |
| 742 |
// PreamblePatch. |
| 743 |
llvm::MemoryBuffer::getMemBufferCopy(PatchContents, PatchFileName); |
| 744 |
// CI will take care of the lifetime of the buffer. |
| 745 |
PPOpts.addRemappedFile(PatchFileName, PatchBuffer.release()); |
| 746 |
// The patch will be parsed after loading the preamble ast and before parsing |
| 747 |
// the main file. |
| 748 |
PPOpts.Includes.push_back(PatchFileName); |
| 749 |
} |
| 750 |
|
| 751 |
std::vector<Inclusion> PreamblePatch::preambleIncludes() const { |
| 752 |
return PreambleIncludes; |
| 753 |
} |
| 754 |
|
| 755 |
PreamblePatch PreamblePatch::unmodified(const PreambleData &Preamble) { |
| 756 |
PreamblePatch PP; |
| 757 |
PP.PreambleIncludes = Preamble.Includes.MainFileIncludes; |
| 758 |
PP.ModifiedBounds = Preamble.Preamble.getBounds(); |
| 759 |
return PP; |
| 760 |
} |
| 761 |
|
| 762 |
SourceLocation translatePreamblePatchLocation(SourceLocation Loc, |
| 763 |
const SourceManager &SM) { |
| 764 |
auto DefFile = SM.getFileID(Loc); |
| 765 |
if (auto FE = SM.getFileEntryRefForID(DefFile)) { |
| 766 |
auto IncludeLoc = SM.getIncludeLoc(DefFile); |
| 767 |
// Preamble patch is included inside the builtin file. |
| 768 |
if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) && |
| 769 |
FE->getName().endswith(PreamblePatchHeaderName)) { |
| 770 |
auto Presumed = SM.getPresumedLoc(Loc); |
| 771 |
// Check that line directive is pointing at main file. |
| 772 |
if (Presumed.isValid() && Presumed.getFileID().isInvalid() && |
| 773 |
isMainFile(Presumed.getFilename(), SM)) { |
| 774 |
Loc = SM.translateLineCol(SM.getMainFileID(), Presumed.getLine(), |
| 775 |
Presumed.getColumn()); |
| 776 |
} |
| 777 |
} |
| 778 |
} |
| 779 |
return Loc; |
| 780 |
} |
| 781 |
|
| 782 |
} // namespace clangd |
| 783 |
} // namespace clang |