Debian Codegraph @

llvm-project/clang-tools-extra/clangd/Preamble.cpp

raw
1 //===--- Preamble.cpp - Reusing expensive parts of the AST ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "Preamble.h"
10 #include "Compiler.h"
11 #include "Config.h"
12 #include "Headers.h"
13 #include "SourceCode.h"
14 #include "clang-include-cleaner/Record.h"
15 #include "support/Logger.h"
16 #include "support/ThreadsafeFS.h"
17 #include "support/Trace.h"
18 #include "clang/AST/DeclTemplate.h"
19 #include "clang/Basic/Diagnostic.h"
20 #include "clang/Basic/DiagnosticLex.h"
21 #include "clang/Basic/LangOptions.h"
22 #include "clang/Basic/SourceLocation.h"
23 #include "clang/Basic/SourceManager.h"
24 #include "clang/Basic/TokenKinds.h"
25 #include "clang/Frontend/CompilerInvocation.h"
26 #include "clang/Frontend/FrontendActions.h"
27 #include "clang/Lex/HeaderSearch.h"
28 #include "clang/Lex/Lexer.h"
29 #include "clang/Lex/PPCallbacks.h"
30 #include "clang/Lex/Preprocessor.h"
31 #include "clang/Lex/PreprocessorOptions.h"
32 #include "clang/Tooling/CompilationDatabase.h"
33 #include "llvm/ADT/ArrayRef.h"
34 #include "llvm/ADT/DenseMap.h"
35 #include "llvm/ADT/IntrusiveRefCntPtr.h"
36 #include "llvm/ADT/STLExtras.h"
37 #include "llvm/ADT/SmallString.h"
38 #include "llvm/ADT/StringRef.h"
39 #include "llvm/Support/Error.h"
40 #include "llvm/Support/ErrorHandling.h"
41 #include "llvm/Support/FormatVariadic.h"
42 #include "llvm/Support/MemoryBuffer.h"
43 #include "llvm/Support/Path.h"
44 #include "llvm/Support/VirtualFileSystem.h"
45 #include "llvm/Support/raw_ostream.h"
46 #include <iterator>
47 #include <memory>
48 #include <string>
49 #include <system_error>
50 #include <utility>
51 #include <vector>
52
53 namespace clang {
54 namespace clangd {
55 namespace {
56 constexpr llvm::StringLiteral PreamblePatchHeaderName = "__preamble_patch__.h";
57
58 bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
59 const tooling::CompileCommand &RHS) {
60 // We don't check for Output, it should not matter to clangd.
61 return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
62 llvm::ArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
63 }
64
65 class CppFilePreambleCallbacks : public PreambleCallbacks {
66 public:
67 CppFilePreambleCallbacks(
68 PathRef File, PreambleParsedCallback ParsedCallback,
69 PreambleBuildStats *Stats, bool ParseForwardingFunctions,
70 std::function<void(CompilerInstance &)> BeforeExecuteCallback)
71 : File(File), ParsedCallback(ParsedCallback), Stats(Stats),
72 ParseForwardingFunctions(ParseForwardingFunctions),
73 BeforeExecuteCallback(std::move(BeforeExecuteCallback)) {}
74
75 IncludeStructure takeIncludes() { return std::move(Includes); }
76
77 MainFileMacros takeMacros() { return std::move(Macros); }
78
79 std::vector<PragmaMark> takeMarks() { return std::move(Marks); }
80
81 include_cleaner::PragmaIncludes takePragmaIncludes() {
82 return std::move(Pragmas);
83 }
84 CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); }
85
86 bool isMainFileIncludeGuarded() const { return IsMainFileIncludeGuarded; }
87
88 void AfterExecute(CompilerInstance &CI) override {
89 if (ParsedCallback) {
90 trace::Span Tracer("Running PreambleCallback");
91 ParsedCallback(CI.getASTContext(), CI.getPreprocessor(), CanonIncludes);
92 }
93
94 const SourceManager &SM = CI.getSourceManager();
95 const FileEntry *MainFE = SM.getFileEntryForID(SM.getMainFileID());
96 IsMainFileIncludeGuarded =
97 CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded(
98 MainFE);
99
100 if (Stats) {
101 const ASTContext &AST = CI.getASTContext();
102 Stats->BuildSize = AST.getASTAllocatedMemory();
103 Stats->BuildSize += AST.getSideTableAllocatedMemory();
104 Stats->BuildSize += AST.Idents.getAllocator().getTotalMemory();
105 Stats->BuildSize += AST.Selectors.getTotalMemory();
106
107 Stats->BuildSize += AST.getSourceManager().getContentCacheSize();
108 Stats->BuildSize += AST.getSourceManager().getDataStructureSizes();
109 Stats->BuildSize +=
110 AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
111
112 const Preprocessor &PP = CI.getPreprocessor();
113 Stats->BuildSize += PP.getTotalMemory();
114 if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
115 Stats->BuildSize += PRec->getTotalMemory();
116 Stats->BuildSize += PP.getHeaderSearchInfo().getTotalMemory();
117 }
118 }
119
120 void BeforeExecute(CompilerInstance &CI) override {
121 CanonIncludes.addSystemHeadersMapping(CI.getLangOpts());
122 LangOpts = &CI.getLangOpts();
123 SourceMgr = &CI.getSourceManager();
124 Includes.collect(CI);
125 if (Config::current().Diagnostics.UnusedIncludes ==
126 Config::UnusedIncludesPolicy::Experiment)
127 Pragmas.record(CI);
128 if (BeforeExecuteCallback)
129 BeforeExecuteCallback(CI);
130 }
131
132 std::unique_ptr<PPCallbacks> createPPCallbacks() override {
133 assert(SourceMgr && LangOpts &&
134 "SourceMgr and LangOpts must be set at this point");
135
136 return std::make_unique<PPChainedCallbacks>(
137 std::make_unique<CollectMainFileMacros>(*SourceMgr, Macros),
138 collectPragmaMarksCallback(*SourceMgr, Marks));
139 }
140
141 CommentHandler *getCommentHandler() override {
142 IWYUHandler = collectIWYUHeaderMaps(&CanonIncludes);
143 return IWYUHandler.get();
144 }
145
146 static bool isLikelyForwardingFunction(FunctionTemplateDecl *FT) {
147 const auto *FD = FT->getTemplatedDecl();
148 const auto NumParams = FD->getNumParams();
149 // Check whether its last parameter is a parameter pack...
150 if (NumParams > 0) {
151 const auto *LastParam = FD->getParamDecl(NumParams - 1);
152 if (const auto *PET = dyn_cast<PackExpansionType>(LastParam->getType())) {
153 // ... of the type T&&... or T...
154 const auto BaseType = PET->getPattern().getNonReferenceType();
155 if (const auto *TTPT =
156 dyn_cast<TemplateTypeParmType>(BaseType.getTypePtr())) {
157 // ... whose template parameter comes from the function directly
158 if (FT->getTemplateParameters()->getDepth() == TTPT->getDepth()) {
159 return true;
160 }
161 }
162 }
163 }
164 return false;
165 }
166
167 bool shouldSkipFunctionBody(Decl *D) override {
168 // Usually we don't need to look inside the bodies of header functions
169 // to understand the program. However when forwarding function like
170 // emplace() forward their arguments to some other function, the
171 // interesting overload resolution happens inside the forwarding
172 // function's body. To provide more meaningful diagnostics,
173 // code completion, and parameter hints we should parse (and later
174 // instantiate) the bodies.
175 if (auto *FT = llvm::dyn_cast<clang::FunctionTemplateDecl>(D)) {
176 if (ParseForwardingFunctions) {
177 // Don't skip parsing the body if it looks like a forwarding function
178 if (isLikelyForwardingFunction(FT))
179 return false;
180 } else {
181 // By default, only take care of make_unique
182 // std::make_unique is trivial, and we diagnose bad constructor calls.
183 if (const auto *II = FT->getDeclName().getAsIdentifierInfo()) {
184 if (II->isStr("make_unique") && FT->isInStdNamespace())
185 return false;
186 }
187 }
188 }
189 return true;
190 }
191
192 private:
193 PathRef File;
194 PreambleParsedCallback ParsedCallback;
195 IncludeStructure Includes;
196 CanonicalIncludes CanonIncludes;
197 include_cleaner::PragmaIncludes Pragmas;
198 MainFileMacros Macros;
199 std::vector<PragmaMark> Marks;
200 bool IsMainFileIncludeGuarded = false;
201 std::unique_ptr<CommentHandler> IWYUHandler = nullptr;
202 const clang::LangOptions *LangOpts = nullptr;
203 const SourceManager *SourceMgr = nullptr;
204 PreambleBuildStats *Stats;
205 bool ParseForwardingFunctions;
206 std::function<void(CompilerInstance &)> BeforeExecuteCallback;
207 };
208
209 // Represents directives other than includes, where basic textual information is
210 // enough.
211 struct TextualPPDirective {
212 unsigned DirectiveLine;
213 // Full text that's representing the directive, including the `#`.
214 std::string Text;
215 unsigned Offset;
216
217 bool operator==(const TextualPPDirective &RHS) const {
218 return std::tie(DirectiveLine, Offset, Text) ==
219 std::tie(RHS.DirectiveLine, RHS.Offset, RHS.Text);
220 }
221 };
222
223 // Formats a PP directive consisting of Prefix (e.g. "#define ") and Body ("X
224 // 10"). The formatting is copied so that the tokens in Body have PresumedLocs
225 // with correct columns and lines.
226 std::string spellDirective(llvm::StringRef Prefix,
227 CharSourceRange DirectiveRange,
228 const LangOptions &LangOpts, const SourceManager &SM,
229 unsigned &DirectiveLine, unsigned &Offset) {
230 std::string SpelledDirective;
231 llvm::raw_string_ostream OS(SpelledDirective);
232 OS << Prefix;
233
234 // Make sure DirectiveRange is a char range and doesn't contain macro ids.
235 DirectiveRange = SM.getExpansionRange(DirectiveRange);
236 if (DirectiveRange.isTokenRange()) {
237 DirectiveRange.setEnd(
238 Lexer::getLocForEndOfToken(DirectiveRange.getEnd(), 0, SM, LangOpts));
239 }
240
241 auto DecompLoc = SM.getDecomposedLoc(DirectiveRange.getBegin());
242 DirectiveLine = SM.getLineNumber(DecompLoc.first, DecompLoc.second);
243 Offset = DecompLoc.second;
244 auto TargetColumn = SM.getColumnNumber(DecompLoc.first, DecompLoc.second) - 1;
245
246 // Pad with spaces before DirectiveRange to make sure it will be on right
247 // column when patched.
248 if (Prefix.size() <= TargetColumn) {
249 // There is enough space for Prefix and space before directive, use it.
250 // We try to squeeze the Prefix into the same line whenever we can, as
251 // putting onto a separate line won't work at the beginning of the file.
252 OS << std::string(TargetColumn - Prefix.size(), ' ');
253 } else {
254 // Prefix was longer than the space we had. We produce e.g.:
255 // #line N-1
256 // #define \
257 // X 10
258 OS << "\\\n" << std::string(TargetColumn, ' ');
259 // Decrement because we put an additional line break before
260 // DirectiveRange.begin().
261 --DirectiveLine;
262 }
263 OS << toSourceCode(SM, DirectiveRange.getAsRange());
264 return OS.str();
265 }
266
267 // Collects #define directives inside the main file.
268 struct DirectiveCollector : public PPCallbacks {
269 DirectiveCollector(const Preprocessor &PP,
270 std::vector<TextualPPDirective> &TextualDirectives)
271 : LangOpts(PP.getLangOpts()), SM(PP.getSourceManager()),
272 TextualDirectives(TextualDirectives) {}
273
274 void FileChanged(SourceLocation Loc, FileChangeReason Reason,
275 SrcMgr::CharacteristicKind FileType,
276 FileID PrevFID) override {
277 InMainFile = SM.isWrittenInMainFile(Loc);
278 }
279
280 void MacroDefined(const Token &MacroNameTok,
281 const MacroDirective *MD) override {
282 if (!InMainFile)
283 return;
284 TextualDirectives.emplace_back();
285 TextualPPDirective &TD = TextualDirectives.back();
286
287 const auto *MI = MD->getMacroInfo();
288 TD.Text =
289 spellDirective("#define ",
290 CharSourceRange::getTokenRange(
291 MI->getDefinitionLoc(), MI->getDefinitionEndLoc()),
292 LangOpts, SM, TD.DirectiveLine, TD.Offset);
293 }
294
295 private:
296 bool InMainFile = true;
297 const LangOptions &LangOpts;
298 const SourceManager &SM;
299 std::vector<TextualPPDirective> &TextualDirectives;
300 };
301
302 struct ScannedPreamble {
303 std::vector<Inclusion> Includes;
304 std::vector<TextualPPDirective> TextualDirectives;
305 PreambleBounds Bounds = {0, false};
306 };
307
308 /// Scans the preprocessor directives in the preamble section of the file by
309 /// running preprocessor over \p Contents. Returned includes do not contain
310 /// resolved paths. \p Cmd is used to build the compiler invocation, which might
311 /// stat/read files.
312 llvm::Expected<ScannedPreamble>
313 scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
314 class EmptyFS : public ThreadsafeFS {
315 private:
316 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
317 return new llvm::vfs::InMemoryFileSystem;
318 }
319 };
320 EmptyFS FS;
321 // Build and run Preprocessor over the preamble.
322 ParseInputs PI;
323 PI.Contents = Contents.str();
324 PI.TFS = &FS;
325 PI.CompileCommand = Cmd;
326 IgnoringDiagConsumer IgnoreDiags;
327 auto CI = buildCompilerInvocation(PI, IgnoreDiags);
328 if (!CI)
329 return error("failed to create compiler invocation");
330 CI->getDiagnosticOpts().IgnoreWarnings = true;
331 auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Contents);
332 // This means we're scanning (though not preprocessing) the preamble section
333 // twice. However, it's important to precisely follow the preamble bounds used
334 // elsewhere.
335 auto Bounds = ComputePreambleBounds(*CI->getLangOpts(), *ContentsBuffer, 0);
336 auto PreambleContents =
337 llvm::MemoryBuffer::getMemBufferCopy(Contents.substr(0, Bounds.Size));
338 auto Clang = prepareCompilerInstance(
339 std::move(CI), nullptr, std::move(PreambleContents),
340 // Provide an empty FS to prevent preprocessor from performing IO. This
341 // also implies missing resolved paths for includes.
342 FS.view(std::nullopt), IgnoreDiags);
343 if (Clang->getFrontendOpts().Inputs.empty())
344 return error("compiler instance had no inputs");
345 // We are only interested in main file includes.
346 Clang->getPreprocessorOpts().SingleFileParseMode = true;
347 Clang->getPreprocessorOpts().UsePredefines = false;
348 PreprocessOnlyAction Action;
349 if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
350 return error("failed BeginSourceFile");
351 Preprocessor &PP = Clang->getPreprocessor();
352 IncludeStructure Includes;
353 Includes.collect(*Clang);
354 ScannedPreamble SP;
355 SP.Bounds = Bounds;
356 PP.addPPCallbacks(
357 std::make_unique<DirectiveCollector>(PP, SP.TextualDirectives));
358 if (llvm::Error Err = Action.Execute())
359 return std::move(Err);
360 Action.EndSourceFile();
361 SP.Includes = std::move(Includes.MainFileIncludes);
362 return SP;
363 }
364
365 const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
366 switch (IncludeDirective) {
367 case tok::pp_include:
368 return "include";
369 case tok::pp_import:
370 return "import";
371 case tok::pp_include_next:
372 return "include_next";
373 default:
374 break;
375 }
376 llvm_unreachable("not an include directive");
377 }
378
379 // Checks whether \p FileName is a valid spelling of main file.
380 bool isMainFile(llvm::StringRef FileName, const SourceManager &SM) {
381 auto FE = SM.getFileManager().getFile(FileName);
382 return FE && *FE == SM.getFileEntryForID(SM.getMainFileID());
383 }
384
385 // Accumulating wall time timer. Similar to llvm::Timer, but much cheaper,
386 // it only tracks wall time.
387 // Since this is a generic timer, We may want to move this to support/ if we
388 // find a use case outside of FS time tracking.
389 class WallTimer {
390 public:
391 WallTimer() : TotalTime(std::chrono::steady_clock::duration::zero()) {}
392 // [Re-]Start the timer.
393 void startTimer() { StartTime = std::chrono::steady_clock::now(); }
394 // Stop the timer and update total time.
395 void stopTimer() {
396 TotalTime += std::chrono::steady_clock::now() - StartTime;
397 }
398 // Return total time, in seconds.
399 double getTime() { return std::chrono::duration<double>(TotalTime).count(); }
400
401 private:
402 std::chrono::steady_clock::duration TotalTime;
403 std::chrono::steady_clock::time_point StartTime;
404 };
405
406 class WallTimerRegion {
407 public:
408 WallTimerRegion(WallTimer &T) : T(T) { T.startTimer(); }
409 ~WallTimerRegion() { T.stopTimer(); }
410
411 private:
412 WallTimer &T;
413 };
414
415 // Used by TimerFS, tracks time spent in status() and getBuffer() calls while
416 // proxying to underlying File implementation.
417 class TimerFile : public llvm::vfs::File {
418 public:
419 TimerFile(WallTimer &Timer, std::unique_ptr<File> InnerFile)
420 : Timer(Timer), InnerFile(std::move(InnerFile)) {}
421
422 llvm::ErrorOr<llvm::vfs::Status> status() override {
423 WallTimerRegion T(Timer);
424 return InnerFile->status();
425 }
426 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
427 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
428 bool IsVolatile) override {
429 WallTimerRegion T(Timer);
430 return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
431 IsVolatile);
432 }
433 std::error_code close() override {
434 WallTimerRegion T(Timer);
435 return InnerFile->close();
436 }
437
438 private:
439 WallTimer &Timer;
440 std::unique_ptr<llvm::vfs::File> InnerFile;
441 };
442
443 // A wrapper for FileSystems that tracks the amount of time spent in status()
444 // and openFileForRead() calls.
445 class TimerFS : public llvm::vfs::ProxyFileSystem {
446 public:
447 TimerFS(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
448 : ProxyFileSystem(std::move(FS)) {}
449
450 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
451 openFileForRead(const llvm::Twine &Path) override {
452 WallTimerRegion T(Timer);
453 auto FileOr = getUnderlyingFS().openFileForRead(Path);
454 if (!FileOr)
455 return FileOr;
456 return std::make_unique<TimerFile>(Timer, std::move(FileOr.get()));
457 }
458
459 llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
460 WallTimerRegion T(Timer);
461 return getUnderlyingFS().status(Path);
462 }
463
464 double getTime() { return Timer.getTime(); }
465
466 private:
467 WallTimer Timer;
468 };
469
470 } // namespace
471
472 std::shared_ptr<const PreambleData>
473 buildPreamble(PathRef FileName, CompilerInvocation CI,
474 const ParseInputs &Inputs, bool StoreInMemory,
475 PreambleParsedCallback PreambleCallback,
476 PreambleBuildStats *Stats) {
477 // Note that we don't need to copy the input contents, preamble can live
478 // without those.
479 auto ContentsBuffer =
480 llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
481 auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
482
483 trace::Span Tracer("BuildPreamble");
484 SPAN_ATTACH(Tracer, "File", FileName);
485 std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
486 if (Inputs.FeatureModules) {
487 for (auto &M : *Inputs.FeatureModules) {
488 if (auto Listener = M.astListeners())
489 ASTListeners.emplace_back(std::move(Listener));
490 }
491 }
492 StoreDiags PreambleDiagnostics;
493 PreambleDiagnostics.setDiagCallback(
494 [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
495 llvm::for_each(ASTListeners,
496 [&](const auto &L) { L->sawDiagnostic(D, Diag); });
497 });
498 llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
499 CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
500 &PreambleDiagnostics, false);
501 const Config &Cfg = Config::current();
502 PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
503 const clang::Diagnostic &Info) {
504 if (Cfg.Diagnostics.SuppressAll ||
505 isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
506 *CI.getLangOpts()))
507 return DiagnosticsEngine::Ignored;
508 switch (Info.getID()) {
509 case diag::warn_no_newline_eof:
510 case diag::warn_cxx98_compat_no_newline_eof:
511 case diag::ext_no_newline_eof:
512 // If the preamble doesn't span the whole file, drop the no newline at
513 // eof warnings.
514 return Bounds.Size != ContentsBuffer->getBufferSize()
515 ? DiagnosticsEngine::Level::Ignored
516 : DiagLevel;
517 }
518 return DiagLevel;
519 });
520
521 // Skip function bodies when building the preamble to speed up building
522 // the preamble and make it smaller.
523 assert(!CI.getFrontendOpts().SkipFunctionBodies);
524 CI.getFrontendOpts().SkipFunctionBodies = true;
525 // We don't want to write comment locations into PCH. They are racy and slow
526 // to read back. We rely on dynamic index for the comments instead.
527 CI.getPreprocessorOpts().WriteCommentListToPCH = false;
528
529 CppFilePreambleCallbacks CapturedInfo(
530 FileName, PreambleCallback, Stats,
531 Inputs.Opts.PreambleParseForwardingFunctions,
532 [&ASTListeners](CompilerInstance &CI) {
533 for (const auto &L : ASTListeners)
534 L->beforeExecute(CI);
535 });
536 auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
537 llvm::SmallString<32> AbsFileName(FileName);
538 VFS->makeAbsolute(AbsFileName);
539 auto StatCache = std::make_unique<PreambleFileStatusCache>(AbsFileName);
540 auto StatCacheFS = StatCache->getProducingFS(VFS);
541 llvm::IntrusiveRefCntPtr<TimerFS> TimedFS(new TimerFS(StatCacheFS));
542
543 WallTimer PreambleTimer;
544 PreambleTimer.startTimer();
545 auto BuiltPreamble = PrecompiledPreamble::Build(
546 CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
547 Stats ? TimedFS : StatCacheFS, std::make_shared<PCHContainerOperations>(),
548 StoreInMemory, CapturedInfo);
549 PreambleTimer.stopTimer();
550
551 // When building the AST for the main file, we do want the function
552 // bodies.
553 CI.getFrontendOpts().SkipFunctionBodies = false;
554
555 if (Stats != nullptr) {
556 Stats->TotalBuildTime = PreambleTimer.getTime();
557 Stats->FileSystemTime = TimedFS->getTime();
558 Stats->SerializedSize = BuiltPreamble ? BuiltPreamble->getSize() : 0;
559 }
560
561 if (BuiltPreamble) {
562 log("Built preamble of size {0} for file {1} version {2} in {3} seconds",
563 BuiltPreamble->getSize(), FileName, Inputs.Version,
564 PreambleTimer.getTime());
565 std::vector<Diag> Diags = PreambleDiagnostics.take();
566 auto Result = std::make_shared<PreambleData>(std::move(*BuiltPreamble));
567 Result->Version = Inputs.Version;
568 Result->CompileCommand = Inputs.CompileCommand;
569 Result->Diags = std::move(Diags);
570 Result->Includes = CapturedInfo.takeIncludes();
571 Result->Pragmas = CapturedInfo.takePragmaIncludes();
572 Result->Macros = CapturedInfo.takeMacros();
573 Result->Marks = CapturedInfo.takeMarks();
574 Result->CanonIncludes = CapturedInfo.takeCanonicalIncludes();
575 Result->StatCache = std::move(StatCache);
576 Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded();
577 return Result;
578 }
579
580 elog("Could not build a preamble for file {0} version {1}: {2}", FileName,
581 Inputs.Version, BuiltPreamble.getError().message());
582 for (const Diag &D : PreambleDiagnostics.take()) {
583 if (D.Severity < DiagnosticsEngine::Error)
584 continue;
585 // Not an ideal way to show errors, but better than nothing!
586 elog(" error: {0}", D.Message);
587 }
588 return nullptr;
589 }
590
591 bool isPreambleCompatible(const PreambleData &Preamble,
592 const ParseInputs &Inputs, PathRef FileName,
593 const CompilerInvocation &CI) {
594 auto ContentsBuffer =
595 llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
596 auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
597 auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
598 return compileCommandsAreEqual(Inputs.CompileCommand,
599 Preamble.CompileCommand) &&
600 Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS);
601 }
602
603 void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) {
604 for (char C : Text) {
605 switch (C) {
606 case '\\':
607 case '"':
608 OS << '\\';
609 break;
610 default:
611 break;
612 }
613 OS << C;
614 }
615 }
616
617 PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
618 const ParseInputs &Modified,
619 const PreambleData &Baseline,
620 PatchType PatchType) {
621 trace::Span Tracer("CreatePreamblePatch");
622 SPAN_ATTACH(Tracer, "File", FileName);
623 assert(llvm::sys::path::is_absolute(FileName) && "relative FileName!");
624 // First scan preprocessor directives in Baseline and Modified. These will be
625 // used to figure out newly added directives in Modified. Scanning can fail,
626 // the code just bails out and creates an empty patch in such cases, as:
627 // - If scanning for Baseline fails, no knowledge of existing includes hence
628 // patch will contain all the includes in Modified. Leading to rebuild of
629 // whole preamble, which is terribly slow.
630 // - If scanning for Modified fails, cannot figure out newly added ones so
631 // there's nothing to do but generate an empty patch.
632 auto BaselineScan = scanPreamble(
633 // Contents needs to be null-terminated.
634 Baseline.Preamble.getContents().str(), Modified.CompileCommand);
635 if (!BaselineScan) {
636 elog("Failed to scan baseline of {0}: {1}", FileName,
637 BaselineScan.takeError());
638 return PreamblePatch::unmodified(Baseline);
639 }
640 auto ModifiedScan = scanPreamble(Modified.Contents, Modified.CompileCommand);
641 if (!ModifiedScan) {
642 elog("Failed to scan modified contents of {0}: {1}", FileName,
643 ModifiedScan.takeError());
644 return PreamblePatch::unmodified(Baseline);
645 }
646
647 bool IncludesChanged = BaselineScan->Includes != ModifiedScan->Includes;
648 bool DirectivesChanged =
649 BaselineScan->TextualDirectives != ModifiedScan->TextualDirectives;
650 if ((PatchType == PatchType::MacroDirectives || !IncludesChanged) &&
651 !DirectivesChanged)
652 return PreamblePatch::unmodified(Baseline);
653
654 PreamblePatch PP;
655 // This shouldn't coincide with any real file name.
656 llvm::SmallString<128> PatchName;
657 llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
658 PreamblePatchHeaderName);
659 PP.PatchFileName = PatchName.str().str();
660 PP.ModifiedBounds = ModifiedScan->Bounds;
661
662 llvm::raw_string_ostream Patch(PP.PatchContents);
663 // Set default filename for subsequent #line directives
664 Patch << "#line 0 \"";
665 // FileName part of a line directive is subject to backslash escaping, which
666 // might lead to problems on windows especially.
667 escapeBackslashAndQuotes(FileName, Patch);
668 Patch << "\"\n";
669
670 if (IncludesChanged && PatchType == PatchType::All) {
671 // We are only interested in newly added includes, record the ones in
672 // Baseline for exclusion.
673 llvm::DenseMap<std::pair<tok::PPKeywordKind, llvm::StringRef>,
674 /*Resolved=*/llvm::StringRef>
675 ExistingIncludes;
676 for (const auto &Inc : Baseline.Includes.MainFileIncludes)
677 ExistingIncludes[{Inc.Directive, Inc.Written}] = Inc.Resolved;
678 // There might be includes coming from disabled regions, record these for
679 // exclusion too. note that we don't have resolved paths for those.
680 for (const auto &Inc : BaselineScan->Includes)
681 ExistingIncludes.try_emplace({Inc.Directive, Inc.Written});
682 // Calculate extra includes that needs to be inserted.
683 for (auto &Inc : ModifiedScan->Includes) {
684 auto It = ExistingIncludes.find({Inc.Directive, Inc.Written});
685 // Include already present in the baseline preamble. Set resolved path and
686 // put into preamble includes.
687 if (It != ExistingIncludes.end()) {
688 Inc.Resolved = It->second.str();
689 PP.PreambleIncludes.push_back(Inc);
690 continue;
691 }
692 // Include is new in the modified preamble. Inject it into the patch and
693 // use #line to set the presumed location to where it is spelled.
694 auto LineCol = offsetToClangLineColumn(Modified.Contents, Inc.HashOffset);
695 Patch << llvm::formatv("#line {0}\n", LineCol.first);
696 Patch << llvm::formatv(
697 "#{0} {1}\n", spellingForIncDirective(Inc.Directive), Inc.Written);
698 }
699 }
700
701 if (DirectivesChanged) {
702 // We need to patch all the directives, since they are order dependent. e.g:
703 // #define BAR(X) NEW(X) // Newly introduced in Modified
704 // #define BAR(X) OLD(X) // Exists in the Baseline
705 //
706 // If we've patched only the first directive, the macro definition would've
707 // been wrong for the rest of the file, since patch is applied after the
708 // baseline preamble.
709 //
710 // Note that we deliberately ignore conditional directives and undefs to
711 // reduce complexity. The former might cause problems because scanning is
712 // imprecise and might pick directives from disabled regions.
713 for (const auto &TD : ModifiedScan->TextualDirectives) {
714 Patch << "#line " << TD.DirectiveLine << '\n';
715 Patch << TD.Text << '\n';
716 }
717 }
718 dlog("Created preamble patch: {0}", Patch.str());
719 Patch.flush();
720 return PP;
721 }
722
723 PreamblePatch PreamblePatch::createFullPatch(llvm::StringRef FileName,
724 const ParseInputs &Modified,
725 const PreambleData &Baseline) {
726 return create(FileName, Modified, Baseline, PatchType::All);
727 }
728
729 PreamblePatch PreamblePatch::createMacroPatch(llvm::StringRef FileName,
730 const ParseInputs &Modified,
731 const PreambleData &Baseline) {
732 return create(FileName, Modified, Baseline, PatchType::MacroDirectives);
733 }
734
735 void PreamblePatch::apply(CompilerInvocation &CI) const {
736 // No need to map an empty file.
737 if (PatchContents.empty())
738 return;
739 auto &PPOpts = CI.getPreprocessorOpts();
740 auto PatchBuffer =
741 // we copy here to ensure contents are still valid if CI outlives the
742 // PreamblePatch.
743 llvm::MemoryBuffer::getMemBufferCopy(PatchContents, PatchFileName);
744 // CI will take care of the lifetime of the buffer.
745 PPOpts.addRemappedFile(PatchFileName, PatchBuffer.release());
746 // The patch will be parsed after loading the preamble ast and before parsing
747 // the main file.
748 PPOpts.Includes.push_back(PatchFileName);
749 }
750
751 std::vector<Inclusion> PreamblePatch::preambleIncludes() const {
752 return PreambleIncludes;
753 }
754
755 PreamblePatch PreamblePatch::unmodified(const PreambleData &Preamble) {
756 PreamblePatch PP;
757 PP.PreambleIncludes = Preamble.Includes.MainFileIncludes;
758 PP.ModifiedBounds = Preamble.Preamble.getBounds();
759 return PP;
760 }
761
762 SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
763 const SourceManager &SM) {
764 auto DefFile = SM.getFileID(Loc);
765 if (auto FE = SM.getFileEntryRefForID(DefFile)) {
766 auto IncludeLoc = SM.getIncludeLoc(DefFile);
767 // Preamble patch is included inside the builtin file.
768 if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) &&
769 FE->getName().endswith(PreamblePatchHeaderName)) {
770 auto Presumed = SM.getPresumedLoc(Loc);
771 // Check that line directive is pointing at main file.
772 if (Presumed.isValid() && Presumed.getFileID().isInvalid() &&
773 isMainFile(Presumed.getFilename(), SM)) {
774 Loc = SM.translateLineCol(SM.getMainFileID(), Presumed.getLine(),
775 Presumed.getColumn());
776 }
777 }
778 }
779 return Loc;
780 }
781
782 } // namespace clangd
783 } // namespace clang