| 1 |
//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// This file implements the Preprocessor interface. |
| 10 |
// |
| 11 |
//===----------------------------------------------------------------------===// |
| 12 |
// |
| 13 |
// Options to support: |
| 14 |
// -H - Print the name of each header file used. |
| 15 |
// -d[DNI] - Dump various things. |
| 16 |
// -fworking-directory - #line's with preprocessor's working dir. |
| 17 |
// -fpreprocessed |
| 18 |
// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD |
| 19 |
// -W* |
| 20 |
// -w |
| 21 |
// |
| 22 |
// Messages to emit: |
| 23 |
// "Multiple include guards may be useful for:\n" |
| 24 |
// |
| 25 |
//===----------------------------------------------------------------------===// |
| 26 |
|
| 27 |
#include "clang/Lex/Preprocessor.h" |
| 28 |
#include "clang/Basic/Builtins.h" |
| 29 |
#include "clang/Basic/FileManager.h" |
| 30 |
#include "clang/Basic/FileSystemStatCache.h" |
| 31 |
#include "clang/Basic/IdentifierTable.h" |
| 32 |
#include "clang/Basic/LLVM.h" |
| 33 |
#include "clang/Basic/LangOptions.h" |
| 34 |
#include "clang/Basic/Module.h" |
| 35 |
#include "clang/Basic/SourceLocation.h" |
| 36 |
#include "clang/Basic/SourceManager.h" |
| 37 |
#include "clang/Basic/TargetInfo.h" |
| 38 |
#include "clang/Lex/CodeCompletionHandler.h" |
| 39 |
#include "clang/Lex/ExternalPreprocessorSource.h" |
| 40 |
#include "clang/Lex/HeaderSearch.h" |
| 41 |
#include "clang/Lex/LexDiagnostic.h" |
| 42 |
#include "clang/Lex/Lexer.h" |
| 43 |
#include "clang/Lex/LiteralSupport.h" |
| 44 |
#include "clang/Lex/MacroArgs.h" |
| 45 |
#include "clang/Lex/MacroInfo.h" |
| 46 |
#include "clang/Lex/ModuleLoader.h" |
| 47 |
#include "clang/Lex/Pragma.h" |
| 48 |
#include "clang/Lex/PreprocessingRecord.h" |
| 49 |
#include "clang/Lex/PreprocessorLexer.h" |
| 50 |
#include "clang/Lex/PreprocessorOptions.h" |
| 51 |
#include "clang/Lex/ScratchBuffer.h" |
| 52 |
#include "clang/Lex/Token.h" |
| 53 |
#include "clang/Lex/TokenLexer.h" |
| 54 |
#include "llvm/ADT/APInt.h" |
| 55 |
#include "llvm/ADT/ArrayRef.h" |
| 56 |
#include "llvm/ADT/DenseMap.h" |
| 57 |
#include "llvm/ADT/STLExtras.h" |
| 58 |
#include "llvm/ADT/SmallString.h" |
| 59 |
#include "llvm/ADT/SmallVector.h" |
| 60 |
#include "llvm/ADT/StringRef.h" |
| 61 |
#include "llvm/Support/Capacity.h" |
| 62 |
#include "llvm/Support/ErrorHandling.h" |
| 63 |
#include "llvm/Support/MemoryBuffer.h" |
| 64 |
#include "llvm/Support/raw_ostream.h" |
| 65 |
#include <algorithm> |
| 66 |
#include <cassert> |
| 67 |
#include <memory> |
| 68 |
#include <optional> |
| 69 |
#include <string> |
| 70 |
#include <utility> |
| 71 |
#include <vector> |
| 72 |
|
| 73 |
using namespace clang; |
| 74 |
|
| 75 |
LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry) |
| 76 |
|
| 77 |
ExternalPreprocessorSource::~ExternalPreprocessorSource() = default; |
| 78 |
|
| 79 |
Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts, |
| 80 |
DiagnosticsEngine &diags, LangOptions &opts, |
| 81 |
SourceManager &SM, HeaderSearch &Headers, |
| 82 |
ModuleLoader &TheModuleLoader, |
| 83 |
IdentifierInfoLookup *IILookup, bool OwnsHeaders, |
| 84 |
TranslationUnitKind TUKind) |
| 85 |
: PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), |
| 86 |
FileMgr(Headers.getFileMgr()), SourceMgr(SM), |
| 87 |
ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers), |
| 88 |
TheModuleLoader(TheModuleLoader), ExternalSource(nullptr), |
| 89 |
// As the language options may have not been loaded yet (when |
| 90 |
// deserializing an ASTUnit), adding keywords to the identifier table is |
| 91 |
// deferred to Preprocessor::Initialize(). |
| 92 |
Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), |
| 93 |
TUKind(TUKind), SkipMainFilePreamble(0, true), |
| 94 |
CurSubmoduleState(&NullSubmoduleState) { |
| 95 |
OwnsHeaderSearch = OwnsHeaders; |
| 96 |
|
| 97 |
// Default to discarding comments. |
| 98 |
KeepComments = false; |
| 99 |
KeepMacroComments = false; |
| 100 |
SuppressIncludeNotFoundError = false; |
| 101 |
|
| 102 |
// Macro expansion is enabled. |
| 103 |
DisableMacroExpansion = false; |
| 104 |
MacroExpansionInDirectivesOverride = false; |
| 105 |
InMacroArgs = false; |
| 106 |
ArgMacro = nullptr; |
| 107 |
InMacroArgPreExpansion = false; |
| 108 |
NumCachedTokenLexers = 0; |
| 109 |
PragmasEnabled = true; |
| 110 |
ParsingIfOrElifDirective = false; |
| 111 |
PreprocessedOutput = false; |
| 112 |
|
| 113 |
// We haven't read anything from the external source. |
| 114 |
ReadMacrosFromExternalSource = false; |
| 115 |
|
| 116 |
BuiltinInfo = std::make_unique<Builtin::Context>(); |
| 117 |
|
| 118 |
// "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of |
| 119 |
// a macro. They get unpoisoned where it is allowed. |
| 120 |
(Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); |
| 121 |
SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use); |
| 122 |
(Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned(); |
| 123 |
SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use); |
| 124 |
|
| 125 |
// Initialize the pragma handlers. |
| 126 |
RegisterBuiltinPragmas(); |
| 127 |
|
| 128 |
// Initialize builtin macros like __LINE__ and friends. |
| 129 |
RegisterBuiltinMacros(); |
| 130 |
|
| 131 |
if(LangOpts.Borland) { |
| 132 |
Ident__exception_info = getIdentifierInfo("_exception_info"); |
| 133 |
Ident___exception_info = getIdentifierInfo("__exception_info"); |
| 134 |
Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation"); |
| 135 |
Ident__exception_code = getIdentifierInfo("_exception_code"); |
| 136 |
Ident___exception_code = getIdentifierInfo("__exception_code"); |
| 137 |
Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode"); |
| 138 |
Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination"); |
| 139 |
Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination"); |
| 140 |
Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination"); |
| 141 |
} else { |
| 142 |
Ident__exception_info = Ident__exception_code = nullptr; |
| 143 |
Ident__abnormal_termination = Ident___exception_info = nullptr; |
| 144 |
Ident___exception_code = Ident___abnormal_termination = nullptr; |
| 145 |
Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr; |
| 146 |
Ident_AbnormalTermination = nullptr; |
| 147 |
} |
| 148 |
|
| 149 |
// If using a PCH where a #pragma hdrstop is expected, start skipping tokens. |
| 150 |
if (usingPCHWithPragmaHdrStop()) |
| 151 |
SkippingUntilPragmaHdrStop = true; |
| 152 |
|
| 153 |
// If using a PCH with a through header, start skipping tokens. |
| 154 |
if (!this->PPOpts->PCHThroughHeader.empty() && |
| 155 |
!this->PPOpts->ImplicitPCHInclude.empty()) |
| 156 |
SkippingUntilPCHThroughHeader = true; |
| 157 |
|
| 158 |
if (this->PPOpts->GeneratePreamble) |
| 159 |
PreambleConditionalStack.startRecording(); |
| 160 |
|
| 161 |
MaxTokens = LangOpts.MaxTokens; |
| 162 |
} |
| 163 |
|
| 164 |
Preprocessor::~Preprocessor() { |
| 165 |
assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!"); |
| 166 |
|
| 167 |
IncludeMacroStack.clear(); |
| 168 |
|
| 169 |
// Free any cached macro expanders. |
| 170 |
// This populates MacroArgCache, so all TokenLexers need to be destroyed |
| 171 |
// before the code below that frees up the MacroArgCache list. |
| 172 |
std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr); |
| 173 |
CurTokenLexer.reset(); |
| 174 |
|
| 175 |
// Free any cached MacroArgs. |
| 176 |
for (MacroArgs *ArgList = MacroArgCache; ArgList;) |
| 177 |
ArgList = ArgList->deallocate(); |
| 178 |
|
| 179 |
// Delete the header search info, if we own it. |
| 180 |
if (OwnsHeaderSearch) |
| 181 |
delete &HeaderInfo; |
| 182 |
} |
| 183 |
|
| 184 |
void Preprocessor::Initialize(const TargetInfo &Target, |
| 185 |
const TargetInfo *AuxTarget) { |
| 186 |
assert((!this->Target || this->Target == &Target) && |
| 187 |
"Invalid override of target information"); |
| 188 |
this->Target = &Target; |
| 189 |
|
| 190 |
assert((!this->AuxTarget || this->AuxTarget == AuxTarget) && |
| 191 |
"Invalid override of aux target information."); |
| 192 |
this->AuxTarget = AuxTarget; |
| 193 |
|
| 194 |
// Initialize information about built-ins. |
| 195 |
BuiltinInfo->InitializeTarget(Target, AuxTarget); |
| 196 |
HeaderInfo.setTarget(Target); |
| 197 |
|
| 198 |
// Populate the identifier table with info about keywords for the current language. |
| 199 |
Identifiers.AddKeywords(LangOpts); |
| 200 |
|
| 201 |
// Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo. |
| 202 |
setTUFPEvalMethod(getTargetInfo().getFPEvalMethod()); |
| 203 |
|
| 204 |
if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine) |
| 205 |
// Use setting from TargetInfo. |
| 206 |
setCurrentFPEvalMethod(SourceLocation(), Target.getFPEvalMethod()); |
| 207 |
else |
| 208 |
// Set initial value of __FLT_EVAL_METHOD__ from the command line. |
| 209 |
setCurrentFPEvalMethod(SourceLocation(), getLangOpts().getFPEvalMethod()); |
| 210 |
} |
| 211 |
|
| 212 |
void Preprocessor::InitializeForModelFile() { |
| 213 |
NumEnteredSourceFiles = 0; |
| 214 |
|
| 215 |
// Reset pragmas |
| 216 |
PragmaHandlersBackup = std::move(PragmaHandlers); |
| 217 |
PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef()); |
| 218 |
RegisterBuiltinPragmas(); |
| 219 |
|
| 220 |
// Reset PredefinesFileID |
| 221 |
PredefinesFileID = FileID(); |
| 222 |
} |
| 223 |
|
| 224 |
void Preprocessor::FinalizeForModelFile() { |
| 225 |
NumEnteredSourceFiles = 1; |
| 226 |
|
| 227 |
PragmaHandlers = std::move(PragmaHandlersBackup); |
| 228 |
} |
| 229 |
|
| 230 |
void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { |
| 231 |
llvm::errs() << tok::getTokenName(Tok.getKind()); |
| 232 |
|
| 233 |
if (!Tok.isAnnotation()) |
| 234 |
llvm::errs() << " '" << getSpelling(Tok) << "'"; |
| 235 |
|
| 236 |
if (!DumpFlags) return; |
| 237 |
|
| 238 |
llvm::errs() << "\t"; |
| 239 |
if (Tok.isAtStartOfLine()) |
| 240 |
llvm::errs() << " [StartOfLine]"; |
| 241 |
if (Tok.hasLeadingSpace()) |
| 242 |
llvm::errs() << " [LeadingSpace]"; |
| 243 |
if (Tok.isExpandDisabled()) |
| 244 |
llvm::errs() << " [ExpandDisabled]"; |
| 245 |
if (Tok.needsCleaning()) { |
| 246 |
const char *Start = SourceMgr.getCharacterData(Tok.getLocation()); |
| 247 |
llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) |
| 248 |
<< "']"; |
| 249 |
} |
| 250 |
|
| 251 |
llvm::errs() << "\tLoc=<"; |
| 252 |
DumpLocation(Tok.getLocation()); |
| 253 |
llvm::errs() << ">"; |
| 254 |
} |
| 255 |
|
| 256 |
void Preprocessor::DumpLocation(SourceLocation Loc) const { |
| 257 |
Loc.print(llvm::errs(), SourceMgr); |
| 258 |
} |
| 259 |
|
| 260 |
void Preprocessor::DumpMacro(const MacroInfo &MI) const { |
| 261 |
llvm::errs() << "MACRO: "; |
| 262 |
for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) { |
| 263 |
DumpToken(MI.getReplacementToken(i)); |
| 264 |
llvm::errs() << " "; |
| 265 |
} |
| 266 |
llvm::errs() << "\n"; |
| 267 |
} |
| 268 |
|
| 269 |
void Preprocessor::PrintStats() { |
| 270 |
llvm::errs() << "\n*** Preprocessor Stats:\n"; |
| 271 |
llvm::errs() << NumDirectives << " directives found:\n"; |
| 272 |
llvm::errs() << " " << NumDefined << " #define.\n"; |
| 273 |
llvm::errs() << " " << NumUndefined << " #undef.\n"; |
| 274 |
llvm::errs() << " #include/#include_next/#import:\n"; |
| 275 |
llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n"; |
| 276 |
llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n"; |
| 277 |
llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n"; |
| 278 |
llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n"; |
| 279 |
llvm::errs() << " " << NumEndif << " #endif.\n"; |
| 280 |
llvm::errs() << " " << NumPragma << " #pragma.\n"; |
| 281 |
llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n"; |
| 282 |
|
| 283 |
llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/" |
| 284 |
<< NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, " |
| 285 |
<< NumFastMacroExpanded << " on the fast path.\n"; |
| 286 |
llvm::errs() << (NumFastTokenPaste+NumTokenPaste) |
| 287 |
<< " token paste (##) operations performed, " |
| 288 |
<< NumFastTokenPaste << " on the fast path.\n"; |
| 289 |
|
| 290 |
llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total"; |
| 291 |
|
| 292 |
llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory(); |
| 293 |
llvm::errs() << "\n Macro Expanded Tokens: " |
| 294 |
<< llvm::capacity_in_bytes(MacroExpandedTokens); |
| 295 |
llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity(); |
| 296 |
// FIXME: List information for all submodules. |
| 297 |
llvm::errs() << "\n Macros: " |
| 298 |
<< llvm::capacity_in_bytes(CurSubmoduleState->Macros); |
| 299 |
llvm::errs() << "\n #pragma push_macro Info: " |
| 300 |
<< llvm::capacity_in_bytes(PragmaPushMacroInfo); |
| 301 |
llvm::errs() << "\n Poison Reasons: " |
| 302 |
<< llvm::capacity_in_bytes(PoisonReasons); |
| 303 |
llvm::errs() << "\n Comment Handlers: " |
| 304 |
<< llvm::capacity_in_bytes(CommentHandlers) << "\n"; |
| 305 |
} |
| 306 |
|
| 307 |
Preprocessor::macro_iterator |
| 308 |
Preprocessor::macro_begin(bool IncludeExternalMacros) const { |
| 309 |
if (IncludeExternalMacros && ExternalSource && |
| 310 |
!ReadMacrosFromExternalSource) { |
| 311 |
ReadMacrosFromExternalSource = true; |
| 312 |
ExternalSource->ReadDefinedMacros(); |
| 313 |
} |
| 314 |
|
| 315 |
// Make sure we cover all macros in visible modules. |
| 316 |
for (const ModuleMacro &Macro : ModuleMacros) |
| 317 |
CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState())); |
| 318 |
|
| 319 |
return CurSubmoduleState->Macros.begin(); |
| 320 |
} |
| 321 |
|
| 322 |
size_t Preprocessor::getTotalMemory() const { |
| 323 |
return BP.getTotalMemory() |
| 324 |
+ llvm::capacity_in_bytes(MacroExpandedTokens) |
| 325 |
+ Predefines.capacity() /* Predefines buffer. */ |
| 326 |
// FIXME: Include sizes from all submodules, and include MacroInfo sizes, |
| 327 |
// and ModuleMacros. |
| 328 |
+ llvm::capacity_in_bytes(CurSubmoduleState->Macros) |
| 329 |
+ llvm::capacity_in_bytes(PragmaPushMacroInfo) |
| 330 |
+ llvm::capacity_in_bytes(PoisonReasons) |
| 331 |
+ llvm::capacity_in_bytes(CommentHandlers); |
| 332 |
} |
| 333 |
|
| 334 |
Preprocessor::macro_iterator |
| 335 |
Preprocessor::macro_end(bool IncludeExternalMacros) const { |
| 336 |
if (IncludeExternalMacros && ExternalSource && |
| 337 |
!ReadMacrosFromExternalSource) { |
| 338 |
ReadMacrosFromExternalSource = true; |
| 339 |
ExternalSource->ReadDefinedMacros(); |
| 340 |
} |
| 341 |
|
| 342 |
return CurSubmoduleState->Macros.end(); |
| 343 |
} |
| 344 |
|
| 345 |
/// Compares macro tokens with a specified token value sequence. |
| 346 |
static bool MacroDefinitionEquals(const MacroInfo *MI, |
| 347 |
ArrayRef<TokenValue> Tokens) { |
| 348 |
return Tokens.size() == MI->getNumTokens() && |
| 349 |
std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin()); |
| 350 |
} |
| 351 |
|
| 352 |
StringRef Preprocessor::getLastMacroWithSpelling( |
| 353 |
SourceLocation Loc, |
| 354 |
ArrayRef<TokenValue> Tokens) const { |
| 355 |
SourceLocation BestLocation; |
| 356 |
StringRef BestSpelling; |
| 357 |
for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end(); |
| 358 |
I != E; ++I) { |
| 359 |
const MacroDirective::DefInfo |
| 360 |
Def = I->second.findDirectiveAtLoc(Loc, SourceMgr); |
| 361 |
if (!Def || !Def.getMacroInfo()) |
| 362 |
continue; |
| 363 |
if (!Def.getMacroInfo()->isObjectLike()) |
| 364 |
continue; |
| 365 |
if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens)) |
| 366 |
continue; |
| 367 |
SourceLocation Location = Def.getLocation(); |
| 368 |
// Choose the macro defined latest. |
| 369 |
if (BestLocation.isInvalid() || |
| 370 |
(Location.isValid() && |
| 371 |
SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) { |
| 372 |
BestLocation = Location; |
| 373 |
BestSpelling = I->first->getName(); |
| 374 |
} |
| 375 |
} |
| 376 |
return BestSpelling; |
| 377 |
} |
| 378 |
|
| 379 |
void Preprocessor::recomputeCurLexerKind() { |
| 380 |
if (CurLexer) |
| 381 |
CurLexerKind = CurLexer->isDependencyDirectivesLexer() |
| 382 |
? CLK_DependencyDirectivesLexer |
| 383 |
: CLK_Lexer; |
| 384 |
else if (CurTokenLexer) |
| 385 |
CurLexerKind = CLK_TokenLexer; |
| 386 |
else |
| 387 |
CurLexerKind = CLK_CachingLexer; |
| 388 |
} |
| 389 |
|
| 390 |
bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, |
| 391 |
unsigned CompleteLine, |
| 392 |
unsigned CompleteColumn) { |
| 393 |
assert(File); |
| 394 |
assert(CompleteLine && CompleteColumn && "Starts from 1:1"); |
| 395 |
assert(!CodeCompletionFile && "Already set"); |
| 396 |
|
| 397 |
// Load the actual file's contents. |
| 398 |
std::optional<llvm::MemoryBufferRef> Buffer = |
| 399 |
SourceMgr.getMemoryBufferForFileOrNone(File); |
| 400 |
if (!Buffer) |
| 401 |
return true; |
| 402 |
|
| 403 |
// Find the byte position of the truncation point. |
| 404 |
const char *Position = Buffer->getBufferStart(); |
| 405 |
for (unsigned Line = 1; Line < CompleteLine; ++Line) { |
| 406 |
for (; *Position; ++Position) { |
| 407 |
if (*Position != '\r' && *Position != '\n') |
| 408 |
continue; |
| 409 |
|
| 410 |
// Eat \r\n or \n\r as a single line. |
| 411 |
if ((Position[1] == '\r' || Position[1] == '\n') && |
| 412 |
Position[0] != Position[1]) |
| 413 |
++Position; |
| 414 |
++Position; |
| 415 |
break; |
| 416 |
} |
| 417 |
} |
| 418 |
|
| 419 |
Position += CompleteColumn - 1; |
| 420 |
|
| 421 |
// If pointing inside the preamble, adjust the position at the beginning of |
| 422 |
// the file after the preamble. |
| 423 |
if (SkipMainFilePreamble.first && |
| 424 |
SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) { |
| 425 |
if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first) |
| 426 |
Position = Buffer->getBufferStart() + SkipMainFilePreamble.first; |
| 427 |
} |
| 428 |
|
| 429 |
if (Position > Buffer->getBufferEnd()) |
| 430 |
Position = Buffer->getBufferEnd(); |
| 431 |
|
| 432 |
CodeCompletionFile = File; |
| 433 |
CodeCompletionOffset = Position - Buffer->getBufferStart(); |
| 434 |
|
| 435 |
auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer( |
| 436 |
Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier()); |
| 437 |
char *NewBuf = NewBuffer->getBufferStart(); |
| 438 |
char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf); |
| 439 |
*NewPos = '\0'; |
| 440 |
std::copy(Position, Buffer->getBufferEnd(), NewPos+1); |
| 441 |
SourceMgr.overrideFileContents(File, std::move(NewBuffer)); |
| 442 |
|
| 443 |
return false; |
| 444 |
} |
| 445 |
|
| 446 |
void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir, |
| 447 |
bool IsAngled) { |
| 448 |
setCodeCompletionReached(); |
| 449 |
if (CodeComplete) |
| 450 |
CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled); |
| 451 |
} |
| 452 |
|
| 453 |
void Preprocessor::CodeCompleteNaturalLanguage() { |
| 454 |
setCodeCompletionReached(); |
| 455 |
if (CodeComplete) |
| 456 |
CodeComplete->CodeCompleteNaturalLanguage(); |
| 457 |
} |
| 458 |
|
| 459 |
/// getSpelling - This method is used to get the spelling of a token into a |
| 460 |
/// SmallVector. Note that the returned StringRef may not point to the |
| 461 |
/// supplied buffer if a copy can be avoided. |
| 462 |
StringRef Preprocessor::getSpelling(const Token &Tok, |
| 463 |
SmallVectorImpl<char> &Buffer, |
| 464 |
bool *Invalid) const { |
| 465 |
// NOTE: this has to be checked *before* testing for an IdentifierInfo. |
| 466 |
if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) { |
| 467 |
// Try the fast path. |
| 468 |
if (const IdentifierInfo *II = Tok.getIdentifierInfo()) |
| 469 |
return II->getName(); |
| 470 |
} |
| 471 |
|
| 472 |
// Resize the buffer if we need to copy into it. |
| 473 |
if (Tok.needsCleaning()) |
| 474 |
Buffer.resize(Tok.getLength()); |
| 475 |
|
| 476 |
const char *Ptr = Buffer.data(); |
| 477 |
unsigned Len = getSpelling(Tok, Ptr, Invalid); |
| 478 |
return StringRef(Ptr, Len); |
| 479 |
} |
| 480 |
|
| 481 |
/// CreateString - Plop the specified string into a scratch buffer and return a |
| 482 |
/// location for it. If specified, the source location provides a source |
| 483 |
/// location for the token. |
| 484 |
void Preprocessor::CreateString(StringRef Str, Token &Tok, |
| 485 |
SourceLocation ExpansionLocStart, |
| 486 |
SourceLocation ExpansionLocEnd) { |
| 487 |
Tok.setLength(Str.size()); |
| 488 |
|
| 489 |
const char *DestPtr; |
| 490 |
SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr); |
| 491 |
|
| 492 |
if (ExpansionLocStart.isValid()) |
| 493 |
Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart, |
| 494 |
ExpansionLocEnd, Str.size()); |
| 495 |
Tok.setLocation(Loc); |
| 496 |
|
| 497 |
// If this is a raw identifier or a literal token, set the pointer data. |
| 498 |
if (Tok.is(tok::raw_identifier)) |
| 499 |
Tok.setRawIdentifierData(DestPtr); |
| 500 |
else if (Tok.isLiteral()) |
| 501 |
Tok.setLiteralData(DestPtr); |
| 502 |
} |
| 503 |
|
| 504 |
SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) { |
| 505 |
auto &SM = getSourceManager(); |
| 506 |
SourceLocation SpellingLoc = SM.getSpellingLoc(Loc); |
| 507 |
std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellingLoc); |
| 508 |
bool Invalid = false; |
| 509 |
StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); |
| 510 |
if (Invalid) |
| 511 |
return SourceLocation(); |
| 512 |
|
| 513 |
// FIXME: We could consider re-using spelling for tokens we see repeatedly. |
| 514 |
const char *DestPtr; |
| 515 |
SourceLocation Spelling = |
| 516 |
ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr); |
| 517 |
return SM.createTokenSplitLoc(Spelling, Loc, Loc.getLocWithOffset(Length)); |
| 518 |
} |
| 519 |
|
| 520 |
Module *Preprocessor::getCurrentModule() { |
| 521 |
if (!getLangOpts().isCompilingModule()) |
| 522 |
return nullptr; |
| 523 |
|
| 524 |
return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule); |
| 525 |
} |
| 526 |
|
| 527 |
Module *Preprocessor::getCurrentModuleImplementation() { |
| 528 |
if (!getLangOpts().isCompilingModuleImplementation()) |
| 529 |
return nullptr; |
| 530 |
|
| 531 |
return getHeaderSearchInfo().lookupModule(getLangOpts().ModuleName); |
| 532 |
} |
| 533 |
|
| 534 |
//===----------------------------------------------------------------------===// |
| 535 |
// Preprocessor Initialization Methods |
| 536 |
//===----------------------------------------------------------------------===// |
| 537 |
|
| 538 |
/// EnterMainSourceFile - Enter the specified FileID as the main source file, |
| 539 |
/// which implicitly adds the builtin defines etc. |
| 540 |
void Preprocessor::EnterMainSourceFile() { |
| 541 |
// We do not allow the preprocessor to reenter the main file. Doing so will |
| 542 |
// cause FileID's to accumulate information from both runs (e.g. #line |
| 543 |
// information) and predefined macros aren't guaranteed to be set properly. |
| 544 |
assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!"); |
| 545 |
FileID MainFileID = SourceMgr.getMainFileID(); |
| 546 |
|
| 547 |
// If MainFileID is loaded it means we loaded an AST file, no need to enter |
| 548 |
// a main file. |
| 549 |
if (!SourceMgr.isLoadedFileID(MainFileID)) { |
| 550 |
// Enter the main file source buffer. |
| 551 |
EnterSourceFile(MainFileID, nullptr, SourceLocation()); |
| 552 |
|
| 553 |
// If we've been asked to skip bytes in the main file (e.g., as part of a |
| 554 |
// precompiled preamble), do so now. |
| 555 |
if (SkipMainFilePreamble.first > 0) |
| 556 |
CurLexer->SetByteOffset(SkipMainFilePreamble.first, |
| 557 |
SkipMainFilePreamble.second); |
| 558 |
|
| 559 |
// Tell the header info that the main file was entered. If the file is later |
| 560 |
// #imported, it won't be re-entered. |
| 561 |
if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID)) |
| 562 |
markIncluded(FE); |
| 563 |
} |
| 564 |
|
| 565 |
// Preprocess Predefines to populate the initial preprocessor state. |
| 566 |
std::unique_ptr<llvm::MemoryBuffer> SB = |
| 567 |
llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>"); |
| 568 |
assert(SB && "Cannot create predefined source buffer"); |
| 569 |
FileID FID = SourceMgr.createFileID(std::move(SB)); |
| 570 |
assert(FID.isValid() && "Could not create FileID for predefines?"); |
| 571 |
setPredefinesFileID(FID); |
| 572 |
|
| 573 |
// Start parsing the predefines. |
| 574 |
EnterSourceFile(FID, nullptr, SourceLocation()); |
| 575 |
|
| 576 |
if (!PPOpts->PCHThroughHeader.empty()) { |
| 577 |
// Lookup and save the FileID for the through header. If it isn't found |
| 578 |
// in the search path, it's a fatal error. |
| 579 |
OptionalFileEntryRef File = LookupFile( |
| 580 |
SourceLocation(), PPOpts->PCHThroughHeader, |
| 581 |
/*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, |
| 582 |
/*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr, |
| 583 |
/*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr, |
| 584 |
/*IsFrameworkFound=*/nullptr); |
| 585 |
if (!File) { |
| 586 |
Diag(SourceLocation(), diag::err_pp_through_header_not_found) |
| 587 |
<< PPOpts->PCHThroughHeader; |
| 588 |
return; |
| 589 |
} |
| 590 |
setPCHThroughHeaderFileID( |
| 591 |
SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User)); |
| 592 |
} |
| 593 |
|
| 594 |
// Skip tokens from the Predefines and if needed the main file. |
| 595 |
if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) || |
| 596 |
(usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop)) |
| 597 |
SkipTokensWhileUsingPCH(); |
| 598 |
} |
| 599 |
|
| 600 |
void Preprocessor::setPCHThroughHeaderFileID(FileID FID) { |
| 601 |
assert(PCHThroughHeaderFileID.isInvalid() && |
| 602 |
"PCHThroughHeaderFileID already set!"); |
| 603 |
PCHThroughHeaderFileID = FID; |
| 604 |
} |
| 605 |
|
| 606 |
bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) { |
| 607 |
assert(PCHThroughHeaderFileID.isValid() && |
| 608 |
"Invalid PCH through header FileID"); |
| 609 |
return FE == SourceMgr.getFileEntryForID(PCHThroughHeaderFileID); |
| 610 |
} |
| 611 |
|
| 612 |
bool Preprocessor::creatingPCHWithThroughHeader() { |
| 613 |
return TUKind == TU_Prefix && !PPOpts->PCHThroughHeader.empty() && |
| 614 |
PCHThroughHeaderFileID.isValid(); |
| 615 |
} |
| 616 |
|
| 617 |
bool Preprocessor::usingPCHWithThroughHeader() { |
| 618 |
return TUKind != TU_Prefix && !PPOpts->PCHThroughHeader.empty() && |
| 619 |
PCHThroughHeaderFileID.isValid(); |
| 620 |
} |
| 621 |
|
| 622 |
bool Preprocessor::creatingPCHWithPragmaHdrStop() { |
| 623 |
return TUKind == TU_Prefix && PPOpts->PCHWithHdrStop; |
| 624 |
} |
| 625 |
|
| 626 |
bool Preprocessor::usingPCHWithPragmaHdrStop() { |
| 627 |
return TUKind != TU_Prefix && PPOpts->PCHWithHdrStop; |
| 628 |
} |
| 629 |
|
| 630 |
/// Skip tokens until after the #include of the through header or |
| 631 |
/// until after a #pragma hdrstop is seen. Tokens in the predefines file |
| 632 |
/// and the main file may be skipped. If the end of the predefines file |
| 633 |
/// is reached, skipping continues into the main file. If the end of the |
| 634 |
/// main file is reached, it's a fatal error. |
| 635 |
void Preprocessor::SkipTokensWhileUsingPCH() { |
| 636 |
bool ReachedMainFileEOF = false; |
| 637 |
bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader; |
| 638 |
bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop; |
| 639 |
Token Tok; |
| 640 |
while (true) { |
| 641 |
bool InPredefines = |
| 642 |
(CurLexer && CurLexer->getFileID() == getPredefinesFileID()); |
| 643 |
switch (CurLexerKind) { |
| 644 |
case CLK_Lexer: |
| 645 |
CurLexer->Lex(Tok); |
| 646 |
break; |
| 647 |
case CLK_TokenLexer: |
| 648 |
CurTokenLexer->Lex(Tok); |
| 649 |
break; |
| 650 |
case CLK_CachingLexer: |
| 651 |
CachingLex(Tok); |
| 652 |
break; |
| 653 |
case CLK_DependencyDirectivesLexer: |
| 654 |
CurLexer->LexDependencyDirectiveToken(Tok); |
| 655 |
break; |
| 656 |
case CLK_LexAfterModuleImport: |
| 657 |
LexAfterModuleImport(Tok); |
| 658 |
break; |
| 659 |
} |
| 660 |
if (Tok.is(tok::eof) && !InPredefines) { |
| 661 |
ReachedMainFileEOF = true; |
| 662 |
break; |
| 663 |
} |
| 664 |
if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader) |
| 665 |
break; |
| 666 |
if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop) |
| 667 |
break; |
| 668 |
} |
| 669 |
if (ReachedMainFileEOF) { |
| 670 |
if (UsingPCHThroughHeader) |
| 671 |
Diag(SourceLocation(), diag::err_pp_through_header_not_seen) |
| 672 |
<< PPOpts->PCHThroughHeader << 1; |
| 673 |
else if (!PPOpts->PCHWithHdrStopCreate) |
| 674 |
Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen); |
| 675 |
} |
| 676 |
} |
| 677 |
|
| 678 |
void Preprocessor::replayPreambleConditionalStack() { |
| 679 |
// Restore the conditional stack from the preamble, if there is one. |
| 680 |
if (PreambleConditionalStack.isReplaying()) { |
| 681 |
assert(CurPPLexer && |
| 682 |
"CurPPLexer is null when calling replayPreambleConditionalStack."); |
| 683 |
CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); |
| 684 |
PreambleConditionalStack.doneReplaying(); |
| 685 |
if (PreambleConditionalStack.reachedEOFWhileSkipping()) |
| 686 |
SkipExcludedConditionalBlock( |
| 687 |
PreambleConditionalStack.SkipInfo->HashTokenLoc, |
| 688 |
PreambleConditionalStack.SkipInfo->IfTokenLoc, |
| 689 |
PreambleConditionalStack.SkipInfo->FoundNonSkipPortion, |
| 690 |
PreambleConditionalStack.SkipInfo->FoundElse, |
| 691 |
PreambleConditionalStack.SkipInfo->ElseLoc); |
| 692 |
} |
| 693 |
} |
| 694 |
|
| 695 |
void Preprocessor::EndSourceFile() { |
| 696 |
// Notify the client that we reached the end of the source file. |
| 697 |
if (Callbacks) |
| 698 |
Callbacks->EndOfMainFile(); |
| 699 |
} |
| 700 |
|
| 701 |
//===----------------------------------------------------------------------===// |
| 702 |
// Lexer Event Handling. |
| 703 |
//===----------------------------------------------------------------------===// |
| 704 |
|
| 705 |
/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the |
| 706 |
/// identifier information for the token and install it into the token, |
| 707 |
/// updating the token kind accordingly. |
| 708 |
IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const { |
| 709 |
assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!"); |
| 710 |
|
| 711 |
// Look up this token, see if it is a macro, or if it is a language keyword. |
| 712 |
IdentifierInfo *II; |
| 713 |
if (!Identifier.needsCleaning() && !Identifier.hasUCN()) { |
| 714 |
// No cleaning needed, just use the characters from the lexed buffer. |
| 715 |
II = getIdentifierInfo(Identifier.getRawIdentifier()); |
| 716 |
} else { |
| 717 |
// Cleaning needed, alloca a buffer, clean into it, then use the buffer. |
| 718 |
SmallString<64> IdentifierBuffer; |
| 719 |
StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer); |
| 720 |
|
| 721 |
if (Identifier.hasUCN()) { |
| 722 |
SmallString<64> UCNIdentifierBuffer; |
| 723 |
expandUCNs(UCNIdentifierBuffer, CleanedStr); |
| 724 |
II = getIdentifierInfo(UCNIdentifierBuffer); |
| 725 |
} else { |
| 726 |
II = getIdentifierInfo(CleanedStr); |
| 727 |
} |
| 728 |
} |
| 729 |
|
| 730 |
// Update the token info (identifier info and appropriate token kind). |
| 731 |
// FIXME: the raw_identifier may contain leading whitespace which is removed |
| 732 |
// from the cleaned identifier token. The SourceLocation should be updated to |
| 733 |
// refer to the non-whitespace character. For instance, the text "\\\nB" (a |
| 734 |
// line continuation before 'B') is parsed as a single tok::raw_identifier and |
| 735 |
// is cleaned to tok::identifier "B". After cleaning the token's length is |
| 736 |
// still 3 and the SourceLocation refers to the location of the backslash. |
| 737 |
Identifier.setIdentifierInfo(II); |
| 738 |
Identifier.setKind(II->getTokenID()); |
| 739 |
|
| 740 |
return II; |
| 741 |
} |
| 742 |
|
| 743 |
void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) { |
| 744 |
PoisonReasons[II] = DiagID; |
| 745 |
} |
| 746 |
|
| 747 |
void Preprocessor::PoisonSEHIdentifiers(bool Poison) { |
| 748 |
assert(Ident__exception_code && Ident__exception_info); |
| 749 |
assert(Ident___exception_code && Ident___exception_info); |
| 750 |
Ident__exception_code->setIsPoisoned(Poison); |
| 751 |
Ident___exception_code->setIsPoisoned(Poison); |
| 752 |
Ident_GetExceptionCode->setIsPoisoned(Poison); |
| 753 |
Ident__exception_info->setIsPoisoned(Poison); |
| 754 |
Ident___exception_info->setIsPoisoned(Poison); |
| 755 |
Ident_GetExceptionInfo->setIsPoisoned(Poison); |
| 756 |
Ident__abnormal_termination->setIsPoisoned(Poison); |
| 757 |
Ident___abnormal_termination->setIsPoisoned(Poison); |
| 758 |
Ident_AbnormalTermination->setIsPoisoned(Poison); |
| 759 |
} |
| 760 |
|
| 761 |
void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) { |
| 762 |
assert(Identifier.getIdentifierInfo() && |
| 763 |
"Can't handle identifiers without identifier info!"); |
| 764 |
llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it = |
| 765 |
PoisonReasons.find(Identifier.getIdentifierInfo()); |
| 766 |
if(it == PoisonReasons.end()) |
| 767 |
Diag(Identifier, diag::err_pp_used_poisoned_id); |
| 768 |
else |
| 769 |
Diag(Identifier,it->second) << Identifier.getIdentifierInfo(); |
| 770 |
} |
| 771 |
|
| 772 |
void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const { |
| 773 |
assert(II.isOutOfDate() && "not out of date"); |
| 774 |
getExternalSource()->updateOutOfDateIdentifier(II); |
| 775 |
} |
| 776 |
|
| 777 |
/// HandleIdentifier - This callback is invoked when the lexer reads an |
| 778 |
/// identifier. This callback looks up the identifier in the map and/or |
| 779 |
/// potentially macro expands it or turns it into a named token (like 'for'). |
| 780 |
/// |
| 781 |
/// Note that callers of this method are guarded by checking the |
| 782 |
/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the |
| 783 |
/// IdentifierInfo methods that compute these properties will need to change to |
| 784 |
/// match. |
| 785 |
bool Preprocessor::HandleIdentifier(Token &Identifier) { |
| 786 |
assert(Identifier.getIdentifierInfo() && |
| 787 |
"Can't handle identifiers without identifier info!"); |
| 788 |
|
| 789 |
IdentifierInfo &II = *Identifier.getIdentifierInfo(); |
| 790 |
|
| 791 |
// If the information about this identifier is out of date, update it from |
| 792 |
// the external source. |
| 793 |
// We have to treat __VA_ARGS__ in a special way, since it gets |
| 794 |
// serialized with isPoisoned = true, but our preprocessor may have |
| 795 |
// unpoisoned it if we're defining a C99 macro. |
| 796 |
if (II.isOutOfDate()) { |
| 797 |
bool CurrentIsPoisoned = false; |
| 798 |
const bool IsSpecialVariadicMacro = |
| 799 |
&II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__; |
| 800 |
if (IsSpecialVariadicMacro) |
| 801 |
CurrentIsPoisoned = II.isPoisoned(); |
| 802 |
|
| 803 |
updateOutOfDateIdentifier(II); |
| 804 |
Identifier.setKind(II.getTokenID()); |
| 805 |
|
| 806 |
if (IsSpecialVariadicMacro) |
| 807 |
II.setIsPoisoned(CurrentIsPoisoned); |
| 808 |
} |
| 809 |
|
| 810 |
// If this identifier was poisoned, and if it was not produced from a macro |
| 811 |
// expansion, emit an error. |
| 812 |
if (II.isPoisoned() && CurPPLexer) { |
| 813 |
HandlePoisonedIdentifier(Identifier); |
| 814 |
} |
| 815 |
|
| 816 |
// If this is a macro to be expanded, do it. |
| 817 |
if (MacroDefinition MD = getMacroDefinition(&II)) { |
| 818 |
auto *MI = MD.getMacroInfo(); |
| 819 |
assert(MI && "macro definition with no macro info?"); |
| 820 |
if (!DisableMacroExpansion) { |
| 821 |
if (!Identifier.isExpandDisabled() && MI->isEnabled()) { |
| 822 |
// C99 6.10.3p10: If the preprocessing token immediately after the |
| 823 |
// macro name isn't a '(', this macro should not be expanded. |
| 824 |
if (!MI->isFunctionLike() || isNextPPTokenLParen()) |
| 825 |
return HandleMacroExpandedIdentifier(Identifier, MD); |
| 826 |
} else { |
| 827 |
// C99 6.10.3.4p2 says that a disabled macro may never again be |
| 828 |
// expanded, even if it's in a context where it could be expanded in the |
| 829 |
// future. |
| 830 |
Identifier.setFlag(Token::DisableExpand); |
| 831 |
if (MI->isObjectLike() || isNextPPTokenLParen()) |
| 832 |
Diag(Identifier, diag::pp_disabled_macro_expansion); |
| 833 |
} |
| 834 |
} |
| 835 |
} |
| 836 |
|
| 837 |
// If this identifier is a keyword in a newer Standard or proposed Standard, |
| 838 |
// produce a warning. Don't warn if we're not considering macro expansion, |
| 839 |
// since this identifier might be the name of a macro. |
| 840 |
// FIXME: This warning is disabled in cases where it shouldn't be, like |
| 841 |
// "#define constexpr constexpr", "int constexpr;" |
| 842 |
if (II.isFutureCompatKeyword() && !DisableMacroExpansion) { |
| 843 |
Diag(Identifier, getIdentifierTable().getFutureCompatDiagKind(II, getLangOpts())) |
| 844 |
<< II.getName(); |
| 845 |
// Don't diagnose this keyword again in this translation unit. |
| 846 |
II.setIsFutureCompatKeyword(false); |
| 847 |
} |
| 848 |
|
| 849 |
// If this is an extension token, diagnose its use. |
| 850 |
// We avoid diagnosing tokens that originate from macro definitions. |
| 851 |
// FIXME: This warning is disabled in cases where it shouldn't be, |
| 852 |
// like "#define TY typeof", "TY(1) x". |
| 853 |
if (II.isExtensionToken() && !DisableMacroExpansion) |
| 854 |
Diag(Identifier, diag::ext_token_used); |
| 855 |
|
| 856 |
// If this is the 'import' contextual keyword following an '@', note |
| 857 |
// that the next token indicates a module name. |
| 858 |
// |
| 859 |
// Note that we do not treat 'import' as a contextual |
| 860 |
// keyword when we're in a caching lexer, because caching lexers only get |
| 861 |
// used in contexts where import declarations are disallowed. |
| 862 |
// |
| 863 |
// Likewise if this is the C++ Modules TS import keyword. |
| 864 |
if (((LastTokenWasAt && II.isModulesImport()) || |
| 865 |
Identifier.is(tok::kw_import)) && |
| 866 |
!InMacroArgs && !DisableMacroExpansion && |
| 867 |
(getLangOpts().Modules || getLangOpts().DebuggerSupport) && |
| 868 |
CurLexerKind != CLK_CachingLexer) { |
| 869 |
ModuleImportLoc = Identifier.getLocation(); |
| 870 |
NamedModuleImportPath.clear(); |
| 871 |
IsAtImport = true; |
| 872 |
ModuleImportExpectsIdentifier = true; |
| 873 |
CurLexerKind = CLK_LexAfterModuleImport; |
| 874 |
} |
| 875 |
return true; |
| 876 |
} |
| 877 |
|
| 878 |
void Preprocessor::Lex(Token &Result) { |
| 879 |
++LexLevel; |
| 880 |
|
| 881 |
// We loop here until a lex function returns a token; this avoids recursion. |
| 882 |
bool ReturnedToken; |
| 883 |
do { |
| 884 |
switch (CurLexerKind) { |
| 885 |
case CLK_Lexer: |
| 886 |
ReturnedToken = CurLexer->Lex(Result); |
| 887 |
break; |
| 888 |
case CLK_TokenLexer: |
| 889 |
ReturnedToken = CurTokenLexer->Lex(Result); |
| 890 |
break; |
| 891 |
case CLK_CachingLexer: |
| 892 |
CachingLex(Result); |
| 893 |
ReturnedToken = true; |
| 894 |
break; |
| 895 |
case CLK_DependencyDirectivesLexer: |
| 896 |
ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result); |
| 897 |
break; |
| 898 |
case CLK_LexAfterModuleImport: |
| 899 |
ReturnedToken = LexAfterModuleImport(Result); |
| 900 |
break; |
| 901 |
} |
| 902 |
} while (!ReturnedToken); |
| 903 |
|
| 904 |
if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure) |
| 905 |
return; |
| 906 |
|
| 907 |
if (Result.is(tok::code_completion) && Result.getIdentifierInfo()) { |
| 908 |
// Remember the identifier before code completion token. |
| 909 |
setCodeCompletionIdentifierInfo(Result.getIdentifierInfo()); |
| 910 |
setCodeCompletionTokenRange(Result.getLocation(), Result.getEndLoc()); |
| 911 |
// Set IdenfitierInfo to null to avoid confusing code that handles both |
| 912 |
// identifiers and completion tokens. |
| 913 |
Result.setIdentifierInfo(nullptr); |
| 914 |
} |
| 915 |
|
| 916 |
// Update StdCXXImportSeqState to track our position within a C++20 import-seq |
| 917 |
// if this token is being produced as a result of phase 4 of translation. |
| 918 |
// Update TrackGMFState to decide if we are currently in a Global Module |
| 919 |
// Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state |
| 920 |
// depends on the prevailing StdCXXImportSeq state in two cases. |
| 921 |
if (getLangOpts().CPlusPlusModules && LexLevel == 1 && |
| 922 |
!Result.getFlag(Token::IsReinjected)) { |
| 923 |
switch (Result.getKind()) { |
| 924 |
case tok::l_paren: case tok::l_square: case tok::l_brace: |
| 925 |
StdCXXImportSeqState.handleOpenBracket(); |
| 926 |
break; |
| 927 |
case tok::r_paren: case tok::r_square: |
| 928 |
StdCXXImportSeqState.handleCloseBracket(); |
| 929 |
break; |
| 930 |
case tok::r_brace: |
| 931 |
StdCXXImportSeqState.handleCloseBrace(); |
| 932 |
break; |
| 933 |
// This token is injected to represent the translation of '#include "a.h"' |
| 934 |
// into "import a.h;". Mimic the notional ';'. |
| 935 |
case tok::annot_module_include: |
| 936 |
case tok::semi: |
| 937 |
TrackGMFState.handleSemi(); |
| 938 |
StdCXXImportSeqState.handleSemi(); |
| 939 |
ModuleDeclState.handleSemi(); |
| 940 |
break; |
| 941 |
case tok::header_name: |
| 942 |
case tok::annot_header_unit: |
| 943 |
StdCXXImportSeqState.handleHeaderName(); |
| 944 |
break; |
| 945 |
case tok::kw_export: |
| 946 |
TrackGMFState.handleExport(); |
| 947 |
StdCXXImportSeqState.handleExport(); |
| 948 |
ModuleDeclState.handleExport(); |
| 949 |
break; |
| 950 |
case tok::colon: |
| 951 |
ModuleDeclState.handleColon(); |
| 952 |
break; |
| 953 |
case tok::period: |
| 954 |
ModuleDeclState.handlePeriod(); |
| 955 |
break; |
| 956 |
case tok::identifier: |
| 957 |
if (Result.getIdentifierInfo()->isModulesImport()) { |
| 958 |
TrackGMFState.handleImport(StdCXXImportSeqState.afterTopLevelSeq()); |
| 959 |
StdCXXImportSeqState.handleImport(); |
| 960 |
if (StdCXXImportSeqState.afterImportSeq()) { |
| 961 |
ModuleImportLoc = Result.getLocation(); |
| 962 |
NamedModuleImportPath.clear(); |
| 963 |
IsAtImport = false; |
| 964 |
ModuleImportExpectsIdentifier = true; |
| 965 |
CurLexerKind = CLK_LexAfterModuleImport; |
| 966 |
} |
| 967 |
break; |
| 968 |
} else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) { |
| 969 |
TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq()); |
| 970 |
ModuleDeclState.handleModule(); |
| 971 |
break; |
| 972 |
} else { |
| 973 |
ModuleDeclState.handleIdentifier(Result.getIdentifierInfo()); |
| 974 |
if (ModuleDeclState.isModuleCandidate()) |
| 975 |
break; |
| 976 |
} |
| 977 |
[[fallthrough]]; |
| 978 |
default: |
| 979 |
TrackGMFState.handleMisc(); |
| 980 |
StdCXXImportSeqState.handleMisc(); |
| 981 |
ModuleDeclState.handleMisc(); |
| 982 |
break; |
| 983 |
} |
| 984 |
} |
| 985 |
|
| 986 |
LastTokenWasAt = Result.is(tok::at); |
| 987 |
--LexLevel; |
| 988 |
|
| 989 |
if ((LexLevel == 0 || PreprocessToken) && |
| 990 |
!Result.getFlag(Token::IsReinjected)) { |
| 991 |
if (LexLevel == 0) |
| 992 |
++TokenCount; |
| 993 |
if (OnToken) |
| 994 |
OnToken(Result); |
| 995 |
} |
| 996 |
} |
| 997 |
|
| 998 |
/// Lex a header-name token (including one formed from header-name-tokens if |
| 999 |
/// \p AllowConcatenation is \c true). |
| 1000 |
/// |
| 1001 |
/// \param FilenameTok Filled in with the next token. On success, this will |
| 1002 |
/// be either a header_name token. On failure, it will be whatever other |
| 1003 |
/// token was found instead. |
| 1004 |
/// \param AllowMacroExpansion If \c true, allow the header name to be formed |
| 1005 |
/// by macro expansion (concatenating tokens as necessary if the first |
| 1006 |
/// token is a '<'). |
| 1007 |
/// \return \c true if we reached EOD or EOF while looking for a > token in |
| 1008 |
/// a concatenated header name and diagnosed it. \c false otherwise. |
| 1009 |
bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { |
| 1010 |
// Lex using header-name tokenization rules if tokens are being lexed from |
| 1011 |
// a file. Just grab a token normally if we're in a macro expansion. |
| 1012 |
if (CurPPLexer) |
| 1013 |
CurPPLexer->LexIncludeFilename(FilenameTok); |
| 1014 |
else |
| 1015 |
Lex(FilenameTok); |
| 1016 |
|
| 1017 |
// This could be a <foo/bar.h> file coming from a macro expansion. In this |
| 1018 |
// case, glue the tokens together into an angle_string_literal token. |
| 1019 |
SmallString<128> FilenameBuffer; |
| 1020 |
if (FilenameTok.is(tok::less) && AllowMacroExpansion) { |
| 1021 |
bool StartOfLine = FilenameTok.isAtStartOfLine(); |
| 1022 |
bool LeadingSpace = FilenameTok.hasLeadingSpace(); |
| 1023 |
bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro(); |
| 1024 |
|
| 1025 |
SourceLocation Start = FilenameTok.getLocation(); |
| 1026 |
SourceLocation End; |
| 1027 |
FilenameBuffer.push_back('<'); |
| 1028 |
|
| 1029 |
// Consume tokens until we find a '>'. |
| 1030 |
// FIXME: A header-name could be formed starting or ending with an |
| 1031 |
// alternative token. It's not clear whether that's ill-formed in all |
| 1032 |
// cases. |
| 1033 |
while (FilenameTok.isNot(tok::greater)) { |
| 1034 |
Lex(FilenameTok); |
| 1035 |
if (FilenameTok.isOneOf(tok::eod, tok::eof)) { |
| 1036 |
Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater; |
| 1037 |
Diag(Start, diag::note_matching) << tok::less; |
| 1038 |
return true; |
| 1039 |
} |
| 1040 |
|
| 1041 |
End = FilenameTok.getLocation(); |
| 1042 |
|
| 1043 |
// FIXME: Provide code completion for #includes. |
| 1044 |
if (FilenameTok.is(tok::code_completion)) { |
| 1045 |
setCodeCompletionReached(); |
| 1046 |
Lex(FilenameTok); |
| 1047 |
continue; |
| 1048 |
} |
| 1049 |
|
| 1050 |
// Append the spelling of this token to the buffer. If there was a space |
| 1051 |
// before it, add it now. |
| 1052 |
if (FilenameTok.hasLeadingSpace()) |
| 1053 |
FilenameBuffer.push_back(' '); |
| 1054 |
|
| 1055 |
// Get the spelling of the token, directly into FilenameBuffer if |
| 1056 |
// possible. |
| 1057 |
size_t PreAppendSize = FilenameBuffer.size(); |
| 1058 |
FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength()); |
| 1059 |
|
| 1060 |
const char *BufPtr = &FilenameBuffer[PreAppendSize]; |
| 1061 |
unsigned ActualLen = getSpelling(FilenameTok, BufPtr); |
| 1062 |
|
| 1063 |
// If the token was spelled somewhere else, copy it into FilenameBuffer. |
| 1064 |
if (BufPtr != &FilenameBuffer[PreAppendSize]) |
| 1065 |
memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen); |
| 1066 |
|
| 1067 |
// Resize FilenameBuffer to the correct size. |
| 1068 |
if (FilenameTok.getLength() != ActualLen) |
| 1069 |
FilenameBuffer.resize(PreAppendSize + ActualLen); |
| 1070 |
} |
| 1071 |
|
| 1072 |
FilenameTok.startToken(); |
| 1073 |
FilenameTok.setKind(tok::header_name); |
| 1074 |
FilenameTok.setFlagValue(Token::StartOfLine, StartOfLine); |
| 1075 |
FilenameTok.setFlagValue(Token::LeadingSpace, LeadingSpace); |
| 1076 |
FilenameTok.setFlagValue(Token::LeadingEmptyMacro, LeadingEmptyMacro); |
| 1077 |
CreateString(FilenameBuffer, FilenameTok, Start, End); |
| 1078 |
} else if (FilenameTok.is(tok::string_literal) && AllowMacroExpansion) { |
| 1079 |
// Convert a string-literal token of the form " h-char-sequence " |
| 1080 |
// (produced by macro expansion) into a header-name token. |
| 1081 |
// |
| 1082 |
// The rules for header-names don't quite match the rules for |
| 1083 |
// string-literals, but all the places where they differ result in |
| 1084 |
// undefined behavior, so we can and do treat them the same. |
| 1085 |
// |
| 1086 |
// A string-literal with a prefix or suffix is not translated into a |
| 1087 |
// header-name. This could theoretically be observable via the C++20 |
| 1088 |
// context-sensitive header-name formation rules. |
| 1089 |
StringRef Str = getSpelling(FilenameTok, FilenameBuffer); |
| 1090 |
if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"') |
| 1091 |
FilenameTok.setKind(tok::header_name); |
| 1092 |
} |
| 1093 |
|
| 1094 |
return false; |
| 1095 |
} |
| 1096 |
|
| 1097 |
/// Collect the tokens of a C++20 pp-import-suffix. |
| 1098 |
void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) { |
| 1099 |
// FIXME: For error recovery, consider recognizing attribute syntax here |
| 1100 |
// and terminating / diagnosing a missing semicolon if we find anything |
| 1101 |
// else? (Can we leave that to the parser?) |
| 1102 |
unsigned BracketDepth = 0; |
| 1103 |
while (true) { |
| 1104 |
Toks.emplace_back(); |
| 1105 |
Lex(Toks.back()); |
| 1106 |
|
| 1107 |
switch (Toks.back().getKind()) { |
| 1108 |
case tok::l_paren: case tok::l_square: case tok::l_brace: |
| 1109 |
++BracketDepth; |
| 1110 |
break; |
| 1111 |
|
| 1112 |
case tok::r_paren: case tok::r_square: case tok::r_brace: |
| 1113 |
if (BracketDepth == 0) |
| 1114 |
return; |
| 1115 |
--BracketDepth; |
| 1116 |
break; |
| 1117 |
|
| 1118 |
case tok::semi: |
| 1119 |
if (BracketDepth == 0) |
| 1120 |
return; |
| 1121 |
break; |
| 1122 |
|
| 1123 |
case tok::eof: |
| 1124 |
return; |
| 1125 |
|
| 1126 |
default: |
| 1127 |
break; |
| 1128 |
} |
| 1129 |
} |
| 1130 |
} |
| 1131 |
|
| 1132 |
|
| 1133 |
/// Lex a token following the 'import' contextual keyword. |
| 1134 |
/// |
| 1135 |
/// pp-import: [C++20] |
| 1136 |
/// import header-name pp-import-suffix[opt] ; |
| 1137 |
/// import header-name-tokens pp-import-suffix[opt] ; |
| 1138 |
/// [ObjC] @ import module-name ; |
| 1139 |
/// [Clang] import module-name ; |
| 1140 |
/// |
| 1141 |
/// header-name-tokens: |
| 1142 |
/// string-literal |
| 1143 |
/// < [any sequence of preprocessing-tokens other than >] > |
| 1144 |
/// |
| 1145 |
/// module-name: |
| 1146 |
/// module-name-qualifier[opt] identifier |
| 1147 |
/// |
| 1148 |
/// module-name-qualifier |
| 1149 |
/// module-name-qualifier[opt] identifier . |
| 1150 |
/// |
| 1151 |
/// We respond to a pp-import by importing macros from the named module. |
| 1152 |
bool Preprocessor::LexAfterModuleImport(Token &Result) { |
| 1153 |
// Figure out what kind of lexer we actually have. |
| 1154 |
recomputeCurLexerKind(); |
| 1155 |
|
| 1156 |
// Lex the next token. The header-name lexing rules are used at the start of |
| 1157 |
// a pp-import. |
| 1158 |
// |
| 1159 |
// For now, we only support header-name imports in C++20 mode. |
| 1160 |
// FIXME: Should we allow this in all language modes that support an import |
| 1161 |
// declaration as an extension? |
| 1162 |
if (NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules) { |
| 1163 |
if (LexHeaderName(Result)) |
| 1164 |
return true; |
| 1165 |
|
| 1166 |
if (Result.is(tok::colon) && ModuleDeclState.isNamedModule()) { |
| 1167 |
std::string Name = ModuleDeclState.getPrimaryName().str(); |
| 1168 |
Name += ":"; |
| 1169 |
NamedModuleImportPath.push_back( |
| 1170 |
{getIdentifierInfo(Name), Result.getLocation()}); |
| 1171 |
CurLexerKind = CLK_LexAfterModuleImport; |
| 1172 |
return true; |
| 1173 |
} |
| 1174 |
} else { |
| 1175 |
Lex(Result); |
| 1176 |
} |
| 1177 |
|
| 1178 |
// Allocate a holding buffer for a sequence of tokens and introduce it into |
| 1179 |
// the token stream. |
| 1180 |
auto EnterTokens = [this](ArrayRef<Token> Toks) { |
| 1181 |
auto ToksCopy = std::make_unique<Token[]>(Toks.size()); |
| 1182 |
std::copy(Toks.begin(), Toks.end(), ToksCopy.get()); |
| 1183 |
EnterTokenStream(std::move(ToksCopy), Toks.size(), |
| 1184 |
/*DisableMacroExpansion*/ true, /*IsReinject*/ false); |
| 1185 |
}; |
| 1186 |
|
| 1187 |
bool ImportingHeader = Result.is(tok::header_name); |
| 1188 |
// Check for a header-name. |
| 1189 |
SmallVector<Token, 32> Suffix; |
| 1190 |
if (ImportingHeader) { |
| 1191 |
// Enter the header-name token into the token stream; a Lex action cannot |
| 1192 |
// both return a token and cache tokens (doing so would corrupt the token |
| 1193 |
// cache if the call to Lex comes from CachingLex / PeekAhead). |
| 1194 |
Suffix.push_back(Result); |
| 1195 |
|
| 1196 |
// Consume the pp-import-suffix and expand any macros in it now. We'll add |
| 1197 |
// it back into the token stream later. |
| 1198 |
CollectPpImportSuffix(Suffix); |
| 1199 |
if (Suffix.back().isNot(tok::semi)) { |
| 1200 |
// This is not a pp-import after all. |
| 1201 |
EnterTokens(Suffix); |
| 1202 |
return false; |
| 1203 |
} |
| 1204 |
|
| 1205 |
// C++2a [cpp.module]p1: |
| 1206 |
// The ';' preprocessing-token terminating a pp-import shall not have |
| 1207 |
// been produced by macro replacement. |
| 1208 |
SourceLocation SemiLoc = Suffix.back().getLocation(); |
| 1209 |
if (SemiLoc.isMacroID()) |
| 1210 |
Diag(SemiLoc, diag::err_header_import_semi_in_macro); |
| 1211 |
|
| 1212 |
// Reconstitute the import token. |
| 1213 |
Token ImportTok; |
| 1214 |
ImportTok.startToken(); |
| 1215 |
ImportTok.setKind(tok::kw_import); |
| 1216 |
ImportTok.setLocation(ModuleImportLoc); |
| 1217 |
ImportTok.setIdentifierInfo(getIdentifierInfo("import")); |
| 1218 |
ImportTok.setLength(6); |
| 1219 |
|
| 1220 |
auto Action = HandleHeaderIncludeOrImport( |
| 1221 |
/*HashLoc*/ SourceLocation(), ImportTok, Suffix.front(), SemiLoc); |
| 1222 |
switch (Action.Kind) { |
| 1223 |
case ImportAction::None: |
| 1224 |
break; |
| 1225 |
|
| 1226 |
case ImportAction::ModuleBegin: |
| 1227 |
// Let the parser know we're textually entering the module. |
| 1228 |
Suffix.emplace_back(); |
| 1229 |
Suffix.back().startToken(); |
| 1230 |
Suffix.back().setKind(tok::annot_module_begin); |
| 1231 |
Suffix.back().setLocation(SemiLoc); |
| 1232 |
Suffix.back().setAnnotationEndLoc(SemiLoc); |
| 1233 |
Suffix.back().setAnnotationValue(Action.ModuleForHeader); |
| 1234 |
[[fallthrough]]; |
| 1235 |
|
| 1236 |
case ImportAction::ModuleImport: |
| 1237 |
case ImportAction::HeaderUnitImport: |
| 1238 |
case ImportAction::SkippedModuleImport: |
| 1239 |
// We chose to import (or textually enter) the file. Convert the |
| 1240 |
// header-name token into a header unit annotation token. |
| 1241 |
Suffix[0].setKind(tok::annot_header_unit); |
| 1242 |
Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation()); |
| 1243 |
Suffix[0].setAnnotationValue(Action.ModuleForHeader); |
| 1244 |
// FIXME: Call the moduleImport callback? |
| 1245 |
break; |
| 1246 |
case ImportAction::Failure: |
| 1247 |
assert(TheModuleLoader.HadFatalFailure && |
| 1248 |
"This should be an early exit only to a fatal error"); |
| 1249 |
Result.setKind(tok::eof); |
| 1250 |
CurLexer->cutOffLexing(); |
| 1251 |
EnterTokens(Suffix); |
| 1252 |
return true; |
| 1253 |
} |
| 1254 |
|
| 1255 |
EnterTokens(Suffix); |
| 1256 |
return false; |
| 1257 |
} |
| 1258 |
|
| 1259 |
// The token sequence |
| 1260 |
// |
| 1261 |
// import identifier (. identifier)* |
| 1262 |
// |
| 1263 |
// indicates a module import directive. We already saw the 'import' |
| 1264 |
// contextual keyword, so now we're looking for the identifiers. |
| 1265 |
if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) { |
| 1266 |
// We expected to see an identifier here, and we did; continue handling |
| 1267 |
// identifiers. |
| 1268 |
NamedModuleImportPath.push_back( |
| 1269 |
std::make_pair(Result.getIdentifierInfo(), Result.getLocation())); |
| 1270 |
ModuleImportExpectsIdentifier = false; |
| 1271 |
CurLexerKind = CLK_LexAfterModuleImport; |
| 1272 |
return true; |
| 1273 |
} |
| 1274 |
|
| 1275 |
// If we're expecting a '.' or a ';', and we got a '.', then wait until we |
| 1276 |
// see the next identifier. (We can also see a '[[' that begins an |
| 1277 |
// attribute-specifier-seq here under the C++ Modules TS.) |
| 1278 |
if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) { |
| 1279 |
ModuleImportExpectsIdentifier = true; |
| 1280 |
CurLexerKind = CLK_LexAfterModuleImport; |
| 1281 |
return true; |
| 1282 |
} |
| 1283 |
|
| 1284 |
// If we didn't recognize a module name at all, this is not a (valid) import. |
| 1285 |
if (NamedModuleImportPath.empty() || Result.is(tok::eof)) |
| 1286 |
return true; |
| 1287 |
|
| 1288 |
// Consume the pp-import-suffix and expand any macros in it now, if we're not |
| 1289 |
// at the semicolon already. |
| 1290 |
SourceLocation SemiLoc = Result.getLocation(); |
| 1291 |
if (Result.isNot(tok::semi)) { |
| 1292 |
Suffix.push_back(Result); |
| 1293 |
CollectPpImportSuffix(Suffix); |
| 1294 |
if (Suffix.back().isNot(tok::semi)) { |
| 1295 |
// This is not an import after all. |
| 1296 |
EnterTokens(Suffix); |
| 1297 |
return false; |
| 1298 |
} |
| 1299 |
SemiLoc = Suffix.back().getLocation(); |
| 1300 |
} |
| 1301 |
|
| 1302 |
// Under the Modules TS, the dot is just part of the module name, and not |
| 1303 |
// a real hierarchy separator. Flatten such module names now. |
| 1304 |
// |
| 1305 |
// FIXME: Is this the right level to be performing this transformation? |
| 1306 |
std::string FlatModuleName; |
| 1307 |
if (getLangOpts().ModulesTS || getLangOpts().CPlusPlusModules) { |
| 1308 |
for (auto &Piece : NamedModuleImportPath) { |
| 1309 |
// If the FlatModuleName ends with colon, it implies it is a partition. |
| 1310 |
if (!FlatModuleName.empty() && FlatModuleName.back() != ':') |
| 1311 |
FlatModuleName += "."; |
| 1312 |
FlatModuleName += Piece.first->getName(); |
| 1313 |
} |
| 1314 |
SourceLocation FirstPathLoc = NamedModuleImportPath[0].second; |
| 1315 |
NamedModuleImportPath.clear(); |
| 1316 |
NamedModuleImportPath.push_back( |
| 1317 |
std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc)); |
| 1318 |
} |
| 1319 |
|
| 1320 |
Module *Imported = nullptr; |
| 1321 |
// We don't/shouldn't load the standard c++20 modules when preprocessing. |
| 1322 |
if (getLangOpts().Modules && !isInImportingCXXNamedModules()) { |
| 1323 |
Imported = TheModuleLoader.loadModule(ModuleImportLoc, |
| 1324 |
NamedModuleImportPath, |
| 1325 |
Module::Hidden, |
| 1326 |
/*IsInclusionDirective=*/false); |
| 1327 |
if (Imported) |
| 1328 |
makeModuleVisible(Imported, SemiLoc); |
| 1329 |
} |
| 1330 |
|
| 1331 |
if (Callbacks) |
| 1332 |
Callbacks->moduleImport(ModuleImportLoc, NamedModuleImportPath, Imported); |
| 1333 |
|
| 1334 |
if (!Suffix.empty()) { |
| 1335 |
EnterTokens(Suffix); |
| 1336 |
return false; |
| 1337 |
} |
| 1338 |
return true; |
| 1339 |
} |
| 1340 |
|
| 1341 |
void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc) { |
| 1342 |
CurSubmoduleState->VisibleModules.setVisible( |
| 1343 |
M, Loc, [](Module *) {}, |
| 1344 |
[&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) { |
| 1345 |
// FIXME: Include the path in the diagnostic. |
| 1346 |
// FIXME: Include the import location for the conflicting module. |
| 1347 |
Diag(ModuleImportLoc, diag::warn_module_conflict) |
| 1348 |
<< Path[0]->getFullModuleName() |
| 1349 |
<< Conflict->getFullModuleName() |
| 1350 |
<< Message; |
| 1351 |
}); |
| 1352 |
|
| 1353 |
// Add this module to the imports list of the currently-built submodule. |
| 1354 |
if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M) |
| 1355 |
BuildingSubmoduleStack.back().M->Imports.insert(M); |
| 1356 |
} |
| 1357 |
|
| 1358 |
bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String, |
| 1359 |
const char *DiagnosticTag, |
| 1360 |
bool AllowMacroExpansion) { |
| 1361 |
// We need at least one string literal. |
| 1362 |
if (Result.isNot(tok::string_literal)) { |
| 1363 |
Diag(Result, diag::err_expected_string_literal) |
| 1364 |
<< /*Source='in...'*/0 << DiagnosticTag; |
| 1365 |
return false; |
| 1366 |
} |
| 1367 |
|
| 1368 |
// Lex string literal tokens, optionally with macro expansion. |
| 1369 |
SmallVector<Token, 4> StrToks; |
| 1370 |
do { |
| 1371 |
StrToks.push_back(Result); |
| 1372 |
|
| 1373 |
if (Result.hasUDSuffix()) |
| 1374 |
Diag(Result, diag::err_invalid_string_udl); |
| 1375 |
|
| 1376 |
if (AllowMacroExpansion) |
| 1377 |
Lex(Result); |
| 1378 |
else |
| 1379 |
LexUnexpandedToken(Result); |
| 1380 |
} while (Result.is(tok::string_literal)); |
| 1381 |
|
| 1382 |
// Concatenate and parse the strings. |
| 1383 |
StringLiteralParser Literal(StrToks, *this); |
| 1384 |
assert(Literal.isOrdinary() && "Didn't allow wide strings in"); |
| 1385 |
|
| 1386 |
if (Literal.hadError) |
| 1387 |
return false; |
| 1388 |
|
| 1389 |
if (Literal.Pascal) { |
| 1390 |
Diag(StrToks[0].getLocation(), diag::err_expected_string_literal) |
| 1391 |
<< /*Source='in...'*/0 << DiagnosticTag; |
| 1392 |
return false; |
| 1393 |
} |
| 1394 |
|
| 1395 |
String = std::string(Literal.GetString()); |
| 1396 |
return true; |
| 1397 |
} |
| 1398 |
|
| 1399 |
bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) { |
| 1400 |
assert(Tok.is(tok::numeric_constant)); |
| 1401 |
SmallString<8> IntegerBuffer; |
| 1402 |
bool NumberInvalid = false; |
| 1403 |
StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid); |
| 1404 |
if (NumberInvalid) |
| 1405 |
return false; |
| 1406 |
NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(), |
| 1407 |
getLangOpts(), getTargetInfo(), |
| 1408 |
getDiagnostics()); |
| 1409 |
if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix()) |
| 1410 |
return false; |
| 1411 |
llvm::APInt APVal(64, 0); |
| 1412 |
if (Literal.GetIntegerValue(APVal)) |
| 1413 |
return false; |
| 1414 |
Lex(Tok); |
| 1415 |
Value = APVal.getLimitedValue(); |
| 1416 |
return true; |
| 1417 |
} |
| 1418 |
|
| 1419 |
void Preprocessor::addCommentHandler(CommentHandler *Handler) { |
| 1420 |
assert(Handler && "NULL comment handler"); |
| 1421 |
assert(!llvm::is_contained(CommentHandlers, Handler) && |
| 1422 |
"Comment handler already registered"); |
| 1423 |
CommentHandlers.push_back(Handler); |
| 1424 |
} |
| 1425 |
|
| 1426 |
void Preprocessor::removeCommentHandler(CommentHandler *Handler) { |
| 1427 |
std::vector<CommentHandler *>::iterator Pos = |
| 1428 |
llvm::find(CommentHandlers, Handler); |
| 1429 |
assert(Pos != CommentHandlers.end() && "Comment handler not registered"); |
| 1430 |
CommentHandlers.erase(Pos); |
| 1431 |
} |
| 1432 |
|
| 1433 |
bool Preprocessor::HandleComment(Token &result, SourceRange Comment) { |
| 1434 |
bool AnyPendingTokens = false; |
| 1435 |
for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(), |
| 1436 |
HEnd = CommentHandlers.end(); |
| 1437 |
H != HEnd; ++H) { |
| 1438 |
if ((*H)->HandleComment(*this, Comment)) |
| 1439 |
AnyPendingTokens = true; |
| 1440 |
} |
| 1441 |
if (!AnyPendingTokens || getCommentRetentionState()) |
| 1442 |
return false; |
| 1443 |
Lex(result); |
| 1444 |
return true; |
| 1445 |
} |
| 1446 |
|
| 1447 |
void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const { |
| 1448 |
const MacroAnnotations &A = |
| 1449 |
getMacroAnnotations(Identifier.getIdentifierInfo()); |
| 1450 |
assert(A.DeprecationInfo && |
| 1451 |
"Macro deprecation warning without recorded annotation!"); |
| 1452 |
const MacroAnnotationInfo &Info = *A.DeprecationInfo; |
| 1453 |
if (Info.Message.empty()) |
| 1454 |
Diag(Identifier, diag::warn_pragma_deprecated_macro_use) |
| 1455 |
<< Identifier.getIdentifierInfo() << 0; |
| 1456 |
else |
| 1457 |
Diag(Identifier, diag::warn_pragma_deprecated_macro_use) |
| 1458 |
<< Identifier.getIdentifierInfo() << 1 << Info.Message; |
| 1459 |
Diag(Info.Location, diag::note_pp_macro_annotation) << 0; |
| 1460 |
} |
| 1461 |
|
| 1462 |
void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const { |
| 1463 |
const MacroAnnotations &A = |
| 1464 |
getMacroAnnotations(Identifier.getIdentifierInfo()); |
| 1465 |
assert(A.RestrictExpansionInfo && |
| 1466 |
"Macro restricted expansion warning without recorded annotation!"); |
| 1467 |
const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo; |
| 1468 |
if (Info.Message.empty()) |
| 1469 |
Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use) |
| 1470 |
<< Identifier.getIdentifierInfo() << 0; |
| 1471 |
else |
| 1472 |
Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use) |
| 1473 |
<< Identifier.getIdentifierInfo() << 1 << Info.Message; |
| 1474 |
Diag(Info.Location, diag::note_pp_macro_annotation) << 1; |
| 1475 |
} |
| 1476 |
|
| 1477 |
void Preprocessor::emitFinalMacroWarning(const Token &Identifier, |
| 1478 |
bool IsUndef) const { |
| 1479 |
const MacroAnnotations &A = |
| 1480 |
getMacroAnnotations(Identifier.getIdentifierInfo()); |
| 1481 |
assert(A.FinalAnnotationLoc && |
| 1482 |
"Final macro warning without recorded annotation!"); |
| 1483 |
|
| 1484 |
Diag(Identifier, diag::warn_pragma_final_macro) |
| 1485 |
<< Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1); |
| 1486 |
Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2; |
| 1487 |
} |
| 1488 |
|
| 1489 |
ModuleLoader::~ModuleLoader() = default; |
| 1490 |
|
| 1491 |
CommentHandler::~CommentHandler() = default; |
| 1492 |
|
| 1493 |
EmptylineHandler::~EmptylineHandler() = default; |
| 1494 |
|
| 1495 |
CodeCompletionHandler::~CodeCompletionHandler() = default; |
| 1496 |
|
| 1497 |
void Preprocessor::createPreprocessingRecord() { |
| 1498 |
if (Record) |
| 1499 |
return; |
| 1500 |
|
| 1501 |
Record = new PreprocessingRecord(getSourceManager()); |
| 1502 |
addPPCallbacks(std::unique_ptr<PPCallbacks>(Record)); |
| 1503 |
} |