| 1 |
//===- TokenLexer.cpp - Lex from a token stream ---------------------------===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// This file implements the TokenLexer interface. |
| 10 |
// |
| 11 |
//===----------------------------------------------------------------------===// |
| 12 |
|
| 13 |
#include "clang/Lex/TokenLexer.h" |
| 14 |
#include "clang/Basic/Diagnostic.h" |
| 15 |
#include "clang/Basic/IdentifierTable.h" |
| 16 |
#include "clang/Basic/LangOptions.h" |
| 17 |
#include "clang/Basic/SourceLocation.h" |
| 18 |
#include "clang/Basic/SourceManager.h" |
| 19 |
#include "clang/Basic/TokenKinds.h" |
| 20 |
#include "clang/Lex/LexDiagnostic.h" |
| 21 |
#include "clang/Lex/Lexer.h" |
| 22 |
#include "clang/Lex/MacroArgs.h" |
| 23 |
#include "clang/Lex/MacroInfo.h" |
| 24 |
#include "clang/Lex/Preprocessor.h" |
| 25 |
#include "clang/Lex/Token.h" |
| 26 |
#include "clang/Lex/VariadicMacroSupport.h" |
| 27 |
#include "llvm/ADT/ArrayRef.h" |
| 28 |
#include "llvm/ADT/STLExtras.h" |
| 29 |
#include "llvm/ADT/SmallString.h" |
| 30 |
#include "llvm/ADT/SmallVector.h" |
| 31 |
#include "llvm/ADT/iterator_range.h" |
| 32 |
#include <cassert> |
| 33 |
#include <cstring> |
| 34 |
#include <optional> |
| 35 |
|
| 36 |
using namespace clang; |
| 37 |
|
| 38 |
/// Create a TokenLexer for the specified macro with the specified actual |
| 39 |
/// arguments. Note that this ctor takes ownership of the ActualArgs pointer. |
| 40 |
void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, |
| 41 |
MacroArgs *Actuals) { |
| 42 |
// If the client is reusing a TokenLexer, make sure to free any memory |
| 43 |
// associated with it. |
| 44 |
destroy(); |
| 45 |
|
| 46 |
Macro = MI; |
| 47 |
ActualArgs = Actuals; |
| 48 |
CurTokenIdx = 0; |
| 49 |
|
| 50 |
ExpandLocStart = Tok.getLocation(); |
| 51 |
ExpandLocEnd = ELEnd; |
| 52 |
AtStartOfLine = Tok.isAtStartOfLine(); |
| 53 |
HasLeadingSpace = Tok.hasLeadingSpace(); |
| 54 |
NextTokGetsSpace = false; |
| 55 |
Tokens = &*Macro->tokens_begin(); |
| 56 |
OwnsTokens = false; |
| 57 |
DisableMacroExpansion = false; |
| 58 |
IsReinject = false; |
| 59 |
NumTokens = Macro->tokens_end()-Macro->tokens_begin(); |
| 60 |
MacroExpansionStart = SourceLocation(); |
| 61 |
|
| 62 |
SourceManager &SM = PP.getSourceManager(); |
| 63 |
MacroStartSLocOffset = SM.getNextLocalOffset(); |
| 64 |
|
| 65 |
if (NumTokens > 0) { |
| 66 |
assert(Tokens[0].getLocation().isValid()); |
| 67 |
assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) && |
| 68 |
"Macro defined in macro?"); |
| 69 |
assert(ExpandLocStart.isValid()); |
| 70 |
|
| 71 |
// Reserve a source location entry chunk for the length of the macro |
| 72 |
// definition. Tokens that get lexed directly from the definition will |
| 73 |
// have their locations pointing inside this chunk. This is to avoid |
| 74 |
// creating separate source location entries for each token. |
| 75 |
MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation()); |
| 76 |
MacroDefLength = Macro->getDefinitionLength(SM); |
| 77 |
MacroExpansionStart = SM.createExpansionLoc(MacroDefStart, |
| 78 |
ExpandLocStart, |
| 79 |
ExpandLocEnd, |
| 80 |
MacroDefLength); |
| 81 |
} |
| 82 |
|
| 83 |
// If this is a function-like macro, expand the arguments and change |
| 84 |
// Tokens to point to the expanded tokens. |
| 85 |
if (Macro->isFunctionLike() && Macro->getNumParams()) |
| 86 |
ExpandFunctionArguments(); |
| 87 |
|
| 88 |
// Mark the macro as currently disabled, so that it is not recursively |
| 89 |
// expanded. The macro must be disabled only after argument pre-expansion of |
| 90 |
// function-like macro arguments occurs. |
| 91 |
Macro->DisableMacro(); |
| 92 |
} |
| 93 |
|
| 94 |
/// Create a TokenLexer for the specified token stream. This does not |
| 95 |
/// take ownership of the specified token vector. |
| 96 |
void TokenLexer::Init(const Token *TokArray, unsigned NumToks, |
| 97 |
bool disableMacroExpansion, bool ownsTokens, |
| 98 |
bool isReinject) { |
| 99 |
assert(!isReinject || disableMacroExpansion); |
| 100 |
// If the client is reusing a TokenLexer, make sure to free any memory |
| 101 |
// associated with it. |
| 102 |
destroy(); |
| 103 |
|
| 104 |
Macro = nullptr; |
| 105 |
ActualArgs = nullptr; |
| 106 |
Tokens = TokArray; |
| 107 |
OwnsTokens = ownsTokens; |
| 108 |
DisableMacroExpansion = disableMacroExpansion; |
| 109 |
IsReinject = isReinject; |
| 110 |
NumTokens = NumToks; |
| 111 |
CurTokenIdx = 0; |
| 112 |
ExpandLocStart = ExpandLocEnd = SourceLocation(); |
| 113 |
AtStartOfLine = false; |
| 114 |
HasLeadingSpace = false; |
| 115 |
NextTokGetsSpace = false; |
| 116 |
MacroExpansionStart = SourceLocation(); |
| 117 |
|
| 118 |
// Set HasLeadingSpace/AtStartOfLine so that the first token will be |
| 119 |
// returned unmodified. |
| 120 |
if (NumToks != 0) { |
| 121 |
AtStartOfLine = TokArray[0].isAtStartOfLine(); |
| 122 |
HasLeadingSpace = TokArray[0].hasLeadingSpace(); |
| 123 |
} |
| 124 |
} |
| 125 |
|
| 126 |
void TokenLexer::destroy() { |
| 127 |
// If this was a function-like macro that actually uses its arguments, delete |
| 128 |
// the expanded tokens. |
| 129 |
if (OwnsTokens) { |
| 130 |
delete [] Tokens; |
| 131 |
Tokens = nullptr; |
| 132 |
OwnsTokens = false; |
| 133 |
} |
| 134 |
|
| 135 |
// TokenLexer owns its formal arguments. |
| 136 |
if (ActualArgs) ActualArgs->destroy(PP); |
| 137 |
} |
| 138 |
|
| 139 |
bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( |
| 140 |
SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro, |
| 141 |
unsigned MacroArgNo, Preprocessor &PP) { |
| 142 |
// Is the macro argument __VA_ARGS__? |
| 143 |
if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1) |
| 144 |
return false; |
| 145 |
|
| 146 |
// In Microsoft-compatibility mode, a comma is removed in the expansion |
| 147 |
// of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is |
| 148 |
// not supported by gcc. |
| 149 |
if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat) |
| 150 |
return false; |
| 151 |
|
| 152 |
// GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if |
| 153 |
// __VA_ARGS__ is empty, but not in strict C99 mode where there are no |
| 154 |
// named arguments, where it remains. In all other modes, including C99 |
| 155 |
// with GNU extensions, it is removed regardless of named arguments. |
| 156 |
// Microsoft also appears to support this extension, unofficially. |
| 157 |
if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode |
| 158 |
&& Macro->getNumParams() < 2) |
| 159 |
return false; |
| 160 |
|
| 161 |
// Is a comma available to be removed? |
| 162 |
if (ResultToks.empty() || !ResultToks.back().is(tok::comma)) |
| 163 |
return false; |
| 164 |
|
| 165 |
// Issue an extension diagnostic for the paste operator. |
| 166 |
if (HasPasteOperator) |
| 167 |
PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); |
| 168 |
|
| 169 |
// Remove the comma. |
| 170 |
ResultToks.pop_back(); |
| 171 |
|
| 172 |
if (!ResultToks.empty()) { |
| 173 |
// If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"), |
| 174 |
// then removal of the comma should produce a placemarker token (in C99 |
| 175 |
// terms) which we model by popping off the previous ##, giving us a plain |
| 176 |
// "X" when __VA_ARGS__ is empty. |
| 177 |
if (ResultToks.back().is(tok::hashhash)) |
| 178 |
ResultToks.pop_back(); |
| 179 |
|
| 180 |
// Remember that this comma was elided. |
| 181 |
ResultToks.back().setFlag(Token::CommaAfterElided); |
| 182 |
} |
| 183 |
|
| 184 |
// Never add a space, even if the comma, ##, or arg had a space. |
| 185 |
NextTokGetsSpace = false; |
| 186 |
return true; |
| 187 |
} |
| 188 |
|
| 189 |
void TokenLexer::stringifyVAOPTContents( |
| 190 |
SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx, |
| 191 |
const SourceLocation VAOPTClosingParenLoc) { |
| 192 |
const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt(); |
| 193 |
const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt; |
| 194 |
Token *const VAOPTTokens = |
| 195 |
NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr; |
| 196 |
|
| 197 |
SmallVector<Token, 64> ConcatenatedVAOPTResultToks; |
| 198 |
// FIXME: Should we keep track within VCtx that we did or didnot |
| 199 |
// encounter pasting - and only then perform this loop. |
| 200 |
|
| 201 |
// Perform token pasting (concatenation) prior to stringization. |
| 202 |
for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens; |
| 203 |
++CurTokenIdx) { |
| 204 |
if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) { |
| 205 |
assert(CurTokenIdx != 0 && |
| 206 |
"Can not have __VAOPT__ contents begin with a ##"); |
| 207 |
Token &LHS = VAOPTTokens[CurTokenIdx - 1]; |
| 208 |
pasteTokens(LHS, llvm::ArrayRef(VAOPTTokens, NumVAOptTokens), |
| 209 |
CurTokenIdx); |
| 210 |
// Replace the token prior to the first ## in this iteration. |
| 211 |
ConcatenatedVAOPTResultToks.back() = LHS; |
| 212 |
if (CurTokenIdx == NumVAOptTokens) |
| 213 |
break; |
| 214 |
} |
| 215 |
ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]); |
| 216 |
} |
| 217 |
|
| 218 |
ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok()); |
| 219 |
// Get the SourceLocation that represents the start location within |
| 220 |
// the macro definition that marks where this string is substituted |
| 221 |
// into: i.e. the __VA_OPT__ and the ')' within the spelling of the |
| 222 |
// macro definition, and use it to indicate that the stringified token |
| 223 |
// was generated from that location. |
| 224 |
const SourceLocation ExpansionLocStartWithinMacro = |
| 225 |
getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc()); |
| 226 |
const SourceLocation ExpansionLocEndWithinMacro = |
| 227 |
getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc); |
| 228 |
|
| 229 |
Token StringifiedVAOPT = MacroArgs::StringifyArgument( |
| 230 |
&ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/, |
| 231 |
ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro); |
| 232 |
|
| 233 |
if (VCtx.getLeadingSpaceForStringifiedToken()) |
| 234 |
StringifiedVAOPT.setFlag(Token::LeadingSpace); |
| 235 |
|
| 236 |
StringifiedVAOPT.setFlag(Token::StringifiedInMacro); |
| 237 |
// Resize (shrink) the token stream to just capture this stringified token. |
| 238 |
ResultToks.resize(NumToksPriorToVAOpt + 1); |
| 239 |
ResultToks.back() = StringifiedVAOPT; |
| 240 |
} |
| 241 |
|
| 242 |
/// Expand the arguments of a function-like macro so that we can quickly |
| 243 |
/// return preexpanded tokens from Tokens. |
| 244 |
void TokenLexer::ExpandFunctionArguments() { |
| 245 |
SmallVector<Token, 128> ResultToks; |
| 246 |
|
| 247 |
// Loop through 'Tokens', expanding them into ResultToks. Keep |
| 248 |
// track of whether we change anything. If not, no need to keep them. If so, |
| 249 |
// we install the newly expanded sequence as the new 'Tokens' list. |
| 250 |
bool MadeChange = false; |
| 251 |
|
| 252 |
std::optional<bool> CalledWithVariadicArguments; |
| 253 |
|
| 254 |
VAOptExpansionContext VCtx(PP); |
| 255 |
|
| 256 |
for (unsigned I = 0, E = NumTokens; I != E; ++I) { |
| 257 |
const Token &CurTok = Tokens[I]; |
| 258 |
// We don't want a space for the next token after a paste |
| 259 |
// operator. In valid code, the token will get smooshed onto the |
| 260 |
// preceding one anyway. In assembler-with-cpp mode, invalid |
| 261 |
// pastes are allowed through: in this case, we do not want the |
| 262 |
// extra whitespace to be added. For example, we want ". ## foo" |
| 263 |
// -> ".foo" not ". foo". |
| 264 |
if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace()) |
| 265 |
NextTokGetsSpace = true; |
| 266 |
|
| 267 |
if (VCtx.isVAOptToken(CurTok)) { |
| 268 |
MadeChange = true; |
| 269 |
assert(Tokens[I + 1].is(tok::l_paren) && |
| 270 |
"__VA_OPT__ must be followed by '('"); |
| 271 |
|
| 272 |
++I; // Skip the l_paren |
| 273 |
VCtx.sawVAOptFollowedByOpeningParens(CurTok.getLocation(), |
| 274 |
ResultToks.size()); |
| 275 |
|
| 276 |
continue; |
| 277 |
} |
| 278 |
|
| 279 |
// We have entered into the __VA_OPT__ context, so handle tokens |
| 280 |
// appropriately. |
| 281 |
if (VCtx.isInVAOpt()) { |
| 282 |
// If we are about to process a token that is either an argument to |
| 283 |
// __VA_OPT__ or its closing rparen, then: |
| 284 |
// 1) If the token is the closing rparen that exits us out of __VA_OPT__, |
| 285 |
// perform any necessary stringification or placemarker processing, |
| 286 |
// and/or skip to the next token. |
| 287 |
// 2) else if macro was invoked without variadic arguments skip this |
| 288 |
// token. |
| 289 |
// 3) else (macro was invoked with variadic arguments) process the token |
| 290 |
// normally. |
| 291 |
|
| 292 |
if (Tokens[I].is(tok::l_paren)) |
| 293 |
VCtx.sawOpeningParen(Tokens[I].getLocation()); |
| 294 |
// Continue skipping tokens within __VA_OPT__ if the macro was not |
| 295 |
// called with variadic arguments, else let the rest of the loop handle |
| 296 |
// this token. Note sawClosingParen() returns true only if the r_paren matches |
| 297 |
// the closing r_paren of the __VA_OPT__. |
| 298 |
if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) { |
| 299 |
// Lazily expand __VA_ARGS__ when we see the first __VA_OPT__. |
| 300 |
if (!CalledWithVariadicArguments) { |
| 301 |
CalledWithVariadicArguments = |
| 302 |
ActualArgs->invokedWithVariadicArgument(Macro, PP); |
| 303 |
} |
| 304 |
if (!*CalledWithVariadicArguments) { |
| 305 |
// Skip this token. |
| 306 |
continue; |
| 307 |
} |
| 308 |
// ... else the macro was called with variadic arguments, and we do not |
| 309 |
// have a closing rparen - so process this token normally. |
| 310 |
} else { |
| 311 |
// Current token is the closing r_paren which marks the end of the |
| 312 |
// __VA_OPT__ invocation, so handle any place-marker pasting (if |
| 313 |
// empty) by removing hashhash either before (if exists) or after. And |
| 314 |
// also stringify the entire contents if VAOPT was preceded by a hash, |
| 315 |
// but do so only after any token concatenation that needs to occur |
| 316 |
// within the contents of VAOPT. |
| 317 |
|
| 318 |
if (VCtx.hasStringifyOrCharifyBefore()) { |
| 319 |
// Replace all the tokens just added from within VAOPT into a single |
| 320 |
// stringified token. This requires token-pasting to eagerly occur |
| 321 |
// within these tokens. If either the contents of VAOPT were empty |
| 322 |
// or the macro wasn't called with any variadic arguments, the result |
| 323 |
// is a token that represents an empty string. |
| 324 |
stringifyVAOPTContents(ResultToks, VCtx, |
| 325 |
/*ClosingParenLoc*/ Tokens[I].getLocation()); |
| 326 |
|
| 327 |
} else if (/*No tokens within VAOPT*/ |
| 328 |
ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) { |
| 329 |
// Treat VAOPT as a placemarker token. Eat either the '##' before the |
| 330 |
// RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that |
| 331 |
// hashhash was not a placemarker) or the '##' |
| 332 |
// after VAOPT, but not both. |
| 333 |
|
| 334 |
if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) { |
| 335 |
ResultToks.pop_back(); |
| 336 |
} else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) { |
| 337 |
++I; // Skip the following hashhash. |
| 338 |
} |
| 339 |
} else { |
| 340 |
// If there's a ## before the __VA_OPT__, we might have discovered |
| 341 |
// that the __VA_OPT__ begins with a placeholder. We delay action on |
| 342 |
// that to now to avoid messing up our stashed count of tokens before |
| 343 |
// __VA_OPT__. |
| 344 |
if (VCtx.beginsWithPlaceholder()) { |
| 345 |
assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 && |
| 346 |
ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() && |
| 347 |
ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is( |
| 348 |
tok::hashhash) && |
| 349 |
"no token paste before __VA_OPT__"); |
| 350 |
ResultToks.erase(ResultToks.begin() + |
| 351 |
VCtx.getNumberOfTokensPriorToVAOpt() - 1); |
| 352 |
} |
| 353 |
// If the expansion of __VA_OPT__ ends with a placeholder, eat any |
| 354 |
// following '##' token. |
| 355 |
if (VCtx.endsWithPlaceholder() && I + 1 != E && |
| 356 |
Tokens[I + 1].is(tok::hashhash)) { |
| 357 |
++I; |
| 358 |
} |
| 359 |
} |
| 360 |
VCtx.reset(); |
| 361 |
// We processed __VA_OPT__'s closing paren (and the exit out of |
| 362 |
// __VA_OPT__), so skip to the next token. |
| 363 |
continue; |
| 364 |
} |
| 365 |
} |
| 366 |
|
| 367 |
// If we found the stringify operator, get the argument stringified. The |
| 368 |
// preprocessor already verified that the following token is a macro |
| 369 |
// parameter or __VA_OPT__ when the #define was lexed. |
| 370 |
|
| 371 |
if (CurTok.isOneOf(tok::hash, tok::hashat)) { |
| 372 |
int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo()); |
| 373 |
assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) && |
| 374 |
"Token following # is not an argument or __VA_OPT__!"); |
| 375 |
|
| 376 |
if (ArgNo == -1) { |
| 377 |
// Handle the __VA_OPT__ case. |
| 378 |
VCtx.sawHashOrHashAtBefore(NextTokGetsSpace, |
| 379 |
CurTok.is(tok::hashat)); |
| 380 |
continue; |
| 381 |
} |
| 382 |
// Else handle the simple argument case. |
| 383 |
SourceLocation ExpansionLocStart = |
| 384 |
getExpansionLocForMacroDefLoc(CurTok.getLocation()); |
| 385 |
SourceLocation ExpansionLocEnd = |
| 386 |
getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation()); |
| 387 |
|
| 388 |
bool Charify = CurTok.is(tok::hashat); |
| 389 |
const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo); |
| 390 |
Token Res = MacroArgs::StringifyArgument( |
| 391 |
UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd); |
| 392 |
Res.setFlag(Token::StringifiedInMacro); |
| 393 |
|
| 394 |
// The stringified/charified string leading space flag gets set to match |
| 395 |
// the #/#@ operator. |
| 396 |
if (NextTokGetsSpace) |
| 397 |
Res.setFlag(Token::LeadingSpace); |
| 398 |
|
| 399 |
ResultToks.push_back(Res); |
| 400 |
MadeChange = true; |
| 401 |
++I; // Skip arg name. |
| 402 |
NextTokGetsSpace = false; |
| 403 |
continue; |
| 404 |
} |
| 405 |
|
| 406 |
// Find out if there is a paste (##) operator before or after the token. |
| 407 |
bool NonEmptyPasteBefore = |
| 408 |
!ResultToks.empty() && ResultToks.back().is(tok::hashhash); |
| 409 |
bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash); |
| 410 |
bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash); |
| 411 |
bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren); |
| 412 |
|
| 413 |
assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) && |
| 414 |
"unexpected ## in ResultToks"); |
| 415 |
|
| 416 |
// Otherwise, if this is not an argument token, just add the token to the |
| 417 |
// output buffer. |
| 418 |
IdentifierInfo *II = CurTok.getIdentifierInfo(); |
| 419 |
int ArgNo = II ? Macro->getParameterNum(II) : -1; |
| 420 |
if (ArgNo == -1) { |
| 421 |
// This isn't an argument, just add it. |
| 422 |
ResultToks.push_back(CurTok); |
| 423 |
|
| 424 |
if (NextTokGetsSpace) { |
| 425 |
ResultToks.back().setFlag(Token::LeadingSpace); |
| 426 |
NextTokGetsSpace = false; |
| 427 |
} else if (PasteBefore && !NonEmptyPasteBefore) |
| 428 |
ResultToks.back().clearFlag(Token::LeadingSpace); |
| 429 |
|
| 430 |
continue; |
| 431 |
} |
| 432 |
|
| 433 |
// An argument is expanded somehow, the result is different than the |
| 434 |
// input. |
| 435 |
MadeChange = true; |
| 436 |
|
| 437 |
// Otherwise, this is a use of the argument. |
| 438 |
|
| 439 |
// In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there |
| 440 |
// are no trailing commas if __VA_ARGS__ is empty. |
| 441 |
if (!PasteBefore && ActualArgs->isVarargsElidedUse() && |
| 442 |
MaybeRemoveCommaBeforeVaArgs(ResultToks, |
| 443 |
/*HasPasteOperator=*/false, |
| 444 |
Macro, ArgNo, PP)) |
| 445 |
continue; |
| 446 |
|
| 447 |
// If it is not the LHS/RHS of a ## operator, we must pre-expand the |
| 448 |
// argument and substitute the expanded tokens into the result. This is |
| 449 |
// C99 6.10.3.1p1. |
| 450 |
if (!PasteBefore && !PasteAfter) { |
| 451 |
const Token *ResultArgToks; |
| 452 |
|
| 453 |
// Only preexpand the argument if it could possibly need it. This |
| 454 |
// avoids some work in common cases. |
| 455 |
const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); |
| 456 |
if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) |
| 457 |
ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0]; |
| 458 |
else |
| 459 |
ResultArgToks = ArgTok; // Use non-preexpanded tokens. |
| 460 |
|
| 461 |
// If the arg token expanded into anything, append it. |
| 462 |
if (ResultArgToks->isNot(tok::eof)) { |
| 463 |
size_t FirstResult = ResultToks.size(); |
| 464 |
unsigned NumToks = MacroArgs::getArgLength(ResultArgToks); |
| 465 |
ResultToks.append(ResultArgToks, ResultArgToks+NumToks); |
| 466 |
|
| 467 |
// In Microsoft-compatibility mode, we follow MSVC's preprocessing |
| 468 |
// behavior by not considering single commas from nested macro |
| 469 |
// expansions as argument separators. Set a flag on the token so we can |
| 470 |
// test for this later when the macro expansion is processed. |
| 471 |
if (PP.getLangOpts().MSVCCompat && NumToks == 1 && |
| 472 |
ResultToks.back().is(tok::comma)) |
| 473 |
ResultToks.back().setFlag(Token::IgnoredComma); |
| 474 |
|
| 475 |
// If the '##' came from expanding an argument, turn it into 'unknown' |
| 476 |
// to avoid pasting. |
| 477 |
for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult)) |
| 478 |
if (Tok.is(tok::hashhash)) |
| 479 |
Tok.setKind(tok::unknown); |
| 480 |
|
| 481 |
if(ExpandLocStart.isValid()) { |
| 482 |
updateLocForMacroArgTokens(CurTok.getLocation(), |
| 483 |
ResultToks.begin()+FirstResult, |
| 484 |
ResultToks.end()); |
| 485 |
} |
| 486 |
|
| 487 |
// If any tokens were substituted from the argument, the whitespace |
| 488 |
// before the first token should match the whitespace of the arg |
| 489 |
// identifier. |
| 490 |
ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, |
| 491 |
NextTokGetsSpace); |
| 492 |
ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false); |
| 493 |
NextTokGetsSpace = false; |
| 494 |
} else { |
| 495 |
// We're creating a placeholder token. Usually this doesn't matter, |
| 496 |
// but it can affect paste behavior when at the start or end of a |
| 497 |
// __VA_OPT__. |
| 498 |
if (NonEmptyPasteBefore) { |
| 499 |
// We're imagining a placeholder token is inserted here. If this is |
| 500 |
// the first token in a __VA_OPT__ after a ##, delete the ##. |
| 501 |
assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__"); |
| 502 |
VCtx.hasPlaceholderAfterHashhashAtStart(); |
| 503 |
} |
| 504 |
if (RParenAfter) |
| 505 |
VCtx.hasPlaceholderBeforeRParen(); |
| 506 |
} |
| 507 |
continue; |
| 508 |
} |
| 509 |
|
| 510 |
// Okay, we have a token that is either the LHS or RHS of a paste (##) |
| 511 |
// argument. It gets substituted as its non-pre-expanded tokens. |
| 512 |
const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); |
| 513 |
unsigned NumToks = MacroArgs::getArgLength(ArgToks); |
| 514 |
if (NumToks) { // Not an empty argument? |
| 515 |
bool VaArgsPseudoPaste = false; |
| 516 |
// If this is the GNU ", ## __VA_ARGS__" extension, and we just learned |
| 517 |
// that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when |
| 518 |
// the expander tries to paste ',' with the first token of the __VA_ARGS__ |
| 519 |
// expansion. |
| 520 |
if (NonEmptyPasteBefore && ResultToks.size() >= 2 && |
| 521 |
ResultToks[ResultToks.size()-2].is(tok::comma) && |
| 522 |
(unsigned)ArgNo == Macro->getNumParams()-1 && |
| 523 |
Macro->isVariadic()) { |
| 524 |
VaArgsPseudoPaste = true; |
| 525 |
// Remove the paste operator, report use of the extension. |
| 526 |
PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma); |
| 527 |
} |
| 528 |
|
| 529 |
ResultToks.append(ArgToks, ArgToks+NumToks); |
| 530 |
|
| 531 |
// If the '##' came from expanding an argument, turn it into 'unknown' |
| 532 |
// to avoid pasting. |
| 533 |
for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks, |
| 534 |
ResultToks.end())) { |
| 535 |
if (Tok.is(tok::hashhash)) |
| 536 |
Tok.setKind(tok::unknown); |
| 537 |
} |
| 538 |
|
| 539 |
if (ExpandLocStart.isValid()) { |
| 540 |
updateLocForMacroArgTokens(CurTok.getLocation(), |
| 541 |
ResultToks.end()-NumToks, ResultToks.end()); |
| 542 |
} |
| 543 |
|
| 544 |
// Transfer the leading whitespace information from the token |
| 545 |
// (the macro argument) onto the first token of the |
| 546 |
// expansion. Note that we don't do this for the GNU |
| 547 |
// pseudo-paste extension ", ## __VA_ARGS__". |
| 548 |
if (!VaArgsPseudoPaste) { |
| 549 |
ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine, |
| 550 |
false); |
| 551 |
ResultToks[ResultToks.size() - NumToks].setFlagValue( |
| 552 |
Token::LeadingSpace, NextTokGetsSpace); |
| 553 |
} |
| 554 |
|
| 555 |
NextTokGetsSpace = false; |
| 556 |
continue; |
| 557 |
} |
| 558 |
|
| 559 |
// If an empty argument is on the LHS or RHS of a paste, the standard (C99 |
| 560 |
// 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We |
| 561 |
// implement this by eating ## operators when a LHS or RHS expands to |
| 562 |
// empty. |
| 563 |
if (PasteAfter) { |
| 564 |
// Discard the argument token and skip (don't copy to the expansion |
| 565 |
// buffer) the paste operator after it. |
| 566 |
++I; |
| 567 |
continue; |
| 568 |
} |
| 569 |
|
| 570 |
if (RParenAfter) |
| 571 |
VCtx.hasPlaceholderBeforeRParen(); |
| 572 |
|
| 573 |
// If this is on the RHS of a paste operator, we've already copied the |
| 574 |
// paste operator to the ResultToks list, unless the LHS was empty too. |
| 575 |
// Remove it. |
| 576 |
assert(PasteBefore); |
| 577 |
if (NonEmptyPasteBefore) { |
| 578 |
assert(ResultToks.back().is(tok::hashhash)); |
| 579 |
// Do not remove the paste operator if it is the one before __VA_OPT__ |
| 580 |
// (and we are still processing tokens within VA_OPT). We handle the case |
| 581 |
// of removing the paste operator if __VA_OPT__ reduces to the notional |
| 582 |
// placemarker above when we encounter the closing paren of VA_OPT. |
| 583 |
if (!VCtx.isInVAOpt() || |
| 584 |
ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt()) |
| 585 |
ResultToks.pop_back(); |
| 586 |
else |
| 587 |
VCtx.hasPlaceholderAfterHashhashAtStart(); |
| 588 |
} |
| 589 |
|
| 590 |
// If this is the __VA_ARGS__ token, and if the argument wasn't provided, |
| 591 |
// and if the macro had at least one real argument, and if the token before |
| 592 |
// the ## was a comma, remove the comma. This is a GCC extension which is |
| 593 |
// disabled when using -std=c99. |
| 594 |
if (ActualArgs->isVarargsElidedUse()) |
| 595 |
MaybeRemoveCommaBeforeVaArgs(ResultToks, |
| 596 |
/*HasPasteOperator=*/true, |
| 597 |
Macro, ArgNo, PP); |
| 598 |
} |
| 599 |
|
| 600 |
// If anything changed, install this as the new Tokens list. |
| 601 |
if (MadeChange) { |
| 602 |
assert(!OwnsTokens && "This would leak if we already own the token list"); |
| 603 |
// This is deleted in the dtor. |
| 604 |
NumTokens = ResultToks.size(); |
| 605 |
// The tokens will be added to Preprocessor's cache and will be removed |
| 606 |
// when this TokenLexer finishes lexing them. |
| 607 |
Tokens = PP.cacheMacroExpandedTokens(this, ResultToks); |
| 608 |
|
| 609 |
// The preprocessor cache of macro expanded tokens owns these tokens,not us. |
| 610 |
OwnsTokens = false; |
| 611 |
} |
| 612 |
} |
| 613 |
|
| 614 |
/// Checks if two tokens form wide string literal. |
| 615 |
static bool isWideStringLiteralFromMacro(const Token &FirstTok, |
| 616 |
const Token &SecondTok) { |
| 617 |
return FirstTok.is(tok::identifier) && |
| 618 |
FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() && |
| 619 |
SecondTok.stringifiedInMacro(); |
| 620 |
} |
| 621 |
|
| 622 |
/// Lex - Lex and return a token from this macro stream. |
| 623 |
bool TokenLexer::Lex(Token &Tok) { |
| 624 |
// Lexing off the end of the macro, pop this macro off the expansion stack. |
| 625 |
if (isAtEnd()) { |
| 626 |
// If this is a macro (not a token stream), mark the macro enabled now |
| 627 |
// that it is no longer being expanded. |
| 628 |
if (Macro) Macro->EnableMacro(); |
| 629 |
|
| 630 |
Tok.startToken(); |
| 631 |
Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); |
| 632 |
Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace); |
| 633 |
if (CurTokenIdx == 0) |
| 634 |
Tok.setFlag(Token::LeadingEmptyMacro); |
| 635 |
return PP.HandleEndOfTokenLexer(Tok); |
| 636 |
} |
| 637 |
|
| 638 |
SourceManager &SM = PP.getSourceManager(); |
| 639 |
|
| 640 |
// If this is the first token of the expanded result, we inherit spacing |
| 641 |
// properties later. |
| 642 |
bool isFirstToken = CurTokenIdx == 0; |
| 643 |
|
| 644 |
// Get the next token to return. |
| 645 |
Tok = Tokens[CurTokenIdx++]; |
| 646 |
if (IsReinject) |
| 647 |
Tok.setFlag(Token::IsReinjected); |
| 648 |
|
| 649 |
bool TokenIsFromPaste = false; |
| 650 |
|
| 651 |
// If this token is followed by a token paste (##) operator, paste the tokens! |
| 652 |
// Note that ## is a normal token when not expanding a macro. |
| 653 |
if (!isAtEnd() && Macro && |
| 654 |
(Tokens[CurTokenIdx].is(tok::hashhash) || |
| 655 |
// Special processing of L#x macros in -fms-compatibility mode. |
| 656 |
// Microsoft compiler is able to form a wide string literal from |
| 657 |
// 'L#macro_arg' construct in a function-like macro. |
| 658 |
(PP.getLangOpts().MSVCCompat && |
| 659 |
isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) { |
| 660 |
// When handling the microsoft /##/ extension, the final token is |
| 661 |
// returned by pasteTokens, not the pasted token. |
| 662 |
if (pasteTokens(Tok)) |
| 663 |
return true; |
| 664 |
|
| 665 |
TokenIsFromPaste = true; |
| 666 |
} |
| 667 |
|
| 668 |
// The token's current location indicate where the token was lexed from. We |
| 669 |
// need this information to compute the spelling of the token, but any |
| 670 |
// diagnostics for the expanded token should appear as if they came from |
| 671 |
// ExpansionLoc. Pull this information together into a new SourceLocation |
| 672 |
// that captures all of this. |
| 673 |
if (ExpandLocStart.isValid() && // Don't do this for token streams. |
| 674 |
// Check that the token's location was not already set properly. |
| 675 |
SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) { |
| 676 |
SourceLocation instLoc; |
| 677 |
if (Tok.is(tok::comment)) { |
| 678 |
instLoc = SM.createExpansionLoc(Tok.getLocation(), |
| 679 |
ExpandLocStart, |
| 680 |
ExpandLocEnd, |
| 681 |
Tok.getLength()); |
| 682 |
} else { |
| 683 |
instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation()); |
| 684 |
} |
| 685 |
|
| 686 |
Tok.setLocation(instLoc); |
| 687 |
} |
| 688 |
|
| 689 |
// If this is the first token, set the lexical properties of the token to |
| 690 |
// match the lexical properties of the macro identifier. |
| 691 |
if (isFirstToken) { |
| 692 |
Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); |
| 693 |
Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace); |
| 694 |
} else { |
| 695 |
// If this is not the first token, we may still need to pass through |
| 696 |
// leading whitespace if we've expanded a macro. |
| 697 |
if (AtStartOfLine) Tok.setFlag(Token::StartOfLine); |
| 698 |
if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace); |
| 699 |
} |
| 700 |
AtStartOfLine = false; |
| 701 |
HasLeadingSpace = false; |
| 702 |
|
| 703 |
// Handle recursive expansion! |
| 704 |
if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) { |
| 705 |
// Change the kind of this identifier to the appropriate token kind, e.g. |
| 706 |
// turning "for" into a keyword. |
| 707 |
IdentifierInfo *II = Tok.getIdentifierInfo(); |
| 708 |
Tok.setKind(II->getTokenID()); |
| 709 |
|
| 710 |
// If this identifier was poisoned and from a paste, emit an error. This |
| 711 |
// won't be handled by Preprocessor::HandleIdentifier because this is coming |
| 712 |
// from a macro expansion. |
| 713 |
if (II->isPoisoned() && TokenIsFromPaste) { |
| 714 |
PP.HandlePoisonedIdentifier(Tok); |
| 715 |
} |
| 716 |
|
| 717 |
if (!DisableMacroExpansion && II->isHandleIdentifierCase()) |
| 718 |
return PP.HandleIdentifier(Tok); |
| 719 |
} |
| 720 |
|
| 721 |
// Otherwise, return a normal token. |
| 722 |
return true; |
| 723 |
} |
| 724 |
|
| 725 |
bool TokenLexer::pasteTokens(Token &Tok) { |
| 726 |
return pasteTokens(Tok, llvm::ArrayRef(Tokens, NumTokens), CurTokenIdx); |
| 727 |
} |
| 728 |
|
| 729 |
/// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ## |
| 730 |
/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there |
| 731 |
/// are more ## after it, chomp them iteratively. Return the result as LHSTok. |
| 732 |
/// If this returns true, the caller should immediately return the token. |
| 733 |
bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream, |
| 734 |
unsigned int &CurIdx) { |
| 735 |
assert(CurIdx > 0 && "## can not be the first token within tokens"); |
| 736 |
assert((TokenStream[CurIdx].is(tok::hashhash) || |
| 737 |
(PP.getLangOpts().MSVCCompat && |
| 738 |
isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) && |
| 739 |
"Token at this Index must be ## or part of the MSVC 'L " |
| 740 |
"#macro-arg' pasting pair"); |
| 741 |
|
| 742 |
// MSVC: If previous token was pasted, this must be a recovery from an invalid |
| 743 |
// paste operation. Ignore spaces before this token to mimic MSVC output. |
| 744 |
// Required for generating valid UUID strings in some MS headers. |
| 745 |
if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) && |
| 746 |
TokenStream[CurIdx - 2].is(tok::hashhash)) |
| 747 |
LHSTok.clearFlag(Token::LeadingSpace); |
| 748 |
|
| 749 |
SmallString<128> Buffer; |
| 750 |
const char *ResultTokStrPtr = nullptr; |
| 751 |
SourceLocation StartLoc = LHSTok.getLocation(); |
| 752 |
SourceLocation PasteOpLoc; |
| 753 |
|
| 754 |
auto IsAtEnd = [&TokenStream, &CurIdx] { |
| 755 |
return TokenStream.size() == CurIdx; |
| 756 |
}; |
| 757 |
|
| 758 |
do { |
| 759 |
// Consume the ## operator if any. |
| 760 |
PasteOpLoc = TokenStream[CurIdx].getLocation(); |
| 761 |
if (TokenStream[CurIdx].is(tok::hashhash)) |
| 762 |
++CurIdx; |
| 763 |
assert(!IsAtEnd() && "No token on the RHS of a paste operator!"); |
| 764 |
|
| 765 |
// Get the RHS token. |
| 766 |
const Token &RHS = TokenStream[CurIdx]; |
| 767 |
|
| 768 |
// Allocate space for the result token. This is guaranteed to be enough for |
| 769 |
// the two tokens. |
| 770 |
Buffer.resize(LHSTok.getLength() + RHS.getLength()); |
| 771 |
|
| 772 |
// Get the spelling of the LHS token in Buffer. |
| 773 |
const char *BufPtr = &Buffer[0]; |
| 774 |
bool Invalid = false; |
| 775 |
unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid); |
| 776 |
if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! |
| 777 |
memcpy(&Buffer[0], BufPtr, LHSLen); |
| 778 |
if (Invalid) |
| 779 |
return true; |
| 780 |
|
| 781 |
BufPtr = Buffer.data() + LHSLen; |
| 782 |
unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid); |
| 783 |
if (Invalid) |
| 784 |
return true; |
| 785 |
if (RHSLen && BufPtr != &Buffer[LHSLen]) |
| 786 |
// Really, we want the chars in Buffer! |
| 787 |
memcpy(&Buffer[LHSLen], BufPtr, RHSLen); |
| 788 |
|
| 789 |
// Trim excess space. |
| 790 |
Buffer.resize(LHSLen+RHSLen); |
| 791 |
|
| 792 |
// Plop the pasted result (including the trailing newline and null) into a |
| 793 |
// scratch buffer where we can lex it. |
| 794 |
Token ResultTokTmp; |
| 795 |
ResultTokTmp.startToken(); |
| 796 |
|
| 797 |
// Claim that the tmp token is a string_literal so that we can get the |
| 798 |
// character pointer back from CreateString in getLiteralData(). |
| 799 |
ResultTokTmp.setKind(tok::string_literal); |
| 800 |
PP.CreateString(Buffer, ResultTokTmp); |
| 801 |
SourceLocation ResultTokLoc = ResultTokTmp.getLocation(); |
| 802 |
ResultTokStrPtr = ResultTokTmp.getLiteralData(); |
| 803 |
|
| 804 |
// Lex the resultant pasted token into Result. |
| 805 |
Token Result; |
| 806 |
|
| 807 |
if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) { |
| 808 |
// Common paste case: identifier+identifier = identifier. Avoid creating |
| 809 |
// a lexer and other overhead. |
| 810 |
PP.IncrementPasteCounter(true); |
| 811 |
Result.startToken(); |
| 812 |
Result.setKind(tok::raw_identifier); |
| 813 |
Result.setRawIdentifierData(ResultTokStrPtr); |
| 814 |
Result.setLocation(ResultTokLoc); |
| 815 |
Result.setLength(LHSLen+RHSLen); |
| 816 |
} else { |
| 817 |
PP.IncrementPasteCounter(false); |
| 818 |
|
| 819 |
assert(ResultTokLoc.isFileID() && |
| 820 |
"Should be a raw location into scratch buffer"); |
| 821 |
SourceManager &SourceMgr = PP.getSourceManager(); |
| 822 |
FileID LocFileID = SourceMgr.getFileID(ResultTokLoc); |
| 823 |
|
| 824 |
bool Invalid = false; |
| 825 |
const char *ScratchBufStart |
| 826 |
= SourceMgr.getBufferData(LocFileID, &Invalid).data(); |
| 827 |
if (Invalid) |
| 828 |
return false; |
| 829 |
|
| 830 |
// Make a lexer to lex this string from. Lex just this one token. |
| 831 |
// Make a lexer object so that we lex and expand the paste result. |
| 832 |
Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID), |
| 833 |
PP.getLangOpts(), ScratchBufStart, |
| 834 |
ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen); |
| 835 |
|
| 836 |
// Lex a token in raw mode. This way it won't look up identifiers |
| 837 |
// automatically, lexing off the end will return an eof token, and |
| 838 |
// warnings are disabled. This returns true if the result token is the |
| 839 |
// entire buffer. |
| 840 |
bool isInvalid = !TL.LexFromRawLexer(Result); |
| 841 |
|
| 842 |
// If we got an EOF token, we didn't form even ONE token. For example, we |
| 843 |
// did "/ ## /" to get "//". |
| 844 |
isInvalid |= Result.is(tok::eof); |
| 845 |
|
| 846 |
// If pasting the two tokens didn't form a full new token, this is an |
| 847 |
// error. This occurs with "x ## +" and other stuff. Return with LHSTok |
| 848 |
// unmodified and with RHS as the next token to lex. |
| 849 |
if (isInvalid) { |
| 850 |
// Explicitly convert the token location to have proper expansion |
| 851 |
// information so that the user knows where it came from. |
| 852 |
SourceManager &SM = PP.getSourceManager(); |
| 853 |
SourceLocation Loc = |
| 854 |
SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2); |
| 855 |
|
| 856 |
// Test for the Microsoft extension of /##/ turning into // here on the |
| 857 |
// error path. |
| 858 |
if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) && |
| 859 |
RHS.is(tok::slash)) { |
| 860 |
HandleMicrosoftCommentPaste(LHSTok, Loc); |
| 861 |
return true; |
| 862 |
} |
| 863 |
|
| 864 |
// Do not emit the error when preprocessing assembler code. |
| 865 |
if (!PP.getLangOpts().AsmPreprocessor) { |
| 866 |
// If we're in microsoft extensions mode, downgrade this from a hard |
| 867 |
// error to an extension that defaults to an error. This allows |
| 868 |
// disabling it. |
| 869 |
PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms |
| 870 |
: diag::err_pp_bad_paste) |
| 871 |
<< Buffer; |
| 872 |
} |
| 873 |
|
| 874 |
// An error has occurred so exit loop. |
| 875 |
break; |
| 876 |
} |
| 877 |
|
| 878 |
// Turn ## into 'unknown' to avoid # ## # from looking like a paste |
| 879 |
// operator. |
| 880 |
if (Result.is(tok::hashhash)) |
| 881 |
Result.setKind(tok::unknown); |
| 882 |
} |
| 883 |
|
| 884 |
// Transfer properties of the LHS over the Result. |
| 885 |
Result.setFlagValue(Token::StartOfLine , LHSTok.isAtStartOfLine()); |
| 886 |
Result.setFlagValue(Token::LeadingSpace, LHSTok.hasLeadingSpace()); |
| 887 |
|
| 888 |
// Finally, replace LHS with the result, consume the RHS, and iterate. |
| 889 |
++CurIdx; |
| 890 |
LHSTok = Result; |
| 891 |
} while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash)); |
| 892 |
|
| 893 |
SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation(); |
| 894 |
|
| 895 |
// The token's current location indicate where the token was lexed from. We |
| 896 |
// need this information to compute the spelling of the token, but any |
| 897 |
// diagnostics for the expanded token should appear as if the token was |
| 898 |
// expanded from the full ## expression. Pull this information together into |
| 899 |
// a new SourceLocation that captures all of this. |
| 900 |
SourceManager &SM = PP.getSourceManager(); |
| 901 |
if (StartLoc.isFileID()) |
| 902 |
StartLoc = getExpansionLocForMacroDefLoc(StartLoc); |
| 903 |
if (EndLoc.isFileID()) |
| 904 |
EndLoc = getExpansionLocForMacroDefLoc(EndLoc); |
| 905 |
FileID MacroFID = SM.getFileID(MacroExpansionStart); |
| 906 |
while (SM.getFileID(StartLoc) != MacroFID) |
| 907 |
StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin(); |
| 908 |
while (SM.getFileID(EndLoc) != MacroFID) |
| 909 |
EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd(); |
| 910 |
|
| 911 |
LHSTok.setLocation(SM.createExpansionLoc(LHSTok.getLocation(), StartLoc, EndLoc, |
| 912 |
LHSTok.getLength())); |
| 913 |
|
| 914 |
// Now that we got the result token, it will be subject to expansion. Since |
| 915 |
// token pasting re-lexes the result token in raw mode, identifier information |
| 916 |
// isn't looked up. As such, if the result is an identifier, look up id info. |
| 917 |
if (LHSTok.is(tok::raw_identifier)) { |
| 918 |
// Look up the identifier info for the token. We disabled identifier lookup |
| 919 |
// by saying we're skipping contents, so we need to do this manually. |
| 920 |
PP.LookUpIdentifierInfo(LHSTok); |
| 921 |
} |
| 922 |
return false; |
| 923 |
} |
| 924 |
|
| 925 |
/// isNextTokenLParen - If the next token lexed will pop this macro off the |
| 926 |
/// expansion stack, return 2. If the next unexpanded token is a '(', return |
| 927 |
/// 1, otherwise return 0. |
| 928 |
unsigned TokenLexer::isNextTokenLParen() const { |
| 929 |
// Out of tokens? |
| 930 |
if (isAtEnd()) |
| 931 |
return 2; |
| 932 |
return Tokens[CurTokenIdx].is(tok::l_paren); |
| 933 |
} |
| 934 |
|
| 935 |
/// isParsingPreprocessorDirective - Return true if we are in the middle of a |
| 936 |
/// preprocessor directive. |
| 937 |
bool TokenLexer::isParsingPreprocessorDirective() const { |
| 938 |
return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd(); |
| 939 |
} |
| 940 |
|
| 941 |
/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes |
| 942 |
/// together to form a comment that comments out everything in the current |
| 943 |
/// macro, other active macros, and anything left on the current physical |
| 944 |
/// source line of the expanded buffer. Handle this by returning the |
| 945 |
/// first token on the next line. |
| 946 |
void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) { |
| 947 |
PP.Diag(OpLoc, diag::ext_comment_paste_microsoft); |
| 948 |
|
| 949 |
// We 'comment out' the rest of this macro by just ignoring the rest of the |
| 950 |
// tokens that have not been lexed yet, if any. |
| 951 |
|
| 952 |
// Since this must be a macro, mark the macro enabled now that it is no longer |
| 953 |
// being expanded. |
| 954 |
assert(Macro && "Token streams can't paste comments"); |
| 955 |
Macro->EnableMacro(); |
| 956 |
|
| 957 |
PP.HandleMicrosoftCommentPaste(Tok); |
| 958 |
} |
| 959 |
|
| 960 |
/// If \arg loc is a file ID and points inside the current macro |
| 961 |
/// definition, returns the appropriate source location pointing at the |
| 962 |
/// macro expansion source location entry, otherwise it returns an invalid |
| 963 |
/// SourceLocation. |
| 964 |
SourceLocation |
| 965 |
TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const { |
| 966 |
assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() && |
| 967 |
"Not appropriate for token streams"); |
| 968 |
assert(loc.isValid() && loc.isFileID()); |
| 969 |
|
| 970 |
SourceManager &SM = PP.getSourceManager(); |
| 971 |
assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) && |
| 972 |
"Expected loc to come from the macro definition"); |
| 973 |
|
| 974 |
SourceLocation::UIntTy relativeOffset = 0; |
| 975 |
SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset); |
| 976 |
return MacroExpansionStart.getLocWithOffset(relativeOffset); |
| 977 |
} |
| 978 |
|
| 979 |
/// Finds the tokens that are consecutive (from the same FileID) |
| 980 |
/// creates a single SLocEntry, and assigns SourceLocations to each token that |
| 981 |
/// point to that SLocEntry. e.g for |
| 982 |
/// assert(foo == bar); |
| 983 |
/// There will be a single SLocEntry for the "foo == bar" chunk and locations |
| 984 |
/// for the 'foo', '==', 'bar' tokens will point inside that chunk. |
| 985 |
/// |
| 986 |
/// \arg begin_tokens will be updated to a position past all the found |
| 987 |
/// consecutive tokens. |
| 988 |
static void updateConsecutiveMacroArgTokens(SourceManager &SM, |
| 989 |
SourceLocation ExpandLoc, |
| 990 |
Token *&begin_tokens, |
| 991 |
Token * end_tokens) { |
| 992 |
assert(begin_tokens + 1 < end_tokens); |
| 993 |
SourceLocation BeginLoc = begin_tokens->getLocation(); |
| 994 |
llvm::MutableArrayRef<Token> All(begin_tokens, end_tokens); |
| 995 |
llvm::MutableArrayRef<Token> Partition; |
| 996 |
|
| 997 |
auto NearLast = [&, Last = BeginLoc](SourceLocation Loc) mutable { |
| 998 |
// The maximum distance between two consecutive tokens in a partition. |
| 999 |
// This is an important trick to avoid using too much SourceLocation address |
| 1000 |
// space! |
| 1001 |
static constexpr SourceLocation::IntTy MaxDistance = 50; |
| 1002 |
auto Distance = Loc.getRawEncoding() - Last.getRawEncoding(); |
| 1003 |
Last = Loc; |
| 1004 |
return Distance <= MaxDistance; |
| 1005 |
}; |
| 1006 |
|
| 1007 |
// Partition the tokens by their FileID. |
| 1008 |
// This is a hot function, and calling getFileID can be expensive, the |
| 1009 |
// implementation is optimized by reducing the number of getFileID. |
| 1010 |
if (BeginLoc.isFileID()) { |
| 1011 |
// Consecutive tokens not written in macros must be from the same file. |
| 1012 |
// (Neither #include nor eof can occur inside a macro argument.) |
| 1013 |
Partition = All.take_while([&](const Token &T) { |
| 1014 |
return T.getLocation().isFileID() && NearLast(T.getLocation()); |
| 1015 |
}); |
| 1016 |
} else { |
| 1017 |
// Call getFileID once to calculate the bounds, and use the cheaper |
| 1018 |
// sourcelocation-against-bounds comparison. |
| 1019 |
FileID BeginFID = SM.getFileID(BeginLoc); |
| 1020 |
SourceLocation Limit = |
| 1021 |
SM.getComposedLoc(BeginFID, SM.getFileIDSize(BeginFID)); |
| 1022 |
Partition = All.take_while([&](const Token &T) { |
| 1023 |
// NOTE: the Limit is included! The lexer recovery only ever inserts a |
| 1024 |
// single token past the end of the FileID, specifically the ) when a |
| 1025 |
// macro-arg containing a comma should be guarded by parentheses. |
| 1026 |
// |
| 1027 |
// It is safe to include the Limit here because SourceManager allocates |
| 1028 |
// FileSize + 1 for each SLocEntry. |
| 1029 |
// |
| 1030 |
// See https://github.com/llvm/llvm-project/issues/60722. |
| 1031 |
return T.getLocation() >= BeginLoc && T.getLocation() <= Limit |
| 1032 |
&& NearLast(T.getLocation()); |
| 1033 |
}); |
| 1034 |
} |
| 1035 |
assert(!Partition.empty()); |
| 1036 |
|
| 1037 |
// For the consecutive tokens, find the length of the SLocEntry to contain |
| 1038 |
// all of them. |
| 1039 |
SourceLocation::UIntTy FullLength = |
| 1040 |
Partition.back().getEndLoc().getRawEncoding() - |
| 1041 |
Partition.front().getLocation().getRawEncoding(); |
| 1042 |
// Create a macro expansion SLocEntry that will "contain" all of the tokens. |
| 1043 |
SourceLocation Expansion = |
| 1044 |
SM.createMacroArgExpansionLoc(BeginLoc, ExpandLoc, FullLength); |
| 1045 |
|
| 1046 |
#ifdef EXPENSIVE_CHECKS |
| 1047 |
assert(llvm::all_of(Partition.drop_front(), |
| 1048 |
[&SM, ID = SM.getFileID(Partition.front().getLocation())]( |
| 1049 |
const Token &T) { |
| 1050 |
return ID == SM.getFileID(T.getLocation()); |
| 1051 |
}) && |
| 1052 |
"Must have the same FIleID!"); |
| 1053 |
#endif |
| 1054 |
// Change the location of the tokens from the spelling location to the new |
| 1055 |
// expanded location. |
| 1056 |
for (Token& T : Partition) { |
| 1057 |
SourceLocation::IntTy RelativeOffset = |
| 1058 |
T.getLocation().getRawEncoding() - BeginLoc.getRawEncoding(); |
| 1059 |
T.setLocation(Expansion.getLocWithOffset(RelativeOffset)); |
| 1060 |
} |
| 1061 |
begin_tokens = &Partition.back() + 1; |
| 1062 |
} |
| 1063 |
|
| 1064 |
/// Creates SLocEntries and updates the locations of macro argument |
| 1065 |
/// tokens to their new expanded locations. |
| 1066 |
/// |
| 1067 |
/// \param ArgIdSpellLoc the location of the macro argument id inside the macro |
| 1068 |
/// definition. |
| 1069 |
void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, |
| 1070 |
Token *begin_tokens, |
| 1071 |
Token *end_tokens) { |
| 1072 |
SourceManager &SM = PP.getSourceManager(); |
| 1073 |
|
| 1074 |
SourceLocation ExpandLoc = |
| 1075 |
getExpansionLocForMacroDefLoc(ArgIdSpellLoc); |
| 1076 |
|
| 1077 |
while (begin_tokens < end_tokens) { |
| 1078 |
// If there's only one token just create a SLocEntry for it. |
| 1079 |
if (end_tokens - begin_tokens == 1) { |
| 1080 |
Token &Tok = *begin_tokens; |
| 1081 |
Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(), |
| 1082 |
ExpandLoc, |
| 1083 |
Tok.getLength())); |
| 1084 |
return; |
| 1085 |
} |
| 1086 |
|
| 1087 |
updateConsecutiveMacroArgTokens(SM, ExpandLoc, begin_tokens, end_tokens); |
| 1088 |
} |
| 1089 |
} |
| 1090 |
|
| 1091 |
void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { |
| 1092 |
AtStartOfLine = Result.isAtStartOfLine(); |
| 1093 |
HasLeadingSpace = Result.hasLeadingSpace(); |
| 1094 |
} |