| 1 |
//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// This contains code to emit Expr nodes as LLVM code. |
| 10 |
// |
| 11 |
//===----------------------------------------------------------------------===// |
| 12 |
|
| 13 |
#include "CGCUDARuntime.h" |
| 14 |
#include "CGCXXABI.h" |
| 15 |
#include "CGCall.h" |
| 16 |
#include "CGCleanup.h" |
| 17 |
#include "CGDebugInfo.h" |
| 18 |
#include "CGObjCRuntime.h" |
| 19 |
#include "CGOpenMPRuntime.h" |
| 20 |
#include "CGRecordLayout.h" |
| 21 |
#include "CodeGenFunction.h" |
| 22 |
#include "CodeGenModule.h" |
| 23 |
#include "ConstantEmitter.h" |
| 24 |
#include "TargetInfo.h" |
| 25 |
#include "clang/AST/ASTContext.h" |
| 26 |
#include "clang/AST/Attr.h" |
| 27 |
#include "clang/AST/DeclObjC.h" |
| 28 |
#include "clang/AST/NSAPI.h" |
| 29 |
#include "clang/Basic/Builtins.h" |
| 30 |
#include "clang/Basic/CodeGenOptions.h" |
| 31 |
#include "clang/Basic/SourceManager.h" |
| 32 |
#include "llvm/ADT/Hashing.h" |
| 33 |
#include "llvm/ADT/StringExtras.h" |
| 34 |
#include "llvm/IR/DataLayout.h" |
| 35 |
#include "llvm/IR/Intrinsics.h" |
| 36 |
#include "llvm/IR/LLVMContext.h" |
| 37 |
#include "llvm/IR/MDBuilder.h" |
| 38 |
#include "llvm/IR/MatrixBuilder.h" |
| 39 |
#include "llvm/Support/ConvertUTF.h" |
| 40 |
#include "llvm/Support/MathExtras.h" |
| 41 |
#include "llvm/Support/Path.h" |
| 42 |
#include "llvm/Support/SaveAndRestore.h" |
| 43 |
#include "llvm/Transforms/Utils/SanitizerStats.h" |
| 44 |
|
| 45 |
#include <optional> |
| 46 |
#include <string> |
| 47 |
|
| 48 |
using namespace clang; |
| 49 |
using namespace CodeGen; |
| 50 |
|
| 51 |
//===--------------------------------------------------------------------===// |
| 52 |
// Miscellaneous Helper Methods |
| 53 |
//===--------------------------------------------------------------------===// |
| 54 |
|
| 55 |
llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) { |
| 56 |
unsigned addressSpace = |
| 57 |
cast<llvm::PointerType>(value->getType())->getAddressSpace(); |
| 58 |
|
| 59 |
llvm::PointerType *destType = Int8PtrTy; |
| 60 |
if (addressSpace) |
| 61 |
destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace); |
| 62 |
|
| 63 |
if (value->getType() == destType) return value; |
| 64 |
return Builder.CreateBitCast(value, destType); |
| 65 |
} |
| 66 |
|
| 67 |
/// CreateTempAlloca - This creates a alloca and inserts it into the entry |
| 68 |
/// block. |
| 69 |
Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, |
| 70 |
CharUnits Align, |
| 71 |
const Twine &Name, |
| 72 |
llvm::Value *ArraySize) { |
| 73 |
auto Alloca = CreateTempAlloca(Ty, Name, ArraySize); |
| 74 |
Alloca->setAlignment(Align.getAsAlign()); |
| 75 |
return Address(Alloca, Ty, Align); |
| 76 |
} |
| 77 |
|
| 78 |
/// CreateTempAlloca - This creates a alloca and inserts it into the entry |
| 79 |
/// block. The alloca is casted to default address space if necessary. |
| 80 |
Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align, |
| 81 |
const Twine &Name, |
| 82 |
llvm::Value *ArraySize, |
| 83 |
Address *AllocaAddr) { |
| 84 |
auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize); |
| 85 |
if (AllocaAddr) |
| 86 |
*AllocaAddr = Alloca; |
| 87 |
llvm::Value *V = Alloca.getPointer(); |
| 88 |
// Alloca always returns a pointer in alloca address space, which may |
| 89 |
// be different from the type defined by the language. For example, |
| 90 |
// in C++ the auto variables are in the default address space. Therefore |
| 91 |
// cast alloca to the default address space when necessary. |
| 92 |
if (getASTAllocaAddressSpace() != LangAS::Default) { |
| 93 |
auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default); |
| 94 |
llvm::IRBuilderBase::InsertPointGuard IPG(Builder); |
| 95 |
// When ArraySize is nullptr, alloca is inserted at AllocaInsertPt, |
| 96 |
// otherwise alloca is inserted at the current insertion point of the |
| 97 |
// builder. |
| 98 |
if (!ArraySize) |
| 99 |
Builder.SetInsertPoint(getPostAllocaInsertPoint()); |
| 100 |
V = getTargetHooks().performAddrSpaceCast( |
| 101 |
*this, V, getASTAllocaAddressSpace(), LangAS::Default, |
| 102 |
Ty->getPointerTo(DestAddrSpace), /*non-null*/ true); |
| 103 |
} |
| 104 |
|
| 105 |
return Address(V, Ty, Align); |
| 106 |
} |
| 107 |
|
| 108 |
/// CreateTempAlloca - This creates an alloca and inserts it into the entry |
| 109 |
/// block if \p ArraySize is nullptr, otherwise inserts it at the current |
| 110 |
/// insertion point of the builder. |
| 111 |
llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, |
| 112 |
const Twine &Name, |
| 113 |
llvm::Value *ArraySize) { |
| 114 |
if (ArraySize) |
| 115 |
return Builder.CreateAlloca(Ty, ArraySize, Name); |
| 116 |
return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(), |
| 117 |
ArraySize, Name, AllocaInsertPt); |
| 118 |
} |
| 119 |
|
| 120 |
/// CreateDefaultAlignTempAlloca - This creates an alloca with the |
| 121 |
/// default alignment of the corresponding LLVM type, which is *not* |
| 122 |
/// guaranteed to be related in any way to the expected alignment of |
| 123 |
/// an AST type that might have been lowered to Ty. |
| 124 |
Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty, |
| 125 |
const Twine &Name) { |
| 126 |
CharUnits Align = |
| 127 |
CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty)); |
| 128 |
return CreateTempAlloca(Ty, Align, Name); |
| 129 |
} |
| 130 |
|
| 131 |
Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) { |
| 132 |
CharUnits Align = getContext().getTypeAlignInChars(Ty); |
| 133 |
return CreateTempAlloca(ConvertType(Ty), Align, Name); |
| 134 |
} |
| 135 |
|
| 136 |
Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name, |
| 137 |
Address *Alloca) { |
| 138 |
// FIXME: Should we prefer the preferred type alignment here? |
| 139 |
return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca); |
| 140 |
} |
| 141 |
|
| 142 |
Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align, |
| 143 |
const Twine &Name, Address *Alloca) { |
| 144 |
Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name, |
| 145 |
/*ArraySize=*/nullptr, Alloca); |
| 146 |
|
| 147 |
if (Ty->isConstantMatrixType()) { |
| 148 |
auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType()); |
| 149 |
auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), |
| 150 |
ArrayTy->getNumElements()); |
| 151 |
|
| 152 |
Result = Address( |
| 153 |
Builder.CreateBitCast(Result.getPointer(), VectorTy->getPointerTo()), |
| 154 |
VectorTy, Result.getAlignment()); |
| 155 |
} |
| 156 |
return Result; |
| 157 |
} |
| 158 |
|
| 159 |
Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align, |
| 160 |
const Twine &Name) { |
| 161 |
return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name); |
| 162 |
} |
| 163 |
|
| 164 |
Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, |
| 165 |
const Twine &Name) { |
| 166 |
return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty), |
| 167 |
Name); |
| 168 |
} |
| 169 |
|
| 170 |
/// EvaluateExprAsBool - Perform the usual unary conversions on the specified |
| 171 |
/// expression and compare the result against zero, returning an Int1Ty value. |
| 172 |
llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) { |
| 173 |
PGO.setCurrentStmt(E); |
| 174 |
if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) { |
| 175 |
llvm::Value *MemPtr = EmitScalarExpr(E); |
| 176 |
return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT); |
| 177 |
} |
| 178 |
|
| 179 |
QualType BoolTy = getContext().BoolTy; |
| 180 |
SourceLocation Loc = E->getExprLoc(); |
| 181 |
CGFPOptionsRAII FPOptsRAII(*this, E); |
| 182 |
if (!E->getType()->isAnyComplexType()) |
| 183 |
return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc); |
| 184 |
|
| 185 |
return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy, |
| 186 |
Loc); |
| 187 |
} |
| 188 |
|
| 189 |
/// EmitIgnoredExpr - Emit code to compute the specified expression, |
| 190 |
/// ignoring the result. |
| 191 |
void CodeGenFunction::EmitIgnoredExpr(const Expr *E) { |
| 192 |
if (E->isPRValue()) |
| 193 |
return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true); |
| 194 |
|
| 195 |
// if this is a bitfield-resulting conditional operator, we can special case |
| 196 |
// emit this. The normal 'EmitLValue' version of this is particularly |
| 197 |
// difficult to codegen for, since creating a single "LValue" for two |
| 198 |
// different sized arguments here is not particularly doable. |
| 199 |
if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>( |
| 200 |
E->IgnoreParenNoopCasts(getContext()))) { |
| 201 |
if (CondOp->getObjectKind() == OK_BitField) |
| 202 |
return EmitIgnoredConditionalOperator(CondOp); |
| 203 |
} |
| 204 |
|
| 205 |
// Just emit it as an l-value and drop the result. |
| 206 |
EmitLValue(E); |
| 207 |
} |
| 208 |
|
| 209 |
/// EmitAnyExpr - Emit code to compute the specified expression which |
| 210 |
/// can have any type. The result is returned as an RValue struct. |
| 211 |
/// If this is an aggregate expression, AggSlot indicates where the |
| 212 |
/// result should be returned. |
| 213 |
RValue CodeGenFunction::EmitAnyExpr(const Expr *E, |
| 214 |
AggValueSlot aggSlot, |
| 215 |
bool ignoreResult) { |
| 216 |
switch (getEvaluationKind(E->getType())) { |
| 217 |
case TEK_Scalar: |
| 218 |
return RValue::get(EmitScalarExpr(E, ignoreResult)); |
| 219 |
case TEK_Complex: |
| 220 |
return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult)); |
| 221 |
case TEK_Aggregate: |
| 222 |
if (!ignoreResult && aggSlot.isIgnored()) |
| 223 |
aggSlot = CreateAggTemp(E->getType(), "agg-temp"); |
| 224 |
EmitAggExpr(E, aggSlot); |
| 225 |
return aggSlot.asRValue(); |
| 226 |
} |
| 227 |
llvm_unreachable("bad evaluation kind"); |
| 228 |
} |
| 229 |
|
| 230 |
/// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will |
| 231 |
/// always be accessible even if no aggregate location is provided. |
| 232 |
RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) { |
| 233 |
AggValueSlot AggSlot = AggValueSlot::ignored(); |
| 234 |
|
| 235 |
if (hasAggregateEvaluationKind(E->getType())) |
| 236 |
AggSlot = CreateAggTemp(E->getType(), "agg.tmp"); |
| 237 |
return EmitAnyExpr(E, AggSlot); |
| 238 |
} |
| 239 |
|
| 240 |
/// EmitAnyExprToMem - Evaluate an expression into a given memory |
| 241 |
/// location. |
| 242 |
void CodeGenFunction::EmitAnyExprToMem(const Expr *E, |
| 243 |
Address Location, |
| 244 |
Qualifiers Quals, |
| 245 |
bool IsInit) { |
| 246 |
// FIXME: This function should take an LValue as an argument. |
| 247 |
switch (getEvaluationKind(E->getType())) { |
| 248 |
case TEK_Complex: |
| 249 |
EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()), |
| 250 |
/*isInit*/ false); |
| 251 |
return; |
| 252 |
|
| 253 |
case TEK_Aggregate: { |
| 254 |
EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals, |
| 255 |
AggValueSlot::IsDestructed_t(IsInit), |
| 256 |
AggValueSlot::DoesNotNeedGCBarriers, |
| 257 |
AggValueSlot::IsAliased_t(!IsInit), |
| 258 |
AggValueSlot::MayOverlap)); |
| 259 |
return; |
| 260 |
} |
| 261 |
|
| 262 |
case TEK_Scalar: { |
| 263 |
RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false)); |
| 264 |
LValue LV = MakeAddrLValue(Location, E->getType()); |
| 265 |
EmitStoreThroughLValue(RV, LV); |
| 266 |
return; |
| 267 |
} |
| 268 |
} |
| 269 |
llvm_unreachable("bad evaluation kind"); |
| 270 |
} |
| 271 |
|
| 272 |
static void |
| 273 |
pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M, |
| 274 |
const Expr *E, Address ReferenceTemporary) { |
| 275 |
// Objective-C++ ARC: |
| 276 |
// If we are binding a reference to a temporary that has ownership, we |
| 277 |
// need to perform retain/release operations on the temporary. |
| 278 |
// |
| 279 |
// FIXME: This should be looking at E, not M. |
| 280 |
if (auto Lifetime = M->getType().getObjCLifetime()) { |
| 281 |
switch (Lifetime) { |
| 282 |
case Qualifiers::OCL_None: |
| 283 |
case Qualifiers::OCL_ExplicitNone: |
| 284 |
// Carry on to normal cleanup handling. |
| 285 |
break; |
| 286 |
|
| 287 |
case Qualifiers::OCL_Autoreleasing: |
| 288 |
// Nothing to do; cleaned up by an autorelease pool. |
| 289 |
return; |
| 290 |
|
| 291 |
case Qualifiers::OCL_Strong: |
| 292 |
case Qualifiers::OCL_Weak: |
| 293 |
switch (StorageDuration Duration = M->getStorageDuration()) { |
| 294 |
case SD_Static: |
| 295 |
// Note: we intentionally do not register a cleanup to release |
| 296 |
// the object on program termination. |
| 297 |
return; |
| 298 |
|
| 299 |
case SD_Thread: |
| 300 |
// FIXME: We should probably register a cleanup in this case. |
| 301 |
return; |
| 302 |
|
| 303 |
case SD_Automatic: |
| 304 |
case SD_FullExpression: |
| 305 |
CodeGenFunction::Destroyer *Destroy; |
| 306 |
CleanupKind CleanupKind; |
| 307 |
if (Lifetime == Qualifiers::OCL_Strong) { |
| 308 |
const ValueDecl *VD = M->getExtendingDecl(); |
| 309 |
bool Precise = |
| 310 |
VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>(); |
| 311 |
CleanupKind = CGF.getARCCleanupKind(); |
| 312 |
Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise |
| 313 |
: &CodeGenFunction::destroyARCStrongImprecise; |
| 314 |
} else { |
| 315 |
// __weak objects always get EH cleanups; otherwise, exceptions |
| 316 |
// could cause really nasty crashes instead of mere leaks. |
| 317 |
CleanupKind = NormalAndEHCleanup; |
| 318 |
Destroy = &CodeGenFunction::destroyARCWeak; |
| 319 |
} |
| 320 |
if (Duration == SD_FullExpression) |
| 321 |
CGF.pushDestroy(CleanupKind, ReferenceTemporary, |
| 322 |
M->getType(), *Destroy, |
| 323 |
CleanupKind & EHCleanup); |
| 324 |
else |
| 325 |
CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary, |
| 326 |
M->getType(), |
| 327 |
*Destroy, CleanupKind & EHCleanup); |
| 328 |
return; |
| 329 |
|
| 330 |
case SD_Dynamic: |
| 331 |
llvm_unreachable("temporary cannot have dynamic storage duration"); |
| 332 |
} |
| 333 |
llvm_unreachable("unknown storage duration"); |
| 334 |
} |
| 335 |
} |
| 336 |
|
| 337 |
CXXDestructorDecl *ReferenceTemporaryDtor = nullptr; |
| 338 |
if (const RecordType *RT = |
| 339 |
E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) { |
| 340 |
// Get the destructor for the reference temporary. |
| 341 |
auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl()); |
| 342 |
if (!ClassDecl->hasTrivialDestructor()) |
| 343 |
ReferenceTemporaryDtor = ClassDecl->getDestructor(); |
| 344 |
} |
| 345 |
|
| 346 |
if (!ReferenceTemporaryDtor) |
| 347 |
return; |
| 348 |
|
| 349 |
// Call the destructor for the temporary. |
| 350 |
switch (M->getStorageDuration()) { |
| 351 |
case SD_Static: |
| 352 |
case SD_Thread: { |
| 353 |
llvm::FunctionCallee CleanupFn; |
| 354 |
llvm::Constant *CleanupArg; |
| 355 |
if (E->getType()->isArrayType()) { |
| 356 |
CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper( |
| 357 |
ReferenceTemporary, E->getType(), |
| 358 |
CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions, |
| 359 |
dyn_cast_or_null<VarDecl>(M->getExtendingDecl())); |
| 360 |
CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy); |
| 361 |
} else { |
| 362 |
CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor( |
| 363 |
GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete)); |
| 364 |
CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer()); |
| 365 |
} |
| 366 |
CGF.CGM.getCXXABI().registerGlobalDtor( |
| 367 |
CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg); |
| 368 |
break; |
| 369 |
} |
| 370 |
|
| 371 |
case SD_FullExpression: |
| 372 |
CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(), |
| 373 |
CodeGenFunction::destroyCXXObject, |
| 374 |
CGF.getLangOpts().Exceptions); |
| 375 |
break; |
| 376 |
|
| 377 |
case SD_Automatic: |
| 378 |
CGF.pushLifetimeExtendedDestroy(NormalAndEHCleanup, |
| 379 |
ReferenceTemporary, E->getType(), |
| 380 |
CodeGenFunction::destroyCXXObject, |
| 381 |
CGF.getLangOpts().Exceptions); |
| 382 |
break; |
| 383 |
|
| 384 |
case SD_Dynamic: |
| 385 |
llvm_unreachable("temporary cannot have dynamic storage duration"); |
| 386 |
} |
| 387 |
} |
| 388 |
|
| 389 |
static Address createReferenceTemporary(CodeGenFunction &CGF, |
| 390 |
const MaterializeTemporaryExpr *M, |
| 391 |
const Expr *Inner, |
| 392 |
Address *Alloca = nullptr) { |
| 393 |
auto &TCG = CGF.getTargetHooks(); |
| 394 |
switch (M->getStorageDuration()) { |
| 395 |
case SD_FullExpression: |
| 396 |
case SD_Automatic: { |
| 397 |
// If we have a constant temporary array or record try to promote it into a |
| 398 |
// constant global under the same rules a normal constant would've been |
| 399 |
// promoted. This is easier on the optimizer and generally emits fewer |
| 400 |
// instructions. |
| 401 |
QualType Ty = Inner->getType(); |
| 402 |
if (CGF.CGM.getCodeGenOpts().MergeAllConstants && |
| 403 |
(Ty->isArrayType() || Ty->isRecordType()) && |
| 404 |
CGF.CGM.isTypeConstant(Ty, true)) |
| 405 |
if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) { |
| 406 |
auto AS = CGF.CGM.GetGlobalConstantAddressSpace(); |
| 407 |
auto *GV = new llvm::GlobalVariable( |
| 408 |
CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, |
| 409 |
llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr, |
| 410 |
llvm::GlobalValue::NotThreadLocal, |
| 411 |
CGF.getContext().getTargetAddressSpace(AS)); |
| 412 |
CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty); |
| 413 |
GV->setAlignment(alignment.getAsAlign()); |
| 414 |
llvm::Constant *C = GV; |
| 415 |
if (AS != LangAS::Default) |
| 416 |
C = TCG.performAddrSpaceCast( |
| 417 |
CGF.CGM, GV, AS, LangAS::Default, |
| 418 |
GV->getValueType()->getPointerTo( |
| 419 |
CGF.getContext().getTargetAddressSpace(LangAS::Default))); |
| 420 |
// FIXME: Should we put the new global into a COMDAT? |
| 421 |
return Address(C, GV->getValueType(), alignment); |
| 422 |
} |
| 423 |
return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca); |
| 424 |
} |
| 425 |
case SD_Thread: |
| 426 |
case SD_Static: |
| 427 |
return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner); |
| 428 |
|
| 429 |
case SD_Dynamic: |
| 430 |
llvm_unreachable("temporary can't have dynamic storage duration"); |
| 431 |
} |
| 432 |
llvm_unreachable("unknown storage duration"); |
| 433 |
} |
| 434 |
|
| 435 |
/// Helper method to check if the underlying ABI is AAPCS |
| 436 |
static bool isAAPCS(const TargetInfo &TargetInfo) { |
| 437 |
return TargetInfo.getABI().startswith("aapcs"); |
| 438 |
} |
| 439 |
|
| 440 |
LValue CodeGenFunction:: |
| 441 |
EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) { |
| 442 |
const Expr *E = M->getSubExpr(); |
| 443 |
|
| 444 |
assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) || |
| 445 |
!cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) && |
| 446 |
"Reference should never be pseudo-strong!"); |
| 447 |
|
| 448 |
// FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so |
| 449 |
// as that will cause the lifetime adjustment to be lost for ARC |
| 450 |
auto ownership = M->getType().getObjCLifetime(); |
| 451 |
if (ownership != Qualifiers::OCL_None && |
| 452 |
ownership != Qualifiers::OCL_ExplicitNone) { |
| 453 |
Address Object = createReferenceTemporary(*this, M, E); |
| 454 |
if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) { |
| 455 |
llvm::Type *Ty = ConvertTypeForMem(E->getType()); |
| 456 |
Object = Address(llvm::ConstantExpr::getBitCast( |
| 457 |
Var, Ty->getPointerTo(Object.getAddressSpace())), |
| 458 |
Ty, Object.getAlignment()); |
| 459 |
|
| 460 |
// createReferenceTemporary will promote the temporary to a global with a |
| 461 |
// constant initializer if it can. It can only do this to a value of |
| 462 |
// ARC-manageable type if the value is global and therefore "immune" to |
| 463 |
// ref-counting operations. Therefore we have no need to emit either a |
| 464 |
// dynamic initialization or a cleanup and we can just return the address |
| 465 |
// of the temporary. |
| 466 |
if (Var->hasInitializer()) |
| 467 |
return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl); |
| 468 |
|
| 469 |
Var->setInitializer(CGM.EmitNullConstant(E->getType())); |
| 470 |
} |
| 471 |
LValue RefTempDst = MakeAddrLValue(Object, M->getType(), |
| 472 |
AlignmentSource::Decl); |
| 473 |
|
| 474 |
switch (getEvaluationKind(E->getType())) { |
| 475 |
default: llvm_unreachable("expected scalar or aggregate expression"); |
| 476 |
case TEK_Scalar: |
| 477 |
EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false); |
| 478 |
break; |
| 479 |
case TEK_Aggregate: { |
| 480 |
EmitAggExpr(E, AggValueSlot::forAddr(Object, |
| 481 |
E->getType().getQualifiers(), |
| 482 |
AggValueSlot::IsDestructed, |
| 483 |
AggValueSlot::DoesNotNeedGCBarriers, |
| 484 |
AggValueSlot::IsNotAliased, |
| 485 |
AggValueSlot::DoesNotOverlap)); |
| 486 |
break; |
| 487 |
} |
| 488 |
} |
| 489 |
|
| 490 |
pushTemporaryCleanup(*this, M, E, Object); |
| 491 |
return RefTempDst; |
| 492 |
} |
| 493 |
|
| 494 |
SmallVector<const Expr *, 2> CommaLHSs; |
| 495 |
SmallVector<SubobjectAdjustment, 2> Adjustments; |
| 496 |
E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments); |
| 497 |
|
| 498 |
for (const auto &Ignored : CommaLHSs) |
| 499 |
EmitIgnoredExpr(Ignored); |
| 500 |
|
| 501 |
if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) { |
| 502 |
if (opaque->getType()->isRecordType()) { |
| 503 |
assert(Adjustments.empty()); |
| 504 |
return EmitOpaqueValueLValue(opaque); |
| 505 |
} |
| 506 |
} |
| 507 |
|
| 508 |
// Create and initialize the reference temporary. |
| 509 |
Address Alloca = Address::invalid(); |
| 510 |
Address Object = createReferenceTemporary(*this, M, E, &Alloca); |
| 511 |
if (auto *Var = dyn_cast<llvm::GlobalVariable>( |
| 512 |
Object.getPointer()->stripPointerCasts())) { |
| 513 |
llvm::Type *TemporaryType = ConvertTypeForMem(E->getType()); |
| 514 |
Object = Address(llvm::ConstantExpr::getBitCast( |
| 515 |
cast<llvm::Constant>(Object.getPointer()), |
| 516 |
TemporaryType->getPointerTo()), |
| 517 |
TemporaryType, |
| 518 |
Object.getAlignment()); |
| 519 |
// If the temporary is a global and has a constant initializer or is a |
| 520 |
// constant temporary that we promoted to a global, we may have already |
| 521 |
// initialized it. |
| 522 |
if (!Var->hasInitializer()) { |
| 523 |
Var->setInitializer(CGM.EmitNullConstant(E->getType())); |
| 524 |
EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true); |
| 525 |
} |
| 526 |
} else { |
| 527 |
switch (M->getStorageDuration()) { |
| 528 |
case SD_Automatic: |
| 529 |
if (auto *Size = EmitLifetimeStart( |
| 530 |
CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()), |
| 531 |
Alloca.getPointer())) { |
| 532 |
pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker, |
| 533 |
Alloca, Size); |
| 534 |
} |
| 535 |
break; |
| 536 |
|
| 537 |
case SD_FullExpression: { |
| 538 |
if (!ShouldEmitLifetimeMarkers) |
| 539 |
break; |
| 540 |
|
| 541 |
// Avoid creating a conditional cleanup just to hold an llvm.lifetime.end |
| 542 |
// marker. Instead, start the lifetime of a conditional temporary earlier |
| 543 |
// so that it's unconditional. Don't do this with sanitizers which need |
| 544 |
// more precise lifetime marks. |
| 545 |
ConditionalEvaluation *OldConditional = nullptr; |
| 546 |
CGBuilderTy::InsertPoint OldIP; |
| 547 |
if (isInConditionalBranch() && !E->getType().isDestructedType() && |
| 548 |
!SanOpts.has(SanitizerKind::HWAddress) && |
| 549 |
!SanOpts.has(SanitizerKind::Memory) && |
| 550 |
!CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) { |
| 551 |
OldConditional = OutermostConditional; |
| 552 |
OutermostConditional = nullptr; |
| 553 |
|
| 554 |
OldIP = Builder.saveIP(); |
| 555 |
llvm::BasicBlock *Block = OldConditional->getStartingBlock(); |
| 556 |
Builder.restoreIP(CGBuilderTy::InsertPoint( |
| 557 |
Block, llvm::BasicBlock::iterator(Block->back()))); |
| 558 |
} |
| 559 |
|
| 560 |
if (auto *Size = EmitLifetimeStart( |
| 561 |
CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()), |
| 562 |
Alloca.getPointer())) { |
| 563 |
pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca, |
| 564 |
Size); |
| 565 |
} |
| 566 |
|
| 567 |
if (OldConditional) { |
| 568 |
OutermostConditional = OldConditional; |
| 569 |
Builder.restoreIP(OldIP); |
| 570 |
} |
| 571 |
break; |
| 572 |
} |
| 573 |
|
| 574 |
default: |
| 575 |
break; |
| 576 |
} |
| 577 |
EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true); |
| 578 |
} |
| 579 |
pushTemporaryCleanup(*this, M, E, Object); |
| 580 |
|
| 581 |
// Perform derived-to-base casts and/or field accesses, to get from the |
| 582 |
// temporary object we created (and, potentially, for which we extended |
| 583 |
// the lifetime) to the subobject we're binding the reference to. |
| 584 |
for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) { |
| 585 |
switch (Adjustment.Kind) { |
| 586 |
case SubobjectAdjustment::DerivedToBaseAdjustment: |
| 587 |
Object = |
| 588 |
GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass, |
| 589 |
Adjustment.DerivedToBase.BasePath->path_begin(), |
| 590 |
Adjustment.DerivedToBase.BasePath->path_end(), |
| 591 |
/*NullCheckValue=*/ false, E->getExprLoc()); |
| 592 |
break; |
| 593 |
|
| 594 |
case SubobjectAdjustment::FieldAdjustment: { |
| 595 |
LValue LV = MakeAddrLValue(Object, E->getType(), AlignmentSource::Decl); |
| 596 |
LV = EmitLValueForField(LV, Adjustment.Field); |
| 597 |
assert(LV.isSimple() && |
| 598 |
"materialized temporary field is not a simple lvalue"); |
| 599 |
Object = LV.getAddress(*this); |
| 600 |
break; |
| 601 |
} |
| 602 |
|
| 603 |
case SubobjectAdjustment::MemberPointerAdjustment: { |
| 604 |
llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS); |
| 605 |
Object = EmitCXXMemberDataPointerAddress(E, Object, Ptr, |
| 606 |
Adjustment.Ptr.MPT); |
| 607 |
break; |
| 608 |
} |
| 609 |
} |
| 610 |
} |
| 611 |
|
| 612 |
return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl); |
| 613 |
} |
| 614 |
|
| 615 |
RValue |
| 616 |
CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) { |
| 617 |
// Emit the expression as an lvalue. |
| 618 |
LValue LV = EmitLValue(E); |
| 619 |
assert(LV.isSimple()); |
| 620 |
llvm::Value *Value = LV.getPointer(*this); |
| 621 |
|
| 622 |
if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) { |
| 623 |
// C++11 [dcl.ref]p5 (as amended by core issue 453): |
| 624 |
// If a glvalue to which a reference is directly bound designates neither |
| 625 |
// an existing object or function of an appropriate type nor a region of |
| 626 |
// storage of suitable size and alignment to contain an object of the |
| 627 |
// reference's type, the behavior is undefined. |
| 628 |
QualType Ty = E->getType(); |
| 629 |
EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty); |
| 630 |
} |
| 631 |
|
| 632 |
return RValue::get(Value); |
| 633 |
} |
| 634 |
|
| 635 |
|
| 636 |
/// getAccessedFieldNo - Given an encoded value and a result number, return the |
| 637 |
/// input field number being accessed. |
| 638 |
unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx, |
| 639 |
const llvm::Constant *Elts) { |
| 640 |
return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx)) |
| 641 |
->getZExtValue(); |
| 642 |
} |
| 643 |
|
| 644 |
/// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h. |
| 645 |
static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low, |
| 646 |
llvm::Value *High) { |
| 647 |
llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL); |
| 648 |
llvm::Value *K47 = Builder.getInt64(47); |
| 649 |
llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul); |
| 650 |
llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0); |
| 651 |
llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul); |
| 652 |
llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0); |
| 653 |
return Builder.CreateMul(B1, KMul); |
| 654 |
} |
| 655 |
|
| 656 |
bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) { |
| 657 |
return TCK == TCK_DowncastPointer || TCK == TCK_Upcast || |
| 658 |
TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation; |
| 659 |
} |
| 660 |
|
| 661 |
bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) { |
| 662 |
CXXRecordDecl *RD = Ty->getAsCXXRecordDecl(); |
| 663 |
return (RD && RD->hasDefinition() && RD->isDynamicClass()) && |
| 664 |
(TCK == TCK_MemberAccess || TCK == TCK_MemberCall || |
| 665 |
TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference || |
| 666 |
TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation); |
| 667 |
} |
| 668 |
|
| 669 |
bool CodeGenFunction::sanitizePerformTypeCheck() const { |
| 670 |
return SanOpts.has(SanitizerKind::Null) || |
| 671 |
SanOpts.has(SanitizerKind::Alignment) || |
| 672 |
SanOpts.has(SanitizerKind::ObjectSize) || |
| 673 |
SanOpts.has(SanitizerKind::Vptr); |
| 674 |
} |
| 675 |
|
| 676 |
void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, |
| 677 |
llvm::Value *Ptr, QualType Ty, |
| 678 |
CharUnits Alignment, |
| 679 |
SanitizerSet SkippedChecks, |
| 680 |
llvm::Value *ArraySize) { |
| 681 |
if (!sanitizePerformTypeCheck()) |
| 682 |
return; |
| 683 |
|
| 684 |
// Don't check pointers outside the default address space. The null check |
| 685 |
// isn't correct, the object-size check isn't supported by LLVM, and we can't |
| 686 |
// communicate the addresses to the runtime handler for the vptr check. |
| 687 |
if (Ptr->getType()->getPointerAddressSpace()) |
| 688 |
return; |
| 689 |
|
| 690 |
// Don't check pointers to volatile data. The behavior here is implementation- |
| 691 |
// defined. |
| 692 |
if (Ty.isVolatileQualified()) |
| 693 |
return; |
| 694 |
|
| 695 |
SanitizerScope SanScope(this); |
| 696 |
|
| 697 |
SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks; |
| 698 |
llvm::BasicBlock *Done = nullptr; |
| 699 |
|
| 700 |
// Quickly determine whether we have a pointer to an alloca. It's possible |
| 701 |
// to skip null checks, and some alignment checks, for these pointers. This |
| 702 |
// can reduce compile-time significantly. |
| 703 |
auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts()); |
| 704 |
|
| 705 |
llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext()); |
| 706 |
llvm::Value *IsNonNull = nullptr; |
| 707 |
bool IsGuaranteedNonNull = |
| 708 |
SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca; |
| 709 |
bool AllowNullPointers = isNullPointerAllowed(TCK); |
| 710 |
if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) && |
| 711 |
!IsGuaranteedNonNull) { |
| 712 |
// The glvalue must not be an empty glvalue. |
| 713 |
IsNonNull = Builder.CreateIsNotNull(Ptr); |
| 714 |
|
| 715 |
// The IR builder can constant-fold the null check if the pointer points to |
| 716 |
// a constant. |
| 717 |
IsGuaranteedNonNull = IsNonNull == True; |
| 718 |
|
| 719 |
// Skip the null check if the pointer is known to be non-null. |
| 720 |
if (!IsGuaranteedNonNull) { |
| 721 |
if (AllowNullPointers) { |
| 722 |
// When performing pointer casts, it's OK if the value is null. |
| 723 |
// Skip the remaining checks in that case. |
| 724 |
Done = createBasicBlock("null"); |
| 725 |
llvm::BasicBlock *Rest = createBasicBlock("not.null"); |
| 726 |
Builder.CreateCondBr(IsNonNull, Rest, Done); |
| 727 |
EmitBlock(Rest); |
| 728 |
} else { |
| 729 |
Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null)); |
| 730 |
} |
| 731 |
} |
| 732 |
} |
| 733 |
|
| 734 |
if (SanOpts.has(SanitizerKind::ObjectSize) && |
| 735 |
!SkippedChecks.has(SanitizerKind::ObjectSize) && |
| 736 |
!Ty->isIncompleteType()) { |
| 737 |
uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity(); |
| 738 |
llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize); |
| 739 |
if (ArraySize) |
| 740 |
Size = Builder.CreateMul(Size, ArraySize); |
| 741 |
|
| 742 |
// Degenerate case: new X[0] does not need an objectsize check. |
| 743 |
llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size); |
| 744 |
if (!ConstantSize || !ConstantSize->isNullValue()) { |
| 745 |
// The glvalue must refer to a large enough storage region. |
| 746 |
// FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation |
| 747 |
// to check this. |
| 748 |
// FIXME: Get object address space |
| 749 |
llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy }; |
| 750 |
llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys); |
| 751 |
llvm::Value *Min = Builder.getFalse(); |
| 752 |
llvm::Value *NullIsUnknown = Builder.getFalse(); |
| 753 |
llvm::Value *Dynamic = Builder.getFalse(); |
| 754 |
llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy); |
| 755 |
llvm::Value *LargeEnough = Builder.CreateICmpUGE( |
| 756 |
Builder.CreateCall(F, {CastAddr, Min, NullIsUnknown, Dynamic}), Size); |
| 757 |
Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize)); |
| 758 |
} |
| 759 |
} |
| 760 |
|
| 761 |
llvm::MaybeAlign AlignVal; |
| 762 |
llvm::Value *PtrAsInt = nullptr; |
| 763 |
|
| 764 |
if (SanOpts.has(SanitizerKind::Alignment) && |
| 765 |
!SkippedChecks.has(SanitizerKind::Alignment)) { |
| 766 |
AlignVal = Alignment.getAsMaybeAlign(); |
| 767 |
if (!Ty->isIncompleteType() && !AlignVal) |
| 768 |
AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr, |
| 769 |
/*ForPointeeType=*/true) |
| 770 |
.getAsMaybeAlign(); |
| 771 |
|
| 772 |
// The glvalue must be suitably aligned. |
| 773 |
if (AlignVal && *AlignVal > llvm::Align(1) && |
| 774 |
(!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) { |
| 775 |
PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy); |
| 776 |
llvm::Value *Align = Builder.CreateAnd( |
| 777 |
PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1)); |
| 778 |
llvm::Value *Aligned = |
| 779 |
Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0)); |
| 780 |
if (Aligned != True) |
| 781 |
Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment)); |
| 782 |
} |
| 783 |
} |
| 784 |
|
| 785 |
if (Checks.size() > 0) { |
| 786 |
llvm::Constant *StaticData[] = { |
| 787 |
EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty), |
| 788 |
llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1), |
| 789 |
llvm::ConstantInt::get(Int8Ty, TCK)}; |
| 790 |
EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData, |
| 791 |
PtrAsInt ? PtrAsInt : Ptr); |
| 792 |
} |
| 793 |
|
| 794 |
// If possible, check that the vptr indicates that there is a subobject of |
| 795 |
// type Ty at offset zero within this object. |
| 796 |
// |
| 797 |
// C++11 [basic.life]p5,6: |
| 798 |
// [For storage which does not refer to an object within its lifetime] |
| 799 |
// The program has undefined behavior if: |
| 800 |
// -- the [pointer or glvalue] is used to access a non-static data member |
| 801 |
// or call a non-static member function |
| 802 |
if (SanOpts.has(SanitizerKind::Vptr) && |
| 803 |
!SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) { |
| 804 |
// Ensure that the pointer is non-null before loading it. If there is no |
| 805 |
// compile-time guarantee, reuse the run-time null check or emit a new one. |
| 806 |
if (!IsGuaranteedNonNull) { |
| 807 |
if (!IsNonNull) |
| 808 |
IsNonNull = Builder.CreateIsNotNull(Ptr); |
| 809 |
if (!Done) |
| 810 |
Done = createBasicBlock("vptr.null"); |
| 811 |
llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null"); |
| 812 |
Builder.CreateCondBr(IsNonNull, VptrNotNull, Done); |
| 813 |
EmitBlock(VptrNotNull); |
| 814 |
} |
| 815 |
|
| 816 |
// Compute a hash of the mangled name of the type. |
| 817 |
// |
| 818 |
// FIXME: This is not guaranteed to be deterministic! Move to a |
| 819 |
// fingerprinting mechanism once LLVM provides one. For the time |
| 820 |
// being the implementation happens to be deterministic. |
| 821 |
SmallString<64> MangledName; |
| 822 |
llvm::raw_svector_ostream Out(MangledName); |
| 823 |
CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(), |
| 824 |
Out); |
| 825 |
|
| 826 |
// Contained in NoSanitizeList based on the mangled type. |
| 827 |
if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr, |
| 828 |
Out.str())) { |
| 829 |
llvm::hash_code TypeHash = hash_value(Out.str()); |
| 830 |
|
| 831 |
// Load the vptr, and compute hash_16_bytes(TypeHash, vptr). |
| 832 |
llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash); |
| 833 |
llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0); |
| 834 |
Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), IntPtrTy, |
| 835 |
getPointerAlign()); |
| 836 |
llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr); |
| 837 |
llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty); |
| 838 |
|
| 839 |
llvm::Value *Hash = emitHash16Bytes(Builder, Low, High); |
| 840 |
Hash = Builder.CreateTrunc(Hash, IntPtrTy); |
| 841 |
|
| 842 |
// Look the hash up in our cache. |
| 843 |
const int CacheSize = 128; |
| 844 |
llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize); |
| 845 |
llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable, |
| 846 |
"__ubsan_vptr_type_cache"); |
| 847 |
llvm::Value *Slot = Builder.CreateAnd(Hash, |
| 848 |
llvm::ConstantInt::get(IntPtrTy, |
| 849 |
CacheSize-1)); |
| 850 |
llvm::Value *Indices[] = { Builder.getInt32(0), Slot }; |
| 851 |
llvm::Value *CacheVal = Builder.CreateAlignedLoad( |
| 852 |
IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices), |
| 853 |
getPointerAlign()); |
| 854 |
|
| 855 |
// If the hash isn't in the cache, call a runtime handler to perform the |
| 856 |
// hard work of checking whether the vptr is for an object of the right |
| 857 |
// type. This will either fill in the cache and return, or produce a |
| 858 |
// diagnostic. |
| 859 |
llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash); |
| 860 |
llvm::Constant *StaticData[] = { |
| 861 |
EmitCheckSourceLocation(Loc), |
| 862 |
EmitCheckTypeDescriptor(Ty), |
| 863 |
CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()), |
| 864 |
llvm::ConstantInt::get(Int8Ty, TCK) |
| 865 |
}; |
| 866 |
llvm::Value *DynamicData[] = { Ptr, Hash }; |
| 867 |
EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr), |
| 868 |
SanitizerHandler::DynamicTypeCacheMiss, StaticData, |
| 869 |
DynamicData); |
| 870 |
} |
| 871 |
} |
| 872 |
|
| 873 |
if (Done) { |
| 874 |
Builder.CreateBr(Done); |
| 875 |
EmitBlock(Done); |
| 876 |
} |
| 877 |
} |
| 878 |
|
| 879 |
llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E, |
| 880 |
QualType EltTy) { |
| 881 |
ASTContext &C = getContext(); |
| 882 |
uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity(); |
| 883 |
if (!EltSize) |
| 884 |
return nullptr; |
| 885 |
|
| 886 |
auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts()); |
| 887 |
if (!ArrayDeclRef) |
| 888 |
return nullptr; |
| 889 |
|
| 890 |
auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl()); |
| 891 |
if (!ParamDecl) |
| 892 |
return nullptr; |
| 893 |
|
| 894 |
auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>(); |
| 895 |
if (!POSAttr) |
| 896 |
return nullptr; |
| 897 |
|
| 898 |
// Don't load the size if it's a lower bound. |
| 899 |
int POSType = POSAttr->getType(); |
| 900 |
if (POSType != 0 && POSType != 1) |
| 901 |
return nullptr; |
| 902 |
|
| 903 |
// Find the implicit size parameter. |
| 904 |
auto PassedSizeIt = SizeArguments.find(ParamDecl); |
| 905 |
if (PassedSizeIt == SizeArguments.end()) |
| 906 |
return nullptr; |
| 907 |
|
| 908 |
const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second; |
| 909 |
assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable"); |
| 910 |
Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second; |
| 911 |
llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false, |
| 912 |
C.getSizeType(), E->getExprLoc()); |
| 913 |
llvm::Value *SizeOfElement = |
| 914 |
llvm::ConstantInt::get(SizeInBytes->getType(), EltSize); |
| 915 |
return Builder.CreateUDiv(SizeInBytes, SizeOfElement); |
| 916 |
} |
| 917 |
|
| 918 |
/// If Base is known to point to the start of an array, return the length of |
| 919 |
/// that array. Return 0 if the length cannot be determined. |
| 920 |
static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF, |
| 921 |
const Expr *Base, |
| 922 |
QualType &IndexedType, |
| 923 |
LangOptions::StrictFlexArraysLevelKind |
| 924 |
StrictFlexArraysLevel) { |
| 925 |
// For the vector indexing extension, the bound is the number of elements. |
| 926 |
if (const VectorType *VT = Base->getType()->getAs<VectorType>()) { |
| 927 |
IndexedType = Base->getType(); |
| 928 |
return CGF.Builder.getInt32(VT->getNumElements()); |
| 929 |
} |
| 930 |
|
| 931 |
Base = Base->IgnoreParens(); |
| 932 |
|
| 933 |
if (const auto *CE = dyn_cast<CastExpr>(Base)) { |
| 934 |
if (CE->getCastKind() == CK_ArrayToPointerDecay && |
| 935 |
!CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(), |
| 936 |
StrictFlexArraysLevel)) { |
| 937 |
IndexedType = CE->getSubExpr()->getType(); |
| 938 |
const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe(); |
| 939 |
if (const auto *CAT = dyn_cast<ConstantArrayType>(AT)) |
| 940 |
return CGF.Builder.getInt(CAT->getSize()); |
| 941 |
else if (const auto *VAT = dyn_cast<VariableArrayType>(AT)) |
| 942 |
return CGF.getVLASize(VAT).NumElts; |
| 943 |
// Ignore pass_object_size here. It's not applicable on decayed pointers. |
| 944 |
} |
| 945 |
} |
| 946 |
|
| 947 |
QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0}; |
| 948 |
if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) { |
| 949 |
IndexedType = Base->getType(); |
| 950 |
return POS; |
| 951 |
} |
| 952 |
|
| 953 |
return nullptr; |
| 954 |
} |
| 955 |
|
| 956 |
void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base, |
| 957 |
llvm::Value *Index, QualType IndexType, |
| 958 |
bool Accessed) { |
| 959 |
assert(SanOpts.has(SanitizerKind::ArrayBounds) && |
| 960 |
"should not be called unless adding bounds checks"); |
| 961 |
SanitizerScope SanScope(this); |
| 962 |
|
| 963 |
const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel = |
| 964 |
getLangOpts().getStrictFlexArraysLevel(); |
| 965 |
|
| 966 |
QualType IndexedType; |
| 967 |
llvm::Value *Bound = |
| 968 |
getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel); |
| 969 |
if (!Bound) |
| 970 |
return; |
| 971 |
|
| 972 |
bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType(); |
| 973 |
llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned); |
| 974 |
llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false); |
| 975 |
|
| 976 |
llvm::Constant *StaticData[] = { |
| 977 |
EmitCheckSourceLocation(E->getExprLoc()), |
| 978 |
EmitCheckTypeDescriptor(IndexedType), |
| 979 |
EmitCheckTypeDescriptor(IndexType) |
| 980 |
}; |
| 981 |
llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal) |
| 982 |
: Builder.CreateICmpULE(IndexVal, BoundVal); |
| 983 |
EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds), |
| 984 |
SanitizerHandler::OutOfBounds, StaticData, Index); |
| 985 |
} |
| 986 |
|
| 987 |
|
| 988 |
CodeGenFunction::ComplexPairTy CodeGenFunction:: |
| 989 |
EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV, |
| 990 |
bool isInc, bool isPre) { |
| 991 |
ComplexPairTy InVal = EmitLoadOfComplex(LV, E->getExprLoc()); |
| 992 |
|
| 993 |
llvm::Value *NextVal; |
| 994 |
if (isa<llvm::IntegerType>(InVal.first->getType())) { |
| 995 |
uint64_t AmountVal = isInc ? 1 : -1; |
| 996 |
NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true); |
| 997 |
|
| 998 |
// Add the inc/dec to the real part. |
| 999 |
NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec"); |
| 1000 |
} else { |
| 1001 |
QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType(); |
| 1002 |
llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1); |
| 1003 |
if (!isInc) |
| 1004 |
FVal.changeSign(); |
| 1005 |
NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal); |
| 1006 |
|
| 1007 |
// Add the inc/dec to the real part. |
| 1008 |
NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec"); |
| 1009 |
} |
| 1010 |
|
| 1011 |
ComplexPairTy IncVal(NextVal, InVal.second); |
| 1012 |
|
| 1013 |
// Store the updated result through the lvalue. |
| 1014 |
EmitStoreOfComplex(IncVal, LV, /*init*/ false); |
| 1015 |
if (getLangOpts().OpenMP) |
| 1016 |
CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this, |
| 1017 |
E->getSubExpr()); |
| 1018 |
|
| 1019 |
// If this is a postinc, return the value read from memory, otherwise use the |
| 1020 |
// updated value. |
| 1021 |
return isPre ? IncVal : InVal; |
| 1022 |
} |
| 1023 |
|
| 1024 |
void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E, |
| 1025 |
CodeGenFunction *CGF) { |
| 1026 |
// Bind VLAs in the cast type. |
| 1027 |
if (CGF && E->getType()->isVariablyModifiedType()) |
| 1028 |
CGF->EmitVariablyModifiedType(E->getType()); |
| 1029 |
|
| 1030 |
if (CGDebugInfo *DI = getModuleDebugInfo()) |
| 1031 |
DI->EmitExplicitCastType(E->getType()); |
| 1032 |
} |
| 1033 |
|
| 1034 |
//===----------------------------------------------------------------------===// |
| 1035 |
// LValue Expression Emission |
| 1036 |
//===----------------------------------------------------------------------===// |
| 1037 |
|
| 1038 |
/// EmitPointerWithAlignment - Given an expression of pointer type, try to |
| 1039 |
/// derive a more accurate bound on the alignment of the pointer. |
| 1040 |
Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E, |
| 1041 |
LValueBaseInfo *BaseInfo, |
| 1042 |
TBAAAccessInfo *TBAAInfo) { |
| 1043 |
// We allow this with ObjC object pointers because of fragile ABIs. |
| 1044 |
assert(E->getType()->isPointerType() || |
| 1045 |
E->getType()->isObjCObjectPointerType()); |
| 1046 |
E = E->IgnoreParens(); |
| 1047 |
|
| 1048 |
// Casts: |
| 1049 |
if (const CastExpr *CE = dyn_cast<CastExpr>(E)) { |
| 1050 |
if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE)) |
| 1051 |
CGM.EmitExplicitCastExprType(ECE, this); |
| 1052 |
|
| 1053 |
switch (CE->getCastKind()) { |
| 1054 |
// Non-converting casts (but not C's implicit conversion from void*). |
| 1055 |
case CK_BitCast: |
| 1056 |
case CK_NoOp: |
| 1057 |
case CK_AddressSpaceConversion: |
| 1058 |
if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) { |
| 1059 |
if (PtrTy->getPointeeType()->isVoidType()) |
| 1060 |
break; |
| 1061 |
|
| 1062 |
LValueBaseInfo InnerBaseInfo; |
| 1063 |
TBAAAccessInfo InnerTBAAInfo; |
| 1064 |
Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), |
| 1065 |
&InnerBaseInfo, |
| 1066 |
&InnerTBAAInfo); |
| 1067 |
if (BaseInfo) *BaseInfo = InnerBaseInfo; |
| 1068 |
if (TBAAInfo) *TBAAInfo = InnerTBAAInfo; |
| 1069 |
|
| 1070 |
if (isa<ExplicitCastExpr>(CE)) { |
| 1071 |
LValueBaseInfo TargetTypeBaseInfo; |
| 1072 |
TBAAAccessInfo TargetTypeTBAAInfo; |
| 1073 |
CharUnits Align = CGM.getNaturalPointeeTypeAlignment( |
| 1074 |
E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo); |
| 1075 |
if (TBAAInfo) |
| 1076 |
*TBAAInfo = CGM.mergeTBAAInfoForCast(*TBAAInfo, |
| 1077 |
TargetTypeTBAAInfo); |
| 1078 |
// If the source l-value is opaque, honor the alignment of the |
| 1079 |
// casted-to type. |
| 1080 |
if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) { |
| 1081 |
if (BaseInfo) |
| 1082 |
BaseInfo->mergeForCast(TargetTypeBaseInfo); |
| 1083 |
Addr = Address(Addr.getPointer(), Addr.getElementType(), Align); |
| 1084 |
} |
| 1085 |
} |
| 1086 |
|
| 1087 |
if (SanOpts.has(SanitizerKind::CFIUnrelatedCast) && |
| 1088 |
CE->getCastKind() == CK_BitCast) { |
| 1089 |
if (auto PT = E->getType()->getAs<PointerType>()) |
| 1090 |
EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr, |
| 1091 |
/*MayBeNull=*/true, |
| 1092 |
CodeGenFunction::CFITCK_UnrelatedCast, |
| 1093 |
CE->getBeginLoc()); |
| 1094 |
} |
| 1095 |
|
| 1096 |
llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType()); |
| 1097 |
Addr = Builder.CreateElementBitCast(Addr, ElemTy); |
| 1098 |
if (CE->getCastKind() == CK_AddressSpaceConversion) |
| 1099 |
Addr = Builder.CreateAddrSpaceCast(Addr, ConvertType(E->getType())); |
| 1100 |
return Addr; |
| 1101 |
} |
| 1102 |
break; |
| 1103 |
|
| 1104 |
// Array-to-pointer decay. |
| 1105 |
case CK_ArrayToPointerDecay: |
| 1106 |
return EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo); |
| 1107 |
|
| 1108 |
// Derived-to-base conversions. |
| 1109 |
case CK_UncheckedDerivedToBase: |
| 1110 |
case CK_DerivedToBase: { |
| 1111 |
// TODO: Support accesses to members of base classes in TBAA. For now, we |
| 1112 |
// conservatively pretend that the complete object is of the base class |
| 1113 |
// type. |
| 1114 |
if (TBAAInfo) |
| 1115 |
*TBAAInfo = CGM.getTBAAAccessInfo(E->getType()); |
| 1116 |
Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), BaseInfo); |
| 1117 |
auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl(); |
| 1118 |
return GetAddressOfBaseClass(Addr, Derived, |
| 1119 |
CE->path_begin(), CE->path_end(), |
| 1120 |
ShouldNullCheckClassCastValue(CE), |
| 1121 |
CE->getExprLoc()); |
| 1122 |
} |
| 1123 |
|
| 1124 |
// TODO: Is there any reason to treat base-to-derived conversions |
| 1125 |
// specially? |
| 1126 |
default: |
| 1127 |
break; |
| 1128 |
} |
| 1129 |
} |
| 1130 |
|
| 1131 |
// Unary &. |
| 1132 |
if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) { |
| 1133 |
if (UO->getOpcode() == UO_AddrOf) { |
| 1134 |
LValue LV = EmitLValue(UO->getSubExpr()); |
| 1135 |
if (BaseInfo) *BaseInfo = LV.getBaseInfo(); |
| 1136 |
if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); |
| 1137 |
return LV.getAddress(*this); |
| 1138 |
} |
| 1139 |
} |
| 1140 |
|
| 1141 |
// std::addressof and variants. |
| 1142 |
if (auto *Call = dyn_cast<CallExpr>(E)) { |
| 1143 |
switch (Call->getBuiltinCallee()) { |
| 1144 |
default: |
| 1145 |
break; |
| 1146 |
case Builtin::BIaddressof: |
| 1147 |
case Builtin::BI__addressof: |
| 1148 |
case Builtin::BI__builtin_addressof: { |
| 1149 |
LValue LV = EmitLValue(Call->getArg(0)); |
| 1150 |
if (BaseInfo) *BaseInfo = LV.getBaseInfo(); |
| 1151 |
if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo(); |
| 1152 |
return LV.getAddress(*this); |
| 1153 |
} |
| 1154 |
} |
| 1155 |
} |
| 1156 |
|
| 1157 |
// TODO: conditional operators, comma. |
| 1158 |
|
| 1159 |
// Otherwise, use the alignment of the type. |
| 1160 |
CharUnits Align = |
| 1161 |
CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo); |
| 1162 |
llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType()); |
| 1163 |
return Address(EmitScalarExpr(E), ElemTy, Align); |
| 1164 |
} |
| 1165 |
|
| 1166 |
llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) { |
| 1167 |
llvm::Value *V = RV.getScalarVal(); |
| 1168 |
if (auto MPT = T->getAs<MemberPointerType>()) |
| 1169 |
return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, V, MPT); |
| 1170 |
return Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType())); |
| 1171 |
} |
| 1172 |
|
| 1173 |
RValue CodeGenFunction::GetUndefRValue(QualType Ty) { |
| 1174 |
if (Ty->isVoidType()) |
| 1175 |
return RValue::get(nullptr); |
| 1176 |
|
| 1177 |
switch (getEvaluationKind(Ty)) { |
| 1178 |
case TEK_Complex: { |
| 1179 |
llvm::Type *EltTy = |
| 1180 |
ConvertType(Ty->castAs<ComplexType>()->getElementType()); |
| 1181 |
llvm::Value *U = llvm::UndefValue::get(EltTy); |
| 1182 |
return RValue::getComplex(std::make_pair(U, U)); |
| 1183 |
} |
| 1184 |
|
| 1185 |
// If this is a use of an undefined aggregate type, the aggregate must have an |
| 1186 |
// identifiable address. Just because the contents of the value are undefined |
| 1187 |
// doesn't mean that the address can't be taken and compared. |
| 1188 |
case TEK_Aggregate: { |
| 1189 |
Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp"); |
| 1190 |
return RValue::getAggregate(DestPtr); |
| 1191 |
} |
| 1192 |
|
| 1193 |
case TEK_Scalar: |
| 1194 |
return RValue::get(llvm::UndefValue::get(ConvertType(Ty))); |
| 1195 |
} |
| 1196 |
llvm_unreachable("bad evaluation kind"); |
| 1197 |
} |
| 1198 |
|
| 1199 |
RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E, |
| 1200 |
const char *Name) { |
| 1201 |
ErrorUnsupported(E, Name); |
| 1202 |
return GetUndefRValue(E->getType()); |
| 1203 |
} |
| 1204 |
|
| 1205 |
LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E, |
| 1206 |
const char *Name) { |
| 1207 |
ErrorUnsupported(E, Name); |
| 1208 |
llvm::Type *ElTy = ConvertType(E->getType()); |
| 1209 |
llvm::Type *Ty = llvm::PointerType::getUnqual(ElTy); |
| 1210 |
return MakeAddrLValue( |
| 1211 |
Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType()); |
| 1212 |
} |
| 1213 |
|
| 1214 |
bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) { |
| 1215 |
const Expr *Base = Obj; |
| 1216 |
while (!isa<CXXThisExpr>(Base)) { |
| 1217 |
// The result of a dynamic_cast can be null. |
| 1218 |
if (isa<CXXDynamicCastExpr>(Base)) |
| 1219 |
return false; |
| 1220 |
|
| 1221 |
if (const auto *CE = dyn_cast<CastExpr>(Base)) { |
| 1222 |
Base = CE->getSubExpr(); |
| 1223 |
} else if (const auto *PE = dyn_cast<ParenExpr>(Base)) { |
| 1224 |
Base = PE->getSubExpr(); |
| 1225 |
} else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) { |
| 1226 |
if (UO->getOpcode() == UO_Extension) |
| 1227 |
Base = UO->getSubExpr(); |
| 1228 |
else |
| 1229 |
return false; |
| 1230 |
} else { |
| 1231 |
return false; |
| 1232 |
} |
| 1233 |
} |
| 1234 |
return true; |
| 1235 |
} |
| 1236 |
|
| 1237 |
LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) { |
| 1238 |
LValue LV; |
| 1239 |
if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E)) |
| 1240 |
LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true); |
| 1241 |
else |
| 1242 |
LV = EmitLValue(E); |
| 1243 |
if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) { |
| 1244 |
SanitizerSet SkippedChecks; |
| 1245 |
if (const auto *ME = dyn_cast<MemberExpr>(E)) { |
| 1246 |
bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase()); |
| 1247 |
if (IsBaseCXXThis) |
| 1248 |
SkippedChecks.set(SanitizerKind::Alignment, true); |
| 1249 |
if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase())) |
| 1250 |
SkippedChecks.set(SanitizerKind::Null, true); |
| 1251 |
} |
| 1252 |
EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(), |
| 1253 |
LV.getAlignment(), SkippedChecks); |
| 1254 |
} |
| 1255 |
return LV; |
| 1256 |
} |
| 1257 |
|
| 1258 |
/// EmitLValue - Emit code to compute a designator that specifies the location |
| 1259 |
/// of the expression. |
| 1260 |
/// |
| 1261 |
/// This can return one of two things: a simple address or a bitfield reference. |
| 1262 |
/// In either case, the LLVM Value* in the LValue structure is guaranteed to be |
| 1263 |
/// an LLVM pointer type. |
| 1264 |
/// |
| 1265 |
/// If this returns a bitfield reference, nothing about the pointee type of the |
| 1266 |
/// LLVM value is known: For example, it may not be a pointer to an integer. |
| 1267 |
/// |
| 1268 |
/// If this returns a normal address, and if the lvalue's C type is fixed size, |
| 1269 |
/// this method guarantees that the returned pointer type will point to an LLVM |
| 1270 |
/// type of the same size of the lvalue's type. If the lvalue has a variable |
| 1271 |
/// length type, this is not possible. |
| 1272 |
/// |
| 1273 |
LValue CodeGenFunction::EmitLValue(const Expr *E) { |
| 1274 |
ApplyDebugLocation DL(*this, E); |
| 1275 |
switch (E->getStmtClass()) { |
| 1276 |
default: return EmitUnsupportedLValue(E, "l-value expression"); |
| 1277 |
|
| 1278 |
case Expr::ObjCPropertyRefExprClass: |
| 1279 |
llvm_unreachable("cannot emit a property reference directly"); |
| 1280 |
|
| 1281 |
case Expr::ObjCSelectorExprClass: |
| 1282 |
return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E)); |
| 1283 |
case Expr::ObjCIsaExprClass: |
| 1284 |
return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E)); |
| 1285 |
case Expr::BinaryOperatorClass: |
| 1286 |
return EmitBinaryOperatorLValue(cast<BinaryOperator>(E)); |
| 1287 |
case Expr::CompoundAssignOperatorClass: { |
| 1288 |
QualType Ty = E->getType(); |
| 1289 |
if (const AtomicType *AT = Ty->getAs<AtomicType>()) |
| 1290 |
Ty = AT->getValueType(); |
| 1291 |
if (!Ty->isAnyComplexType()) |
| 1292 |
return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E)); |
| 1293 |
return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E)); |
| 1294 |
} |
| 1295 |
case Expr::CallExprClass: |
| 1296 |
case Expr::CXXMemberCallExprClass: |
| 1297 |
case Expr::CXXOperatorCallExprClass: |
| 1298 |
case Expr::UserDefinedLiteralClass: |
| 1299 |
return EmitCallExprLValue(cast<CallExpr>(E)); |
| 1300 |
case Expr::CXXRewrittenBinaryOperatorClass: |
| 1301 |
return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm()); |
| 1302 |
case Expr::VAArgExprClass: |
| 1303 |
return EmitVAArgExprLValue(cast<VAArgExpr>(E)); |
| 1304 |
case Expr::DeclRefExprClass: |
| 1305 |
return EmitDeclRefLValue(cast<DeclRefExpr>(E)); |
| 1306 |
case Expr::ConstantExprClass: { |
| 1307 |
const ConstantExpr *CE = cast<ConstantExpr>(E); |
| 1308 |
if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) { |
| 1309 |
QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit()) |
| 1310 |
->getCallReturnType(getContext()) |
| 1311 |
->getPointeeType(); |
| 1312 |
return MakeNaturalAlignAddrLValue(Result, RetType); |
| 1313 |
} |
| 1314 |
return EmitLValue(cast<ConstantExpr>(E)->getSubExpr()); |
| 1315 |
} |
| 1316 |
case Expr::ParenExprClass: |
| 1317 |
return EmitLValue(cast<ParenExpr>(E)->getSubExpr()); |
| 1318 |
case Expr::GenericSelectionExprClass: |
| 1319 |
return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr()); |
| 1320 |
case Expr::PredefinedExprClass: |
| 1321 |
return EmitPredefinedLValue(cast<PredefinedExpr>(E)); |
| 1322 |
case Expr::StringLiteralClass: |
| 1323 |
return EmitStringLiteralLValue(cast<StringLiteral>(E)); |
| 1324 |
case Expr::ObjCEncodeExprClass: |
| 1325 |
return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E)); |
| 1326 |
case Expr::PseudoObjectExprClass: |
| 1327 |
return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E)); |
| 1328 |
case Expr::InitListExprClass: |
| 1329 |
return EmitInitListLValue(cast<InitListExpr>(E)); |
| 1330 |
case Expr::CXXTemporaryObjectExprClass: |
| 1331 |
case Expr::CXXConstructExprClass: |
| 1332 |
return EmitCXXConstructLValue(cast<CXXConstructExpr>(E)); |
| 1333 |
case Expr::CXXBindTemporaryExprClass: |
| 1334 |
return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E)); |
| 1335 |
case Expr::CXXUuidofExprClass: |
| 1336 |
return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E)); |
| 1337 |
case Expr::LambdaExprClass: |
| 1338 |
return EmitAggExprToLValue(E); |
| 1339 |
|
| 1340 |
case Expr::ExprWithCleanupsClass: { |
| 1341 |
const auto *cleanups = cast<ExprWithCleanups>(E); |
| 1342 |
RunCleanupsScope Scope(*this); |
| 1343 |
LValue LV = EmitLValue(cleanups->getSubExpr()); |
| 1344 |
if (LV.isSimple()) { |
| 1345 |
// Defend against branches out of gnu statement expressions surrounded by |
| 1346 |
// cleanups. |
| 1347 |
Address Addr = LV.getAddress(*this); |
| 1348 |
llvm::Value *V = Addr.getPointer(); |
| 1349 |
Scope.ForceCleanup({&V}); |
| 1350 |
return LValue::MakeAddr(Addr.withPointer(V), LV.getType(), getContext(), |
| 1351 |
LV.getBaseInfo(), LV.getTBAAInfo()); |
| 1352 |
} |
| 1353 |
// FIXME: Is it possible to create an ExprWithCleanups that produces a |
| 1354 |
// bitfield lvalue or some other non-simple lvalue? |
| 1355 |
return LV; |
| 1356 |
} |
| 1357 |
|
| 1358 |
case Expr::CXXDefaultArgExprClass: { |
| 1359 |
auto *DAE = cast<CXXDefaultArgExpr>(E); |
| 1360 |
CXXDefaultArgExprScope Scope(*this, DAE); |
| 1361 |
return EmitLValue(DAE->getExpr()); |
| 1362 |
} |
| 1363 |
case Expr::CXXDefaultInitExprClass: { |
| 1364 |
auto *DIE = cast<CXXDefaultInitExpr>(E); |
| 1365 |
CXXDefaultInitExprScope Scope(*this, DIE); |
| 1366 |
return EmitLValue(DIE->getExpr()); |
| 1367 |
} |
| 1368 |
case Expr::CXXTypeidExprClass: |
| 1369 |
return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E)); |
| 1370 |
|
| 1371 |
case Expr::ObjCMessageExprClass: |
| 1372 |
return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E)); |
| 1373 |
case Expr::ObjCIvarRefExprClass: |
| 1374 |
return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E)); |
| 1375 |
case Expr::StmtExprClass: |
| 1376 |
return EmitStmtExprLValue(cast<StmtExpr>(E)); |
| 1377 |
case Expr::UnaryOperatorClass: |
| 1378 |
return EmitUnaryOpLValue(cast<UnaryOperator>(E)); |
| 1379 |
case Expr::ArraySubscriptExprClass: |
| 1380 |
return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E)); |
| 1381 |
case Expr::MatrixSubscriptExprClass: |
| 1382 |
return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E)); |
| 1383 |
case Expr::OMPArraySectionExprClass: |
| 1384 |
return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E)); |
| 1385 |
case Expr::ExtVectorElementExprClass: |
| 1386 |
return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E)); |
| 1387 |
case Expr::CXXThisExprClass: |
| 1388 |
return MakeAddrLValue(LoadCXXThisAddress(), E->getType()); |
| 1389 |
case Expr::MemberExprClass: |
| 1390 |
return EmitMemberExpr(cast<MemberExpr>(E)); |
| 1391 |
case Expr::CompoundLiteralExprClass: |
| 1392 |
return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E)); |
| 1393 |
case Expr::ConditionalOperatorClass: |
| 1394 |
return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E)); |
| 1395 |
case Expr::BinaryConditionalOperatorClass: |
| 1396 |
return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E)); |
| 1397 |
case Expr::ChooseExprClass: |
| 1398 |
return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr()); |
| 1399 |
case Expr::OpaqueValueExprClass: |
| 1400 |
return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E)); |
| 1401 |
case Expr::SubstNonTypeTemplateParmExprClass: |
| 1402 |
return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement()); |
| 1403 |
case Expr::ImplicitCastExprClass: |
| 1404 |
case Expr::CStyleCastExprClass: |
| 1405 |
case Expr::CXXFunctionalCastExprClass: |
| 1406 |
case Expr::CXXStaticCastExprClass: |
| 1407 |
case Expr::CXXDynamicCastExprClass: |
| 1408 |
case Expr::CXXReinterpretCastExprClass: |
| 1409 |
case Expr::CXXConstCastExprClass: |
| 1410 |
case Expr::CXXAddrspaceCastExprClass: |
| 1411 |
case Expr::ObjCBridgedCastExprClass: |
| 1412 |
return EmitCastLValue(cast<CastExpr>(E)); |
| 1413 |
|
| 1414 |
case Expr::MaterializeTemporaryExprClass: |
| 1415 |
return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E)); |
| 1416 |
|
| 1417 |
case Expr::CoawaitExprClass: |
| 1418 |
return EmitCoawaitLValue(cast<CoawaitExpr>(E)); |
| 1419 |
case Expr::CoyieldExprClass: |
| 1420 |
return EmitCoyieldLValue(cast<CoyieldExpr>(E)); |
| 1421 |
} |
| 1422 |
} |
| 1423 |
|
| 1424 |
/// Given an object of the given canonical type, can we safely copy a |
| 1425 |
/// value out of it based on its initializer? |
| 1426 |
static bool isConstantEmittableObjectType(QualType type) { |
| 1427 |
assert(type.isCanonical()); |
| 1428 |
assert(!type->isReferenceType()); |
| 1429 |
|
| 1430 |
// Must be const-qualified but non-volatile. |
| 1431 |
Qualifiers qs = type.getLocalQualifiers(); |
| 1432 |
if (!qs.hasConst() || qs.hasVolatile()) return false; |
| 1433 |
|
| 1434 |
// Otherwise, all object types satisfy this except C++ classes with |
| 1435 |
// mutable subobjects or non-trivial copy/destroy behavior. |
| 1436 |
if (const auto *RT = dyn_cast<RecordType>(type)) |
| 1437 |
if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl())) |
| 1438 |
if (RD->hasMutableFields() || !RD->isTrivial()) |
| 1439 |
return false; |
| 1440 |
|
| 1441 |
return true; |
| 1442 |
} |
| 1443 |
|
| 1444 |
/// Can we constant-emit a load of a reference to a variable of the |
| 1445 |
/// given type? This is different from predicates like |
| 1446 |
/// Decl::mightBeUsableInConstantExpressions because we do want it to apply |
| 1447 |
/// in situations that don't necessarily satisfy the language's rules |
| 1448 |
/// for this (e.g. C++'s ODR-use rules). For example, we want to able |
| 1449 |
/// to do this with const float variables even if those variables |
| 1450 |
/// aren't marked 'constexpr'. |
| 1451 |
enum ConstantEmissionKind { |
| 1452 |
CEK_None, |
| 1453 |
CEK_AsReferenceOnly, |
| 1454 |
CEK_AsValueOrReference, |
| 1455 |
CEK_AsValueOnly |
| 1456 |
}; |
| 1457 |
static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) { |
| 1458 |
type = type.getCanonicalType(); |
| 1459 |
if (const auto *ref = dyn_cast<ReferenceType>(type)) { |
| 1460 |
if (isConstantEmittableObjectType(ref->getPointeeType())) |
| 1461 |
return CEK_AsValueOrReference; |
| 1462 |
return CEK_AsReferenceOnly; |
| 1463 |
} |
| 1464 |
if (isConstantEmittableObjectType(type)) |
| 1465 |
return CEK_AsValueOnly; |
| 1466 |
return CEK_None; |
| 1467 |
} |
| 1468 |
|
| 1469 |
/// Try to emit a reference to the given value without producing it as |
| 1470 |
/// an l-value. This is just an optimization, but it avoids us needing |
| 1471 |
/// to emit global copies of variables if they're named without triggering |
| 1472 |
/// a formal use in a context where we can't emit a direct reference to them, |
| 1473 |
/// for instance if a block or lambda or a member of a local class uses a |
| 1474 |
/// const int variable or constexpr variable from an enclosing function. |
| 1475 |
CodeGenFunction::ConstantEmission |
| 1476 |
CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) { |
| 1477 |
ValueDecl *value = refExpr->getDecl(); |
| 1478 |
|
| 1479 |
// The value needs to be an enum constant or a constant variable. |
| 1480 |
ConstantEmissionKind CEK; |
| 1481 |
if (isa<ParmVarDecl>(value)) { |
| 1482 |
CEK = CEK_None; |
| 1483 |
} else if (auto *var = dyn_cast<VarDecl>(value)) { |
| 1484 |
CEK = checkVarTypeForConstantEmission(var->getType()); |
| 1485 |
} else if (isa<EnumConstantDecl>(value)) { |
| 1486 |
CEK = CEK_AsValueOnly; |
| 1487 |
} else { |
| 1488 |
CEK = CEK_None; |
| 1489 |
} |
| 1490 |
if (CEK == CEK_None) return ConstantEmission(); |
| 1491 |
|
| 1492 |
Expr::EvalResult result; |
| 1493 |
bool resultIsReference; |
| 1494 |
QualType resultType; |
| 1495 |
|
| 1496 |
// It's best to evaluate all the way as an r-value if that's permitted. |
| 1497 |
if (CEK != CEK_AsReferenceOnly && |
| 1498 |
refExpr->EvaluateAsRValue(result, getContext())) { |
| 1499 |
resultIsReference = false; |
| 1500 |
resultType = refExpr->getType(); |
| 1501 |
|
| 1502 |
// Otherwise, try to evaluate as an l-value. |
| 1503 |
} else if (CEK != CEK_AsValueOnly && |
| 1504 |
refExpr->EvaluateAsLValue(result, getContext())) { |
| 1505 |
resultIsReference = true; |
| 1506 |
resultType = value->getType(); |
| 1507 |
|
| 1508 |
// Failure. |
| 1509 |
} else { |
| 1510 |
return ConstantEmission(); |
| 1511 |
} |
| 1512 |
|
| 1513 |
// In any case, if the initializer has side-effects, abandon ship. |
| 1514 |
if (result.HasSideEffects) |
| 1515 |
return ConstantEmission(); |
| 1516 |
|
| 1517 |
// In CUDA/HIP device compilation, a lambda may capture a reference variable |
| 1518 |
// referencing a global host variable by copy. In this case the lambda should |
| 1519 |
// make a copy of the value of the global host variable. The DRE of the |
| 1520 |
// captured reference variable cannot be emitted as load from the host |
| 1521 |
// global variable as compile time constant, since the host variable is not |
| 1522 |
// accessible on device. The DRE of the captured reference variable has to be |
| 1523 |
// loaded from captures. |
| 1524 |
if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() && |
| 1525 |
refExpr->refersToEnclosingVariableOrCapture()) { |
| 1526 |
auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl); |
| 1527 |
if (MD && MD->getParent()->isLambda() && |
| 1528 |
MD->getOverloadedOperator() == OO_Call) { |
| 1529 |
const APValue::LValueBase &base = result.Val.getLValueBase(); |
| 1530 |
if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) { |
| 1531 |
if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) { |
| 1532 |
if (!VD->hasAttr<CUDADeviceAttr>()) { |
| 1533 |
return ConstantEmission(); |
| 1534 |
} |
| 1535 |
} |
| 1536 |
} |
| 1537 |
} |
| 1538 |
} |
| 1539 |
|
| 1540 |
// Emit as a constant. |
| 1541 |
auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(), |
| 1542 |
result.Val, resultType); |
| 1543 |
|
| 1544 |
// Make sure we emit a debug reference to the global variable. |
| 1545 |
// This should probably fire even for |
| 1546 |
if (isa<VarDecl>(value)) { |
| 1547 |
if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value))) |
| 1548 |
EmitDeclRefExprDbgValue(refExpr, result.Val); |
| 1549 |
} else { |
| 1550 |
assert(isa<EnumConstantDecl>(value)); |
| 1551 |
EmitDeclRefExprDbgValue(refExpr, result.Val); |
| 1552 |
} |
| 1553 |
|
| 1554 |
// If we emitted a reference constant, we need to dereference that. |
| 1555 |
if (resultIsReference) |
| 1556 |
return ConstantEmission::forReference(C); |
| 1557 |
|
| 1558 |
return ConstantEmission::forValue(C); |
| 1559 |
} |
| 1560 |
|
| 1561 |
static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF, |
| 1562 |
const MemberExpr *ME) { |
| 1563 |
if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) { |
| 1564 |
// Try to emit static variable member expressions as DREs. |
| 1565 |
return DeclRefExpr::Create( |
| 1566 |
CGF.getContext(), NestedNameSpecifierLoc(), SourceLocation(), VD, |
| 1567 |
/*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(), |
| 1568 |
ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse()); |
| 1569 |
} |
| 1570 |
return nullptr; |
| 1571 |
} |
| 1572 |
|
| 1573 |
CodeGenFunction::ConstantEmission |
| 1574 |
CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) { |
| 1575 |
if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, ME)) |
| 1576 |
return tryEmitAsConstant(DRE); |
| 1577 |
return ConstantEmission(); |
| 1578 |
} |
| 1579 |
|
| 1580 |
llvm::Value *CodeGenFunction::emitScalarConstant( |
| 1581 |
const CodeGenFunction::ConstantEmission &Constant, Expr *E) { |
| 1582 |
assert(Constant && "not a constant"); |
| 1583 |
if (Constant.isReference()) |
| 1584 |
return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E), |
| 1585 |
E->getExprLoc()) |
| 1586 |
.getScalarVal(); |
| 1587 |
return Constant.getValue(); |
| 1588 |
} |
| 1589 |
|
| 1590 |
llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue, |
| 1591 |
SourceLocation Loc) { |
| 1592 |
return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(), |
| 1593 |
lvalue.getType(), Loc, lvalue.getBaseInfo(), |
| 1594 |
lvalue.getTBAAInfo(), lvalue.isNontemporal()); |
| 1595 |
} |
| 1596 |
|
| 1597 |
static bool hasBooleanRepresentation(QualType Ty) { |
| 1598 |
if (Ty->isBooleanType()) |
| 1599 |
return true; |
| 1600 |
|
| 1601 |
if (const EnumType *ET = Ty->getAs<EnumType>()) |
| 1602 |
return ET->getDecl()->getIntegerType()->isBooleanType(); |
| 1603 |
|
| 1604 |
if (const AtomicType *AT = Ty->getAs<AtomicType>()) |
| 1605 |
return hasBooleanRepresentation(AT->getValueType()); |
| 1606 |
|
| 1607 |
return false; |
| 1608 |
} |
| 1609 |
|
| 1610 |
static bool getRangeForType(CodeGenFunction &CGF, QualType Ty, |
| 1611 |
llvm::APInt &Min, llvm::APInt &End, |
| 1612 |
bool StrictEnums, bool IsBool) { |
| 1613 |
const EnumType *ET = Ty->getAs<EnumType>(); |
| 1614 |
bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums && |
| 1615 |
ET && !ET->getDecl()->isFixed(); |
| 1616 |
if (!IsBool && !IsRegularCPlusPlusEnum) |
| 1617 |
return false; |
| 1618 |
|
| 1619 |
if (IsBool) { |
| 1620 |
Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0); |
| 1621 |
End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2); |
| 1622 |
} else { |
| 1623 |
const EnumDecl *ED = ET->getDecl(); |
| 1624 |
ED->getValueRange(End, Min); |
| 1625 |
} |
| 1626 |
return true; |
| 1627 |
} |
| 1628 |
|
| 1629 |
llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) { |
| 1630 |
llvm::APInt Min, End; |
| 1631 |
if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums, |
| 1632 |
hasBooleanRepresentation(Ty))) |
| 1633 |
return nullptr; |
| 1634 |
|
| 1635 |
llvm::MDBuilder MDHelper(getLLVMContext()); |
| 1636 |
return MDHelper.createRange(Min, End); |
| 1637 |
} |
| 1638 |
|
| 1639 |
bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty, |
| 1640 |
SourceLocation Loc) { |
| 1641 |
bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool); |
| 1642 |
bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum); |
| 1643 |
if (!HasBoolCheck && !HasEnumCheck) |
| 1644 |
return false; |
| 1645 |
|
| 1646 |
bool IsBool = hasBooleanRepresentation(Ty) || |
| 1647 |
NSAPI(CGM.getContext()).isObjCBOOLType(Ty); |
| 1648 |
bool NeedsBoolCheck = HasBoolCheck && IsBool; |
| 1649 |
bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>(); |
| 1650 |
if (!NeedsBoolCheck && !NeedsEnumCheck) |
| 1651 |
return false; |
| 1652 |
|
| 1653 |
// Single-bit booleans don't need to be checked. Special-case this to avoid |
| 1654 |
// a bit width mismatch when handling bitfield values. This is handled by |
| 1655 |
// EmitFromMemory for the non-bitfield case. |
| 1656 |
if (IsBool && |
| 1657 |
cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1) |
| 1658 |
return false; |
| 1659 |
|
| 1660 |
llvm::APInt Min, End; |
| 1661 |
if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool)) |
| 1662 |
return true; |
| 1663 |
|
| 1664 |
auto &Ctx = getLLVMContext(); |
| 1665 |
SanitizerScope SanScope(this); |
| 1666 |
llvm::Value *Check; |
| 1667 |
--End; |
| 1668 |
if (!Min) { |
| 1669 |
Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End)); |
| 1670 |
} else { |
| 1671 |
llvm::Value *Upper = |
| 1672 |
Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End)); |
| 1673 |
llvm::Value *Lower = |
| 1674 |
Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min)); |
| 1675 |
Check = Builder.CreateAnd(Upper, Lower); |
| 1676 |
} |
| 1677 |
llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc), |
| 1678 |
EmitCheckTypeDescriptor(Ty)}; |
| 1679 |
SanitizerMask Kind = |
| 1680 |
NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool; |
| 1681 |
EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue, |
| 1682 |
StaticArgs, EmitCheckValue(Value)); |
| 1683 |
return true; |
| 1684 |
} |
| 1685 |
|
| 1686 |
llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile, |
| 1687 |
QualType Ty, |
| 1688 |
SourceLocation Loc, |
| 1689 |
LValueBaseInfo BaseInfo, |
| 1690 |
TBAAAccessInfo TBAAInfo, |
| 1691 |
bool isNontemporal) { |
| 1692 |
if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer())) |
| 1693 |
if (GV->isThreadLocal()) |
| 1694 |
Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV)); |
| 1695 |
|
| 1696 |
if (const auto *ClangVecTy = Ty->getAs<VectorType>()) { |
| 1697 |
// Boolean vectors use `iN` as storage type. |
| 1698 |
if (ClangVecTy->isExtVectorBoolType()) { |
| 1699 |
llvm::Type *ValTy = ConvertType(Ty); |
| 1700 |
unsigned ValNumElems = |
| 1701 |
cast<llvm::FixedVectorType>(ValTy)->getNumElements(); |
| 1702 |
// Load the `iP` storage object (P is the padded vector size). |
| 1703 |
auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits"); |
| 1704 |
const auto *RawIntTy = RawIntV->getType(); |
| 1705 |
assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors"); |
| 1706 |
// Bitcast iP --> <P x i1>. |
| 1707 |
auto *PaddedVecTy = llvm::FixedVectorType::get( |
| 1708 |
Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits()); |
| 1709 |
llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy); |
| 1710 |
// Shuffle <P x i1> --> <N x i1> (N is the actual bit size). |
| 1711 |
V = emitBoolVecConversion(V, ValNumElems, "extractvec"); |
| 1712 |
|
| 1713 |
return EmitFromMemory(V, Ty); |
| 1714 |
} |
| 1715 |
|
| 1716 |
// Handle vectors of size 3 like size 4 for better performance. |
| 1717 |
const llvm::Type *EltTy = Addr.getElementType(); |
| 1718 |
const auto *VTy = cast<llvm::FixedVectorType>(EltTy); |
| 1719 |
|
| 1720 |
if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) { |
| 1721 |
|
| 1722 |
// Bitcast to vec4 type. |
| 1723 |
llvm::VectorType *vec4Ty = |
| 1724 |
llvm::FixedVectorType::get(VTy->getElementType(), 4); |
| 1725 |
Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4"); |
| 1726 |
// Now load value. |
| 1727 |
llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4"); |
| 1728 |
|
| 1729 |
// Shuffle vector to get vec3. |
| 1730 |
V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec"); |
| 1731 |
return EmitFromMemory(V, Ty); |
| 1732 |
} |
| 1733 |
} |
| 1734 |
|
| 1735 |
// Atomic operations have to be done on integral types. |
| 1736 |
LValue AtomicLValue = |
| 1737 |
LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo); |
| 1738 |
if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) { |
| 1739 |
return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal(); |
| 1740 |
} |
| 1741 |
|
| 1742 |
llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile); |
| 1743 |
if (isNontemporal) { |
| 1744 |
llvm::MDNode *Node = llvm::MDNode::get( |
| 1745 |
Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1))); |
| 1746 |
Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); |
| 1747 |
} |
| 1748 |
|
| 1749 |
CGM.DecorateInstructionWithTBAA(Load, TBAAInfo); |
| 1750 |
|
| 1751 |
if (EmitScalarRangeCheck(Load, Ty, Loc)) { |
| 1752 |
// In order to prevent the optimizer from throwing away the check, don't |
| 1753 |
// attach range metadata to the load. |
| 1754 |
} else if (CGM.getCodeGenOpts().OptimizationLevel > 0) |
| 1755 |
if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) { |
| 1756 |
Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo); |
| 1757 |
Load->setMetadata(llvm::LLVMContext::MD_noundef, |
| 1758 |
llvm::MDNode::get(getLLVMContext(), std::nullopt)); |
| 1759 |
} |
| 1760 |
|
| 1761 |
return EmitFromMemory(Load, Ty); |
| 1762 |
} |
| 1763 |
|
| 1764 |
llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) { |
| 1765 |
// Bool has a different representation in memory than in registers. |
| 1766 |
if (hasBooleanRepresentation(Ty)) { |
| 1767 |
// This should really always be an i1, but sometimes it's already |
| 1768 |
// an i8, and it's awkward to track those cases down. |
| 1769 |
if (Value->getType()->isIntegerTy(1)) |
| 1770 |
return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool"); |
| 1771 |
assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) && |
| 1772 |
"wrong value rep of bool"); |
| 1773 |
} |
| 1774 |
|
| 1775 |
return Value; |
| 1776 |
} |
| 1777 |
|
| 1778 |
llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) { |
| 1779 |
// Bool has a different representation in memory than in registers. |
| 1780 |
if (hasBooleanRepresentation(Ty)) { |
| 1781 |
assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) && |
| 1782 |
"wrong value rep of bool"); |
| 1783 |
return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool"); |
| 1784 |
} |
| 1785 |
if (Ty->isExtVectorBoolType()) { |
| 1786 |
const auto *RawIntTy = Value->getType(); |
| 1787 |
// Bitcast iP --> <P x i1>. |
| 1788 |
auto *PaddedVecTy = llvm::FixedVectorType::get( |
| 1789 |
Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits()); |
| 1790 |
auto *V = Builder.CreateBitCast(Value, PaddedVecTy); |
| 1791 |
// Shuffle <P x i1> --> <N x i1> (N is the actual bit size). |
| 1792 |
llvm::Type *ValTy = ConvertType(Ty); |
| 1793 |
unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements(); |
| 1794 |
return emitBoolVecConversion(V, ValNumElems, "extractvec"); |
| 1795 |
} |
| 1796 |
|
| 1797 |
return Value; |
| 1798 |
} |
| 1799 |
|
| 1800 |
// Convert the pointer of \p Addr to a pointer to a vector (the value type of |
| 1801 |
// MatrixType), if it points to a array (the memory type of MatrixType). |
| 1802 |
static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF, |
| 1803 |
bool IsVector = true) { |
| 1804 |
auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType()); |
| 1805 |
if (ArrayTy && IsVector) { |
| 1806 |
auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(), |
| 1807 |
ArrayTy->getNumElements()); |
| 1808 |
|
| 1809 |
return Address(CGF.Builder.CreateElementBitCast(Addr, VectorTy)); |
| 1810 |
} |
| 1811 |
auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType()); |
| 1812 |
if (VectorTy && !IsVector) { |
| 1813 |
auto *ArrayTy = llvm::ArrayType::get( |
| 1814 |
VectorTy->getElementType(), |
| 1815 |
cast<llvm::FixedVectorType>(VectorTy)->getNumElements()); |
| 1816 |
|
| 1817 |
return Address(CGF.Builder.CreateElementBitCast(Addr, ArrayTy)); |
| 1818 |
} |
| 1819 |
|
| 1820 |
return Addr; |
| 1821 |
} |
| 1822 |
|
| 1823 |
// Emit a store of a matrix LValue. This may require casting the original |
| 1824 |
// pointer to memory address (ArrayType) to a pointer to the value type |
| 1825 |
// (VectorType). |
| 1826 |
static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue, |
| 1827 |
bool isInit, CodeGenFunction &CGF) { |
| 1828 |
Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF, |
| 1829 |
value->getType()->isVectorTy()); |
| 1830 |
CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(), |
| 1831 |
lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit, |
| 1832 |
lvalue.isNontemporal()); |
| 1833 |
} |
| 1834 |
|
| 1835 |
void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr, |
| 1836 |
bool Volatile, QualType Ty, |
| 1837 |
LValueBaseInfo BaseInfo, |
| 1838 |
TBAAAccessInfo TBAAInfo, |
| 1839 |
bool isInit, bool isNontemporal) { |
| 1840 |
if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer())) |
| 1841 |
if (GV->isThreadLocal()) |
| 1842 |
Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV)); |
| 1843 |
|
| 1844 |
llvm::Type *SrcTy = Value->getType(); |
| 1845 |
if (const auto *ClangVecTy = Ty->getAs<VectorType>()) { |
| 1846 |
auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy); |
| 1847 |
if (VecTy && ClangVecTy->isExtVectorBoolType()) { |
| 1848 |
auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType()); |
| 1849 |
// Expand to the memory bit width. |
| 1850 |
unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits(); |
| 1851 |
// <N x i1> --> <P x i1>. |
| 1852 |
Value = emitBoolVecConversion(Value, MemNumElems, "insertvec"); |
| 1853 |
// <P x i1> --> iP. |
| 1854 |
Value = Builder.CreateBitCast(Value, MemIntTy); |
| 1855 |
} else if (!CGM.getCodeGenOpts().PreserveVec3Type) { |
| 1856 |
// Handle vec3 special. |
| 1857 |
if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) { |
| 1858 |
// Our source is a vec3, do a shuffle vector to make it a vec4. |
| 1859 |
Value = Builder.CreateShuffleVector(Value, ArrayRef<int>{0, 1, 2, -1}, |
| 1860 |
"extractVec"); |
| 1861 |
SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4); |
| 1862 |
} |
| 1863 |
if (Addr.getElementType() != SrcTy) { |
| 1864 |
Addr = Builder.CreateElementBitCast(Addr, SrcTy, "storetmp"); |
| 1865 |
} |
| 1866 |
} |
| 1867 |
} |
| 1868 |
|
| 1869 |
Value = EmitToMemory(Value, Ty); |
| 1870 |
|
| 1871 |
LValue AtomicLValue = |
| 1872 |
LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo); |
| 1873 |
if (Ty->isAtomicType() || |
| 1874 |
(!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) { |
| 1875 |
EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit); |
| 1876 |
return; |
| 1877 |
} |
| 1878 |
|
| 1879 |
llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile); |
| 1880 |
if (isNontemporal) { |
| 1881 |
llvm::MDNode *Node = |
| 1882 |
llvm::MDNode::get(Store->getContext(), |
| 1883 |
llvm::ConstantAsMetadata::get(Builder.getInt32(1))); |
| 1884 |
Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node); |
| 1885 |
} |
| 1886 |
|
| 1887 |
CGM.DecorateInstructionWithTBAA(Store, TBAAInfo); |
| 1888 |
} |
| 1889 |
|
| 1890 |
void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue, |
| 1891 |
bool isInit) { |
| 1892 |
if (lvalue.getType()->isConstantMatrixType()) { |
| 1893 |
EmitStoreOfMatrixScalar(value, lvalue, isInit, *this); |
| 1894 |
return; |
| 1895 |
} |
| 1896 |
|
| 1897 |
EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(), |
| 1898 |
lvalue.getType(), lvalue.getBaseInfo(), |
| 1899 |
lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal()); |
| 1900 |
} |
| 1901 |
|
| 1902 |
// Emit a load of a LValue of matrix type. This may require casting the pointer |
| 1903 |
// to memory address (ArrayType) to a pointer to the value type (VectorType). |
| 1904 |
static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc, |
| 1905 |
CodeGenFunction &CGF) { |
| 1906 |
assert(LV.getType()->isConstantMatrixType()); |
| 1907 |
Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF); |
| 1908 |
LV.setAddress(Addr); |
| 1909 |
return RValue::get(CGF.EmitLoadOfScalar(LV, Loc)); |
| 1910 |
} |
| 1911 |
|
| 1912 |
/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this |
| 1913 |
/// method emits the address of the lvalue, then loads the result as an rvalue, |
| 1914 |
/// returning the rvalue. |
| 1915 |
RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { |
| 1916 |
if (LV.isObjCWeak()) { |
| 1917 |
// load of a __weak object. |
| 1918 |
Address AddrWeakObj = LV.getAddress(*this); |
| 1919 |
return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this, |
| 1920 |
AddrWeakObj)); |
| 1921 |
} |
| 1922 |
if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) { |
| 1923 |
// In MRC mode, we do a load+autorelease. |
| 1924 |
if (!getLangOpts().ObjCAutoRefCount) { |
| 1925 |
return RValue::get(EmitARCLoadWeak(LV.getAddress(*this))); |
| 1926 |
} |
| 1927 |
|
| 1928 |
// In ARC mode, we load retained and then consume the value. |
| 1929 |
llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this)); |
| 1930 |
Object = EmitObjCConsumeObject(LV.getType(), Object); |
| 1931 |
return RValue::get(Object); |
| 1932 |
} |
| 1933 |
|
| 1934 |
if (LV.isSimple()) { |
| 1935 |
assert(!LV.getType()->isFunctionType()); |
| 1936 |
|
| 1937 |
if (LV.getType()->isConstantMatrixType()) |
| 1938 |
return EmitLoadOfMatrixLValue(LV, Loc, *this); |
| 1939 |
|
| 1940 |
// Everything needs a load. |
| 1941 |
return RValue::get(EmitLoadOfScalar(LV, Loc)); |
| 1942 |
} |
| 1943 |
|
| 1944 |
if (LV.isVectorElt()) { |
| 1945 |
llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(), |
| 1946 |
LV.isVolatileQualified()); |
| 1947 |
return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(), |
| 1948 |
"vecext")); |
| 1949 |
} |
| 1950 |
|
| 1951 |
// If this is a reference to a subset of the elements of a vector, either |
| 1952 |
// shuffle the input or extract/insert them as appropriate. |
| 1953 |
if (LV.isExtVectorElt()) { |
| 1954 |
return EmitLoadOfExtVectorElementLValue(LV); |
| 1955 |
} |
| 1956 |
|
| 1957 |
// Global Register variables always invoke intrinsics |
| 1958 |
if (LV.isGlobalReg()) |
| 1959 |
return EmitLoadOfGlobalRegLValue(LV); |
| 1960 |
|
| 1961 |
if (LV.isMatrixElt()) { |
| 1962 |
llvm::Value *Idx = LV.getMatrixIdx(); |
| 1963 |
if (CGM.getCodeGenOpts().OptimizationLevel > 0) { |
| 1964 |
const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>(); |
| 1965 |
llvm::MatrixBuilder MB(Builder); |
| 1966 |
MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened()); |
| 1967 |
} |
| 1968 |
llvm::LoadInst *Load = |
| 1969 |
Builder.CreateLoad(LV.getMatrixAddress(), LV.isVolatileQualified()); |
| 1970 |
return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext")); |
| 1971 |
} |
| 1972 |
|
| 1973 |
assert(LV.isBitField() && "Unknown LValue type!"); |
| 1974 |
return EmitLoadOfBitfieldLValue(LV, Loc); |
| 1975 |
} |
| 1976 |
|
| 1977 |
RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV, |
| 1978 |
SourceLocation Loc) { |
| 1979 |
const CGBitFieldInfo &Info = LV.getBitFieldInfo(); |
| 1980 |
|
| 1981 |
// Get the output type. |
| 1982 |
llvm::Type *ResLTy = ConvertType(LV.getType()); |
| 1983 |
|
| 1984 |
Address Ptr = LV.getBitFieldAddress(); |
| 1985 |
llvm::Value *Val = |
| 1986 |
Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load"); |
| 1987 |
|
| 1988 |
bool UseVolatile = LV.isVolatileQualified() && |
| 1989 |
Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget()); |
| 1990 |
const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset; |
| 1991 |
const unsigned StorageSize = |
| 1992 |
UseVolatile ? Info.VolatileStorageSize : Info.StorageSize; |
| 1993 |
if (Info.IsSigned) { |
| 1994 |
assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize); |
| 1995 |
unsigned HighBits = StorageSize - Offset - Info.Size; |
| 1996 |
if (HighBits) |
| 1997 |
Val = Builder.CreateShl(Val, HighBits, "bf.shl"); |
| 1998 |
if (Offset + HighBits) |
| 1999 |
Val = Builder.CreateAShr(Val, Offset + HighBits, "bf.ashr"); |
| 2000 |
} else { |
| 2001 |
if (Offset) |
| 2002 |
Val = Builder.CreateLShr(Val, Offset, "bf.lshr"); |
| 2003 |
if (static_cast<unsigned>(Offset) + Info.Size < StorageSize) |
| 2004 |
Val = Builder.CreateAnd( |
| 2005 |
Val, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), "bf.clear"); |
| 2006 |
} |
| 2007 |
Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast"); |
| 2008 |
EmitScalarRangeCheck(Val, LV.getType(), Loc); |
| 2009 |
return RValue::get(Val); |
| 2010 |
} |
| 2011 |
|
| 2012 |
// If this is a reference to a subset of the elements of a vector, create an |
| 2013 |
// appropriate shufflevector. |
| 2014 |
RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) { |
| 2015 |
llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(), |
| 2016 |
LV.isVolatileQualified()); |
| 2017 |
|
| 2018 |
const llvm::Constant *Elts = LV.getExtVectorElts(); |
| 2019 |
|
| 2020 |
// If the result of the expression is a non-vector type, we must be extracting |
| 2021 |
// a single element. Just codegen as an extractelement. |
| 2022 |
const VectorType *ExprVT = LV.getType()->getAs<VectorType>(); |
| 2023 |
if (!ExprVT) { |
| 2024 |
unsigned InIdx = getAccessedFieldNo(0, Elts); |
| 2025 |
llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx); |
| 2026 |
return RValue::get(Builder.CreateExtractElement(Vec, Elt)); |
| 2027 |
} |
| 2028 |
|
| 2029 |
// Always use shuffle vector to try to retain the original program structure |
| 2030 |
unsigned NumResultElts = ExprVT->getNumElements(); |
| 2031 |
|
| 2032 |
SmallVector<int, 4> Mask; |
| 2033 |
for (unsigned i = 0; i != NumResultElts; ++i) |
| 2034 |
Mask.push_back(getAccessedFieldNo(i, Elts)); |
| 2035 |
|
| 2036 |
Vec = Builder.CreateShuffleVector(Vec, Mask); |
| 2037 |
return RValue::get(Vec); |
| 2038 |
} |
| 2039 |
|
| 2040 |
/// Generates lvalue for partial ext_vector access. |
| 2041 |
Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) { |
| 2042 |
Address VectorAddress = LV.getExtVectorAddress(); |
| 2043 |
QualType EQT = LV.getType()->castAs<VectorType>()->getElementType(); |
| 2044 |
llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT); |
| 2045 |
|
| 2046 |
Address CastToPointerElement = |
| 2047 |
Builder.CreateElementBitCast(VectorAddress, VectorElementTy, |
| 2048 |
"conv.ptr.element"); |
| 2049 |
|
| 2050 |
const llvm::Constant *Elts = LV.getExtVectorElts(); |
| 2051 |
unsigned ix = getAccessedFieldNo(0, Elts); |
| 2052 |
|
| 2053 |
Address VectorBasePtrPlusIx = |
| 2054 |
Builder.CreateConstInBoundsGEP(CastToPointerElement, ix, |
| 2055 |
"vector.elt"); |
| 2056 |
|
| 2057 |
return VectorBasePtrPlusIx; |
| 2058 |
} |
| 2059 |
|
| 2060 |
/// Load of global gamed gegisters are always calls to intrinsics. |
| 2061 |
RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) { |
| 2062 |
assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) && |
| 2063 |
"Bad type for register variable"); |
| 2064 |
llvm::MDNode *RegName = cast<llvm::MDNode>( |
| 2065 |
cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata()); |
| 2066 |
|
| 2067 |
// We accept integer and pointer types only |
| 2068 |
llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType()); |
| 2069 |
llvm::Type *Ty = OrigTy; |
| 2070 |
if (OrigTy->isPointerTy()) |
| 2071 |
Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy); |
| 2072 |
llvm::Type *Types[] = { Ty }; |
| 2073 |
|
| 2074 |
llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types); |
| 2075 |
llvm::Value *Call = Builder.CreateCall( |
| 2076 |
F, llvm::MetadataAsValue::get(Ty->getContext(), RegName)); |
| 2077 |
if (OrigTy->isPointerTy()) |
| 2078 |
Call = Builder.CreateIntToPtr(Call, OrigTy); |
| 2079 |
return RValue::get(Call); |
| 2080 |
} |
| 2081 |
|
| 2082 |
/// EmitStoreThroughLValue - Store the specified rvalue into the specified |
| 2083 |
/// lvalue, where both are guaranteed to the have the same type, and that type |
| 2084 |
/// is 'Ty'. |
| 2085 |
void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, |
| 2086 |
bool isInit) { |
| 2087 |
if (!Dst.isSimple()) { |
| 2088 |
if (Dst.isVectorElt()) { |
| 2089 |
// Read/modify/write the vector, inserting the new element. |
| 2090 |
llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(), |
| 2091 |
Dst.isVolatileQualified()); |
| 2092 |
auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType()); |
| 2093 |
if (IRStoreTy) { |
| 2094 |
auto *IRVecTy = llvm::FixedVectorType::get( |
| 2095 |
Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits()); |
| 2096 |
Vec = Builder.CreateBitCast(Vec, IRVecTy); |
| 2097 |
// iN --> <N x i1>. |
| 2098 |
} |
| 2099 |
Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(), |
| 2100 |
Dst.getVectorIdx(), "vecins"); |
| 2101 |
if (IRStoreTy) { |
| 2102 |
// <N x i1> --> <iN>. |
| 2103 |
Vec = Builder.CreateBitCast(Vec, IRStoreTy); |
| 2104 |
} |
| 2105 |
Builder.CreateStore(Vec, Dst.getVectorAddress(), |
| 2106 |
Dst.isVolatileQualified()); |
| 2107 |
return; |
| 2108 |
} |
| 2109 |
|
| 2110 |
// If this is an update of extended vector elements, insert them as |
| 2111 |
// appropriate. |
| 2112 |
if (Dst.isExtVectorElt()) |
| 2113 |
return EmitStoreThroughExtVectorComponentLValue(Src, Dst); |
| 2114 |
|
| 2115 |
if (Dst.isGlobalReg()) |
| 2116 |
return EmitStoreThroughGlobalRegLValue(Src, Dst); |
| 2117 |
|
| 2118 |
if (Dst.isMatrixElt()) { |
| 2119 |
llvm::Value *Idx = Dst.getMatrixIdx(); |
| 2120 |
if (CGM.getCodeGenOpts().OptimizationLevel > 0) { |
| 2121 |
const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>(); |
| 2122 |
llvm::MatrixBuilder MB(Builder); |
| 2123 |
MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened()); |
| 2124 |
} |
| 2125 |
llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress()); |
| 2126 |
llvm::Value *Vec = |
| 2127 |
Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins"); |
| 2128 |
Builder.CreateStore(Vec, Dst.getMatrixAddress(), |
| 2129 |
Dst.isVolatileQualified()); |
| 2130 |
return; |
| 2131 |
} |
| 2132 |
|
| 2133 |
assert(Dst.isBitField() && "Unknown LValue type"); |
| 2134 |
return EmitStoreThroughBitfieldLValue(Src, Dst); |
| 2135 |
} |
| 2136 |
|
| 2137 |
// There's special magic for assigning into an ARC-qualified l-value. |
| 2138 |
if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) { |
| 2139 |
switch (Lifetime) { |
| 2140 |
case Qualifiers::OCL_None: |
| 2141 |
llvm_unreachable("present but none"); |
| 2142 |
|
| 2143 |
case Qualifiers::OCL_ExplicitNone: |
| 2144 |
// nothing special |
| 2145 |
break; |
| 2146 |
|
| 2147 |
case Qualifiers::OCL_Strong: |
| 2148 |
if (isInit) { |
| 2149 |
Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal())); |
| 2150 |
break; |
| 2151 |
} |
| 2152 |
EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true); |
| 2153 |
return; |
| 2154 |
|
| 2155 |
case Qualifiers::OCL_Weak: |
| 2156 |
if (isInit) |
| 2157 |
// Initialize and then skip the primitive store. |
| 2158 |
EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal()); |
| 2159 |
else |
| 2160 |
EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(), |
| 2161 |
/*ignore*/ true); |
| 2162 |
return; |
| 2163 |
|
| 2164 |
case Qualifiers::OCL_Autoreleasing: |
| 2165 |
Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(), |
| 2166 |
Src.getScalarVal())); |
| 2167 |
// fall into the normal path |
| 2168 |
break; |
| 2169 |
} |
| 2170 |
} |
| 2171 |
|
| 2172 |
if (Dst.isObjCWeak() && !Dst.isNonGC()) { |
| 2173 |
// load of a __weak object. |
| 2174 |
Address LvalueDst = Dst.getAddress(*this); |
| 2175 |
llvm::Value *src = Src.getScalarVal(); |
| 2176 |
CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst); |
| 2177 |
return; |
| 2178 |
} |
| 2179 |
|
| 2180 |
if (Dst.isObjCStrong() && !Dst.isNonGC()) { |
| 2181 |
// load of a __strong object. |
| 2182 |
Address LvalueDst = Dst.getAddress(*this); |
| 2183 |
llvm::Value *src = Src.getScalarVal(); |
| 2184 |
if (Dst.isObjCIvar()) { |
| 2185 |
assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL"); |
| 2186 |
llvm::Type *ResultType = IntPtrTy; |
| 2187 |
Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp()); |
| 2188 |
llvm::Value *RHS = dst.getPointer(); |
| 2189 |
RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast"); |
| 2190 |
llvm::Value *LHS = |
| 2191 |
Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType, |
| 2192 |
"sub.ptr.lhs.cast"); |
| 2193 |
llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset"); |
| 2194 |
CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst, |
| 2195 |
BytesBetween); |
| 2196 |
} else if (Dst.isGlobalObjCRef()) { |
| 2197 |
CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst, |
| 2198 |
Dst.isThreadLocalRef()); |
| 2199 |
} |
| 2200 |
else |
| 2201 |
CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst); |
| 2202 |
return; |
| 2203 |
} |
| 2204 |
|
| 2205 |
assert(Src.isScalar() && "Can't emit an agg store with this method"); |
| 2206 |
EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit); |
| 2207 |
} |
| 2208 |
|
| 2209 |
void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst, |
| 2210 |
llvm::Value **Result) { |
| 2211 |
const CGBitFieldInfo &Info = Dst.getBitFieldInfo(); |
| 2212 |
llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType()); |
| 2213 |
Address Ptr = Dst.getBitFieldAddress(); |
| 2214 |
|
| 2215 |
// Get the source value, truncated to the width of the bit-field. |
| 2216 |
llvm::Value *SrcVal = Src.getScalarVal(); |
| 2217 |
|
| 2218 |
// Cast the source to the storage type and shift it into place. |
| 2219 |
SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(), |
| 2220 |
/*isSigned=*/false); |
| 2221 |
llvm::Value *MaskedVal = SrcVal; |
| 2222 |
|
| 2223 |
const bool UseVolatile = |
| 2224 |
CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() && |
| 2225 |
Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget()); |
| 2226 |
const unsigned StorageSize = |
| 2227 |
UseVolatile ? Info.VolatileStorageSize : Info.StorageSize; |
| 2228 |
const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset; |
| 2229 |
// See if there are other bits in the bitfield's storage we'll need to load |
| 2230 |
// and mask together with source before storing. |
| 2231 |
if (StorageSize != Info.Size) { |
| 2232 |
assert(StorageSize > Info.Size && "Invalid bitfield size."); |
| 2233 |
llvm::Value *Val = |
| 2234 |
Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load"); |
| 2235 |
|
| 2236 |
// Mask the source value as needed. |
| 2237 |
if (!hasBooleanRepresentation(Dst.getType())) |
| 2238 |
SrcVal = Builder.CreateAnd( |
| 2239 |
SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), |
| 2240 |
"bf.value"); |
| 2241 |
MaskedVal = SrcVal; |
| 2242 |
if (Offset) |
| 2243 |
SrcVal = Builder.CreateShl(SrcVal, Offset, "bf.shl"); |
| 2244 |
|
| 2245 |
// Mask out the original value. |
| 2246 |
Val = Builder.CreateAnd( |
| 2247 |
Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.Size), |
| 2248 |
"bf.clear"); |
| 2249 |
|
| 2250 |
// Or together the unchanged values and the source value. |
| 2251 |
SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set"); |
| 2252 |
} else { |
| 2253 |
assert(Offset == 0); |
| 2254 |
// According to the AACPS: |
| 2255 |
// When a volatile bit-field is written, and its container does not overlap |
| 2256 |
// with any non-bit-field member, its container must be read exactly once |
| 2257 |
// and written exactly once using the access width appropriate to the type |
| 2258 |
// of the container. The two accesses are not atomic. |
| 2259 |
if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) && |
| 2260 |
CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad) |
| 2261 |
Builder.CreateLoad(Ptr, true, "bf.load"); |
| 2262 |
} |
| 2263 |
|
| 2264 |
// Write the new value back out. |
| 2265 |
Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified()); |
| 2266 |
|
| 2267 |
// Return the new value of the bit-field, if requested. |
| 2268 |
if (Result) { |
| 2269 |
llvm::Value *ResultVal = MaskedVal; |
| 2270 |
|
| 2271 |
// Sign extend the value if needed. |
| 2272 |
if (Info.IsSigned) { |
| 2273 |
assert(Info.Size <= StorageSize); |
| 2274 |
unsigned HighBits = StorageSize - Info.Size; |
| 2275 |
if (HighBits) { |
| 2276 |
ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl"); |
| 2277 |
ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr"); |
| 2278 |
} |
| 2279 |
} |
| 2280 |
|
| 2281 |
ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned, |
| 2282 |
"bf.result.cast"); |
| 2283 |
*Result = EmitFromMemory(ResultVal, Dst.getType()); |
| 2284 |
} |
| 2285 |
} |
| 2286 |
|
| 2287 |
void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src, |
| 2288 |
LValue Dst) { |
| 2289 |
// This access turns into a read/modify/write of the vector. Load the input |
| 2290 |
// value now. |
| 2291 |
llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(), |
| 2292 |
Dst.isVolatileQualified()); |
| 2293 |
const llvm::Constant *Elts = Dst.getExtVectorElts(); |
| 2294 |
|
| 2295 |
llvm::Value *SrcVal = Src.getScalarVal(); |
| 2296 |
|
| 2297 |
if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) { |
| 2298 |
unsigned NumSrcElts = VTy->getNumElements(); |
| 2299 |
unsigned NumDstElts = |
| 2300 |
cast<llvm::FixedVectorType>(Vec->getType())->getNumElements(); |
| 2301 |
if (NumDstElts == NumSrcElts) { |
| 2302 |
// Use shuffle vector is the src and destination are the same number of |
| 2303 |
// elements and restore the vector mask since it is on the side it will be |
| 2304 |
// stored. |
| 2305 |
SmallVector<int, 4> Mask(NumDstElts); |
| 2306 |
for (unsigned i = 0; i != NumSrcElts; ++i) |
| 2307 |
Mask[getAccessedFieldNo(i, Elts)] = i; |
| 2308 |
|
| 2309 |
Vec = Builder.CreateShuffleVector(SrcVal, Mask); |
| 2310 |
} else if (NumDstElts > NumSrcElts) { |
| 2311 |
// Extended the source vector to the same length and then shuffle it |
| 2312 |
// into the destination. |
| 2313 |
// FIXME: since we're shuffling with undef, can we just use the indices |
| 2314 |
// into that? This could be simpler. |
| 2315 |
SmallVector<int, 4> ExtMask; |
| 2316 |
for (unsigned i = 0; i != NumSrcElts; ++i) |
| 2317 |
ExtMask.push_back(i); |
| 2318 |
ExtMask.resize(NumDstElts, -1); |
| 2319 |
llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask); |
| 2320 |
// build identity |
| 2321 |
SmallVector<int, 4> Mask; |
| 2322 |
for (unsigned i = 0; i != NumDstElts; ++i) |
| 2323 |
Mask.push_back(i); |
| 2324 |
|
| 2325 |
// When the vector size is odd and .odd or .hi is used, the last element |
| 2326 |
// of the Elts constant array will be one past the size of the vector. |
| 2327 |
// Ignore the last element here, if it is greater than the mask size. |
| 2328 |
if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size()) |
| 2329 |
NumSrcElts--; |
| 2330 |
|
| 2331 |
// modify when what gets shuffled in |
| 2332 |
for (unsigned i = 0; i != NumSrcElts; ++i) |
| 2333 |
Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts; |
| 2334 |
Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask); |
| 2335 |
} else { |
| 2336 |
// We should never shorten the vector |
| 2337 |
llvm_unreachable("unexpected shorten vector length"); |
| 2338 |
} |
| 2339 |
} else { |
| 2340 |
// If the Src is a scalar (not a vector) it must be updating one element. |
| 2341 |
unsigned InIdx = getAccessedFieldNo(0, Elts); |
| 2342 |
llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx); |
| 2343 |
Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt); |
| 2344 |
} |
| 2345 |
|
| 2346 |
Builder.CreateStore(Vec, Dst.getExtVectorAddress(), |
| 2347 |
Dst.isVolatileQualified()); |
| 2348 |
} |
| 2349 |
|
| 2350 |
/// Store of global named registers are always calls to intrinsics. |
| 2351 |
void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) { |
| 2352 |
assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) && |
| 2353 |
"Bad type for register variable"); |
| 2354 |
llvm::MDNode *RegName = cast<llvm::MDNode>( |
| 2355 |
cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata()); |
| 2356 |
assert(RegName && "Register LValue is not metadata"); |
| 2357 |
|
| 2358 |
// We accept integer and pointer types only |
| 2359 |
llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType()); |
| 2360 |
llvm::Type *Ty = OrigTy; |
| 2361 |
if (OrigTy->isPointerTy()) |
| 2362 |
Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy); |
| 2363 |
llvm::Type *Types[] = { Ty }; |
| 2364 |
|
| 2365 |
llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types); |
| 2366 |
llvm::Value *Value = Src.getScalarVal(); |
| 2367 |
if (OrigTy->isPointerTy()) |
| 2368 |
Value = Builder.CreatePtrToInt(Value, Ty); |
| 2369 |
Builder.CreateCall( |
| 2370 |
F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value}); |
| 2371 |
} |
| 2372 |
|
| 2373 |
// setObjCGCLValueClass - sets class of the lvalue for the purpose of |
| 2374 |
// generating write-barries API. It is currently a global, ivar, |
| 2375 |
// or neither. |
| 2376 |
static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E, |
| 2377 |
LValue &LV, |
| 2378 |
bool IsMemberAccess=false) { |
| 2379 |
if (Ctx.getLangOpts().getGC() == LangOptions::NonGC) |
| 2380 |
return; |
| 2381 |
|
| 2382 |
if (isa<ObjCIvarRefExpr>(E)) { |
| 2383 |
QualType ExpTy = E->getType(); |
| 2384 |
if (IsMemberAccess && ExpTy->isPointerType()) { |
| 2385 |
// If ivar is a structure pointer, assigning to field of |
| 2386 |
// this struct follows gcc's behavior and makes it a non-ivar |
| 2387 |
// writer-barrier conservatively. |
| 2388 |
ExpTy = ExpTy->castAs<PointerType>()->getPointeeType(); |
| 2389 |
if (ExpTy->isRecordType()) { |
| 2390 |
LV.setObjCIvar(false); |
| 2391 |
return; |
| 2392 |
} |
| 2393 |
} |
| 2394 |
LV.setObjCIvar(true); |
| 2395 |
auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E)); |
| 2396 |
LV.setBaseIvarExp(Exp->getBase()); |
| 2397 |
LV.setObjCArray(E->getType()->isArrayType()); |
| 2398 |
return; |
| 2399 |
} |
| 2400 |
|
| 2401 |
if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) { |
| 2402 |
if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) { |
| 2403 |
if (VD->hasGlobalStorage()) { |
| 2404 |
LV.setGlobalObjCRef(true); |
| 2405 |
LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None); |
| 2406 |
} |
| 2407 |
} |
| 2408 |
LV.setObjCArray(E->getType()->isArrayType()); |
| 2409 |
return; |
| 2410 |
} |
| 2411 |
|
| 2412 |
if (const auto *Exp = dyn_cast<UnaryOperator>(E)) { |
| 2413 |
setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| 2414 |
return; |
| 2415 |
} |
| 2416 |
|
| 2417 |
if (const auto *Exp = dyn_cast<ParenExpr>(E)) { |
| 2418 |
setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| 2419 |
if (LV.isObjCIvar()) { |
| 2420 |
// If cast is to a structure pointer, follow gcc's behavior and make it |
| 2421 |
// a non-ivar write-barrier. |
| 2422 |
QualType ExpTy = E->getType(); |
| 2423 |
if (ExpTy->isPointerType()) |
| 2424 |
ExpTy = ExpTy->castAs<PointerType>()->getPointeeType(); |
| 2425 |
if (ExpTy->isRecordType()) |
| 2426 |
LV.setObjCIvar(false); |
| 2427 |
} |
| 2428 |
return; |
| 2429 |
} |
| 2430 |
|
| 2431 |
if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) { |
| 2432 |
setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV); |
| 2433 |
return; |
| 2434 |
} |
| 2435 |
|
| 2436 |
if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) { |
| 2437 |
setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| 2438 |
return; |
| 2439 |
} |
| 2440 |
|
| 2441 |
if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) { |
| 2442 |
setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| 2443 |
return; |
| 2444 |
} |
| 2445 |
|
| 2446 |
if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) { |
| 2447 |
setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess); |
| 2448 |
return; |
| 2449 |
} |
| 2450 |
|
| 2451 |
if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) { |
| 2452 |
setObjCGCLValueClass(Ctx, Exp->getBase(), LV); |
| 2453 |
if (LV.isObjCIvar() && !LV.isObjCArray()) |
| 2454 |
// Using array syntax to assigning to what an ivar points to is not |
| 2455 |
// same as assigning to the ivar itself. {id *Names;} Names[i] = 0; |
| 2456 |
LV.setObjCIvar(false); |
| 2457 |
else if (LV.isGlobalObjCRef() && !LV.isObjCArray()) |
| 2458 |
// Using array syntax to assigning to what global points to is not |
| 2459 |
// same as assigning to the global itself. {id *G;} G[i] = 0; |
| 2460 |
LV.setGlobalObjCRef(false); |
| 2461 |
return; |
| 2462 |
} |
| 2463 |
|
| 2464 |
if (const auto *Exp = dyn_cast<MemberExpr>(E)) { |
| 2465 |
setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true); |
| 2466 |
// We don't know if member is an 'ivar', but this flag is looked at |
| 2467 |
// only in the context of LV.isObjCIvar(). |
| 2468 |
LV.setObjCArray(E->getType()->isArrayType()); |
| 2469 |
return; |
| 2470 |
} |
| 2471 |
} |
| 2472 |
|
| 2473 |
static llvm::Value * |
| 2474 |
EmitBitCastOfLValueToProperType(CodeGenFunction &CGF, |
| 2475 |
llvm::Value *V, llvm::Type *IRType, |
| 2476 |
StringRef Name = StringRef()) { |
| 2477 |
unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace(); |
| 2478 |
return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name); |
| 2479 |
} |
| 2480 |
|
| 2481 |
static LValue EmitThreadPrivateVarDeclLValue( |
| 2482 |
CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr, |
| 2483 |
llvm::Type *RealVarTy, SourceLocation Loc) { |
| 2484 |
if (CGF.CGM.getLangOpts().OpenMPIRBuilder) |
| 2485 |
Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate( |
| 2486 |
CGF, VD, Addr, Loc); |
| 2487 |
else |
| 2488 |
Addr = |
| 2489 |
CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc); |
| 2490 |
|
| 2491 |
Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy); |
| 2492 |
return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); |
| 2493 |
} |
| 2494 |
|
| 2495 |
static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF, |
| 2496 |
const VarDecl *VD, QualType T) { |
| 2497 |
std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = |
| 2498 |
OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); |
| 2499 |
// Return an invalid address if variable is MT_To (or MT_Enter starting with |
| 2500 |
// OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link |
| 2501 |
// and MT_To (or MT_Enter) with unified memory, return a valid address. |
| 2502 |
if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To || |
| 2503 |
*Res == OMPDeclareTargetDeclAttr::MT_Enter) && |
| 2504 |
!CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) |
| 2505 |
return Address::invalid(); |
| 2506 |
assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) || |
| 2507 |
((*Res == OMPDeclareTargetDeclAttr::MT_To || |
| 2508 |
*Res == OMPDeclareTargetDeclAttr::MT_Enter) && |
| 2509 |
CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) && |
| 2510 |
"Expected link clause OR to clause with unified memory enabled."); |
| 2511 |
QualType PtrTy = CGF.getContext().getPointerType(VD->getType()); |
| 2512 |
Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); |
| 2513 |
return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>()); |
| 2514 |
} |
| 2515 |
|
| 2516 |
Address |
| 2517 |
CodeGenFunction::EmitLoadOfReference(LValue RefLVal, |
| 2518 |
LValueBaseInfo *PointeeBaseInfo, |
| 2519 |
TBAAAccessInfo *PointeeTBAAInfo) { |
| 2520 |
llvm::LoadInst *Load = |
| 2521 |
Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile()); |
| 2522 |
CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo()); |
| 2523 |
|
| 2524 |
QualType PointeeType = RefLVal.getType()->getPointeeType(); |
| 2525 |
CharUnits Align = CGM.getNaturalTypeAlignment( |
| 2526 |
PointeeType, PointeeBaseInfo, PointeeTBAAInfo, |
| 2527 |
/* forPointeeType= */ true); |
| 2528 |
return Address(Load, ConvertTypeForMem(PointeeType), Align); |
| 2529 |
} |
| 2530 |
|
| 2531 |
LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) { |
| 2532 |
LValueBaseInfo PointeeBaseInfo; |
| 2533 |
TBAAAccessInfo PointeeTBAAInfo; |
| 2534 |
Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo, |
| 2535 |
&PointeeTBAAInfo); |
| 2536 |
return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(), |
| 2537 |
PointeeBaseInfo, PointeeTBAAInfo); |
| 2538 |
} |
| 2539 |
|
| 2540 |
Address CodeGenFunction::EmitLoadOfPointer(Address Ptr, |
| 2541 |
const PointerType *PtrTy, |
| 2542 |
LValueBaseInfo *BaseInfo, |
| 2543 |
TBAAAccessInfo *TBAAInfo) { |
| 2544 |
llvm::Value *Addr = Builder.CreateLoad(Ptr); |
| 2545 |
return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()), |
| 2546 |
CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo, |
| 2547 |
TBAAInfo, |
| 2548 |
/*forPointeeType=*/true)); |
| 2549 |
} |
| 2550 |
|
| 2551 |
LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr, |
| 2552 |
const PointerType *PtrTy) { |
| 2553 |
LValueBaseInfo BaseInfo; |
| 2554 |
TBAAAccessInfo TBAAInfo; |
| 2555 |
Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo); |
| 2556 |
return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo); |
| 2557 |
} |
| 2558 |
|
| 2559 |
static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF, |
| 2560 |
const Expr *E, const VarDecl *VD) { |
| 2561 |
QualType T = E->getType(); |
| 2562 |
|
| 2563 |
// If it's thread_local, emit a call to its wrapper function instead. |
| 2564 |
if (VD->getTLSKind() == VarDecl::TLS_Dynamic && |
| 2565 |
CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD)) |
| 2566 |
return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T); |
| 2567 |
// Check if the variable is marked as declare target with link clause in |
| 2568 |
// device codegen. |
| 2569 |
if (CGF.getLangOpts().OpenMPIsDevice) { |
| 2570 |
Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T); |
| 2571 |
if (Addr.isValid()) |
| 2572 |
return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); |
| 2573 |
} |
| 2574 |
|
| 2575 |
llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD); |
| 2576 |
|
| 2577 |
if (VD->getTLSKind() != VarDecl::TLS_None) |
| 2578 |
V = CGF.Builder.CreateThreadLocalAddress(V); |
| 2579 |
|
| 2580 |
llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType()); |
| 2581 |
V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy); |
| 2582 |
CharUnits Alignment = CGF.getContext().getDeclAlign(VD); |
| 2583 |
Address Addr(V, RealVarTy, Alignment); |
| 2584 |
// Emit reference to the private copy of the variable if it is an OpenMP |
| 2585 |
// threadprivate variable. |
| 2586 |
if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd && |
| 2587 |
VD->hasAttr<OMPThreadPrivateDeclAttr>()) { |
| 2588 |
return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy, |
| 2589 |
E->getExprLoc()); |
| 2590 |
} |
| 2591 |
LValue LV = VD->getType()->isReferenceType() ? |
| 2592 |
CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(), |
| 2593 |
AlignmentSource::Decl) : |
| 2594 |
CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl); |
| 2595 |
setObjCGCLValueClass(CGF.getContext(), E, LV); |
| 2596 |
return LV; |
| 2597 |
} |
| 2598 |
|
| 2599 |
static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM, |
| 2600 |
GlobalDecl GD) { |
| 2601 |
const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); |
| 2602 |
if (FD->hasAttr<WeakRefAttr>()) { |
| 2603 |
ConstantAddress aliasee = CGM.GetWeakRefReference(FD); |
| 2604 |
return aliasee.getPointer(); |
| 2605 |
} |
| 2606 |
|
| 2607 |
llvm::Constant *V = CGM.GetAddrOfFunction(GD); |
| 2608 |
if (!FD->hasPrototype()) { |
| 2609 |
if (const FunctionProtoType *Proto = |
| 2610 |
FD->getType()->getAs<FunctionProtoType>()) { |
| 2611 |
// Ugly case: for a K&R-style definition, the type of the definition |
| 2612 |
// isn't the same as the type of a use. Correct for this with a |
| 2613 |
// bitcast. |
| 2614 |
QualType NoProtoType = |
| 2615 |
CGM.getContext().getFunctionNoProtoType(Proto->getReturnType()); |
| 2616 |
NoProtoType = CGM.getContext().getPointerType(NoProtoType); |
| 2617 |
V = llvm::ConstantExpr::getBitCast(V, |
| 2618 |
CGM.getTypes().ConvertType(NoProtoType)); |
| 2619 |
} |
| 2620 |
} |
| 2621 |
return V; |
| 2622 |
} |
| 2623 |
|
| 2624 |
static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E, |
| 2625 |
GlobalDecl GD) { |
| 2626 |
const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); |
| 2627 |
llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD); |
| 2628 |
CharUnits Alignment = CGF.getContext().getDeclAlign(FD); |
| 2629 |
return CGF.MakeAddrLValue(V, E->getType(), Alignment, |
| 2630 |
AlignmentSource::Decl); |
| 2631 |
} |
| 2632 |
|
| 2633 |
static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD, |
| 2634 |
llvm::Value *ThisValue) { |
| 2635 |
QualType TagType = CGF.getContext().getTagDeclType(FD->getParent()); |
| 2636 |
LValue LV = CGF.MakeNaturalAlignAddrLValue(ThisValue, TagType); |
| 2637 |
return CGF.EmitLValueForField(LV, FD); |
| 2638 |
} |
| 2639 |
|
| 2640 |
/// Named Registers are named metadata pointing to the register name |
| 2641 |
/// which will be read from/written to as an argument to the intrinsic |
| 2642 |
/// @llvm.read/write_register. |
| 2643 |
/// So far, only the name is being passed down, but other options such as |
| 2644 |
/// register type, allocation type or even optimization options could be |
| 2645 |
/// passed down via the metadata node. |
| 2646 |
static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) { |
| 2647 |
SmallString<64> Name("llvm.named.register."); |
| 2648 |
AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>(); |
| 2649 |
assert(Asm->getLabel().size() < 64-Name.size() && |
| 2650 |
"Register name too big"); |
| 2651 |
Name.append(Asm->getLabel()); |
| 2652 |
llvm::NamedMDNode *M = |
| 2653 |
CGM.getModule().getOrInsertNamedMetadata(Name); |
| 2654 |
if (M->getNumOperands() == 0) { |
| 2655 |
llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(), |
| 2656 |
Asm->getLabel()); |
| 2657 |
llvm::Metadata *Ops[] = {Str}; |
| 2658 |
M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops)); |
| 2659 |
} |
| 2660 |
|
| 2661 |
CharUnits Alignment = CGM.getContext().getDeclAlign(VD); |
| 2662 |
|
| 2663 |
llvm::Value *Ptr = |
| 2664 |
llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0)); |
| 2665 |
return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType()); |
| 2666 |
} |
| 2667 |
|
| 2668 |
/// Determine whether we can emit a reference to \p VD from the current |
| 2669 |
/// context, despite not necessarily having seen an odr-use of the variable in |
| 2670 |
/// this context. |
| 2671 |
static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF, |
| 2672 |
const DeclRefExpr *E, |
| 2673 |
const VarDecl *VD, |
| 2674 |
bool IsConstant) { |
| 2675 |
// For a variable declared in an enclosing scope, do not emit a spurious |
| 2676 |
// reference even if we have a capture, as that will emit an unwarranted |
| 2677 |
// reference to our capture state, and will likely generate worse code than |
| 2678 |
// emitting a local copy. |
| 2679 |
if (E->refersToEnclosingVariableOrCapture()) |
| 2680 |
return false; |
| 2681 |
|
| 2682 |
// For a local declaration declared in this function, we can always reference |
| 2683 |
// it even if we don't have an odr-use. |
| 2684 |
if (VD->hasLocalStorage()) { |
| 2685 |
return VD->getDeclContext() == |
| 2686 |
dyn_cast_or_null<DeclContext>(CGF.CurCodeDecl); |
| 2687 |
} |
| 2688 |
|
| 2689 |
// For a global declaration, we can emit a reference to it if we know |
| 2690 |
// for sure that we are able to emit a definition of it. |
| 2691 |
VD = VD->getDefinition(CGF.getContext()); |
| 2692 |
if (!VD) |
| 2693 |
return false; |
| 2694 |
|
| 2695 |
// Don't emit a spurious reference if it might be to a variable that only |
| 2696 |
// exists on a different device / target. |
| 2697 |
// FIXME: This is unnecessarily broad. Check whether this would actually be a |
| 2698 |
// cross-target reference. |
| 2699 |
if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA || |
| 2700 |
CGF.getLangOpts().OpenCL) { |
| 2701 |
return false; |
| 2702 |
} |
| 2703 |
|
| 2704 |
// We can emit a spurious reference only if the linkage implies that we'll |
| 2705 |
// be emitting a non-interposable symbol that will be retained until link |
| 2706 |
// time. |
| 2707 |
switch (CGF.CGM.getLLVMLinkageVarDefinition(VD, IsConstant)) { |
| 2708 |
case llvm::GlobalValue::ExternalLinkage: |
| 2709 |
case llvm::GlobalValue::LinkOnceODRLinkage: |
| 2710 |
case llvm::GlobalValue::WeakODRLinkage: |
| 2711 |
case llvm::GlobalValue::InternalLinkage: |
| 2712 |
case llvm::GlobalValue::PrivateLinkage: |
| 2713 |
return true; |
| 2714 |
default: |
| 2715 |
return false; |
| 2716 |
} |
| 2717 |
} |
| 2718 |
|
| 2719 |
LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) { |
| 2720 |
const NamedDecl *ND = E->getDecl(); |
| 2721 |
QualType T = E->getType(); |
| 2722 |
|
| 2723 |
assert(E->isNonOdrUse() != NOUR_Unevaluated && |
| 2724 |
"should not emit an unevaluated operand"); |
| 2725 |
|
| 2726 |
if (const auto *VD = dyn_cast<VarDecl>(ND)) { |
| 2727 |
// Global Named registers access via intrinsics only |
| 2728 |
if (VD->getStorageClass() == SC_Register && |
| 2729 |
VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl()) |
| 2730 |
return EmitGlobalNamedRegister(VD, CGM); |
| 2731 |
|
| 2732 |
// If this DeclRefExpr does not constitute an odr-use of the variable, |
| 2733 |
// we're not permitted to emit a reference to it in general, and it might |
| 2734 |
// not be captured if capture would be necessary for a use. Emit the |
| 2735 |
// constant value directly instead. |
| 2736 |
if (E->isNonOdrUse() == NOUR_Constant && |
| 2737 |
(VD->getType()->isReferenceType() || |
| 2738 |
!canEmitSpuriousReferenceToVariable(*this, E, VD, true))) { |
| 2739 |
VD->getAnyInitializer(VD); |
| 2740 |
llvm::Constant *Val = ConstantEmitter(*this).emitAbstract( |
| 2741 |
E->getLocation(), *VD->evaluateValue(), VD->getType()); |
| 2742 |
assert(Val && "failed to emit constant expression"); |
| 2743 |
|
| 2744 |
Address Addr = Address::invalid(); |
| 2745 |
if (!VD->getType()->isReferenceType()) { |
| 2746 |
// Spill the constant value to a global. |
| 2747 |
Addr = CGM.createUnnamedGlobalFrom(*VD, Val, |
| 2748 |
getContext().getDeclAlign(VD)); |
| 2749 |
llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType()); |
| 2750 |
auto *PTy = llvm::PointerType::get( |
| 2751 |
VarTy, getTypes().getTargetAddressSpace(VD->getType())); |
| 2752 |
Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy); |
| 2753 |
} else { |
| 2754 |
// Should we be using the alignment of the constant pointer we emitted? |
| 2755 |
CharUnits Alignment = |
| 2756 |
CGM.getNaturalTypeAlignment(E->getType(), |
| 2757 |
/* BaseInfo= */ nullptr, |
| 2758 |
/* TBAAInfo= */ nullptr, |
| 2759 |
/* forPointeeType= */ true); |
| 2760 |
Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment); |
| 2761 |
} |
| 2762 |
return MakeAddrLValue(Addr, T, AlignmentSource::Decl); |
| 2763 |
} |
| 2764 |
|
| 2765 |
// FIXME: Handle other kinds of non-odr-use DeclRefExprs. |
| 2766 |
|
| 2767 |
// Check for captured variables. |
| 2768 |
if (E->refersToEnclosingVariableOrCapture()) { |
| 2769 |
VD = VD->getCanonicalDecl(); |
| 2770 |
if (auto *FD = LambdaCaptureFields.lookup(VD)) |
| 2771 |
return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue); |
| 2772 |
if (CapturedStmtInfo) { |
| 2773 |
auto I = LocalDeclMap.find(VD); |
| 2774 |
if (I != LocalDeclMap.end()) { |
| 2775 |
LValue CapLVal; |
| 2776 |
if (VD->getType()->isReferenceType()) |
| 2777 |
CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(), |
| 2778 |
AlignmentSource::Decl); |
| 2779 |
else |
| 2780 |
CapLVal = MakeAddrLValue(I->second, T); |
| 2781 |
// Mark lvalue as nontemporal if the variable is marked as nontemporal |
| 2782 |
// in simd context. |
| 2783 |
if (getLangOpts().OpenMP && |
| 2784 |
CGM.getOpenMPRuntime().isNontemporalDecl(VD)) |
| 2785 |
CapLVal.setNontemporal(/*Value=*/true); |
| 2786 |
return CapLVal; |
| 2787 |
} |
| 2788 |
LValue CapLVal = |
| 2789 |
EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD), |
| 2790 |
CapturedStmtInfo->getContextValue()); |
| 2791 |
Address LValueAddress = CapLVal.getAddress(*this); |
| 2792 |
CapLVal = MakeAddrLValue( |
| 2793 |
Address(LValueAddress.getPointer(), LValueAddress.getElementType(), |
| 2794 |
getContext().getDeclAlign(VD)), |
| 2795 |
CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl), |
| 2796 |
CapLVal.getTBAAInfo()); |
| 2797 |
// Mark lvalue as nontemporal if the variable is marked as nontemporal |
| 2798 |
// in simd context. |
| 2799 |
if (getLangOpts().OpenMP && |
| 2800 |
CGM.getOpenMPRuntime().isNontemporalDecl(VD)) |
| 2801 |
CapLVal.setNontemporal(/*Value=*/true); |
| 2802 |
return CapLVal; |
| 2803 |
} |
| 2804 |
|
| 2805 |
assert(isa<BlockDecl>(CurCodeDecl)); |
| 2806 |
Address addr = GetAddrOfBlockDecl(VD); |
| 2807 |
return MakeAddrLValue(addr, T, AlignmentSource::Decl); |
| 2808 |
} |
| 2809 |
} |
| 2810 |
|
| 2811 |
// FIXME: We should be able to assert this for FunctionDecls as well! |
| 2812 |
// FIXME: We should be able to assert this for all DeclRefExprs, not just |
| 2813 |
// those with a valid source location. |
| 2814 |
assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() || |
| 2815 |
!E->getLocation().isValid()) && |
| 2816 |
"Should not use decl without marking it used!"); |
| 2817 |
|
| 2818 |
if (ND->hasAttr<WeakRefAttr>()) { |
| 2819 |
const auto *VD = cast<ValueDecl>(ND); |
| 2820 |
ConstantAddress Aliasee = CGM.GetWeakRefReference(VD); |
| 2821 |
return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl); |
| 2822 |
} |
| 2823 |
|
| 2824 |
if (const auto *VD = dyn_cast<VarDecl>(ND)) { |
| 2825 |
// Check if this is a global variable. |
| 2826 |
if (VD->hasLinkage() || VD->isStaticDataMember()) |
| 2827 |
return EmitGlobalVarDeclLValue(*this, E, VD); |
| 2828 |
|
| 2829 |
Address addr = Address::invalid(); |
| 2830 |
|
| 2831 |
// The variable should generally be present in the local decl map. |
| 2832 |
auto iter = LocalDeclMap.find(VD); |
| 2833 |
if (iter != LocalDeclMap.end()) { |
| 2834 |
addr = iter->second; |
| 2835 |
|
| 2836 |
// Otherwise, it might be static local we haven't emitted yet for |
| 2837 |
// some reason; most likely, because it's in an outer function. |
| 2838 |
} else if (VD->isStaticLocal()) { |
| 2839 |
llvm::Constant *var = CGM.getOrCreateStaticVarDecl( |
| 2840 |
*VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false)); |
| 2841 |
addr = Address( |
| 2842 |
var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD)); |
| 2843 |
|
| 2844 |
// No other cases for now. |
| 2845 |
} else { |
| 2846 |
llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?"); |
| 2847 |
} |
| 2848 |
|
| 2849 |
// Handle threadlocal function locals. |
| 2850 |
if (VD->getTLSKind() != VarDecl::TLS_None) |
| 2851 |
addr = |
| 2852 |
addr.withPointer(Builder.CreateThreadLocalAddress(addr.getPointer())); |
| 2853 |
|
| 2854 |
// Check for OpenMP threadprivate variables. |
| 2855 |
if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd && |
| 2856 |
VD->hasAttr<OMPThreadPrivateDeclAttr>()) { |
| 2857 |
return EmitThreadPrivateVarDeclLValue( |
| 2858 |
*this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()), |
| 2859 |
E->getExprLoc()); |
| 2860 |
} |
| 2861 |
|
| 2862 |
// Drill into block byref variables. |
| 2863 |
bool isBlockByref = VD->isEscapingByref(); |
| 2864 |
if (isBlockByref) { |
| 2865 |
addr = emitBlockByrefAddress(addr, VD); |
| 2866 |
} |
| 2867 |
|
| 2868 |
// Drill into reference types. |
| 2869 |
LValue LV = VD->getType()->isReferenceType() ? |
| 2870 |
EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) : |
| 2871 |
MakeAddrLValue(addr, T, AlignmentSource::Decl); |
| 2872 |
|
| 2873 |
bool isLocalStorage = VD->hasLocalStorage(); |
| 2874 |
|
| 2875 |
bool NonGCable = isLocalStorage && |
| 2876 |
!VD->getType()->isReferenceType() && |
| 2877 |
!isBlockByref; |
| 2878 |
if (NonGCable) { |
| 2879 |
LV.getQuals().removeObjCGCAttr(); |
| 2880 |
LV.setNonGC(true); |
| 2881 |
} |
| 2882 |
|
| 2883 |
bool isImpreciseLifetime = |
| 2884 |
(isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>()); |
| 2885 |
if (isImpreciseLifetime) |
| 2886 |
LV.setARCPreciseLifetime(ARCImpreciseLifetime); |
| 2887 |
setObjCGCLValueClass(getContext(), E, LV); |
| 2888 |
return LV; |
| 2889 |
} |
| 2890 |
|
| 2891 |
if (const auto *FD = dyn_cast<FunctionDecl>(ND)) { |
| 2892 |
LValue LV = EmitFunctionDeclLValue(*this, E, FD); |
| 2893 |
|
| 2894 |
// Emit debuginfo for the function declaration if the target wants to. |
| 2895 |
if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) { |
| 2896 |
if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) { |
| 2897 |
auto *Fn = |
| 2898 |
cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts()); |
| 2899 |
if (!Fn->getSubprogram()) |
| 2900 |
DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn); |
| 2901 |
} |
| 2902 |
} |
| 2903 |
|
| 2904 |
return LV; |
| 2905 |
} |
| 2906 |
|
| 2907 |
// FIXME: While we're emitting a binding from an enclosing scope, all other |
| 2908 |
// DeclRefExprs we see should be implicitly treated as if they also refer to |
| 2909 |
// an enclosing scope. |
| 2910 |
if (const auto *BD = dyn_cast<BindingDecl>(ND)) { |
| 2911 |
if (E->refersToEnclosingVariableOrCapture()) { |
| 2912 |
auto *FD = LambdaCaptureFields.lookup(BD); |
| 2913 |
return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue); |
| 2914 |
} |
| 2915 |
return EmitLValue(BD->getBinding()); |
| 2916 |
} |
| 2917 |
|
| 2918 |
// We can form DeclRefExprs naming GUID declarations when reconstituting |
| 2919 |
// non-type template parameters into expressions. |
| 2920 |
if (const auto *GD = dyn_cast<MSGuidDecl>(ND)) |
| 2921 |
return MakeAddrLValue(CGM.GetAddrOfMSGuidDecl(GD), T, |
| 2922 |
AlignmentSource::Decl); |
| 2923 |
|
| 2924 |
if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND)) |
| 2925 |
return MakeAddrLValue(CGM.GetAddrOfTemplateParamObject(TPO), T, |
| 2926 |
AlignmentSource::Decl); |
| 2927 |
|
| 2928 |
llvm_unreachable("Unhandled DeclRefExpr"); |
| 2929 |
} |
| 2930 |
|
| 2931 |
LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) { |
| 2932 |
// __extension__ doesn't affect lvalue-ness. |
| 2933 |
if (E->getOpcode() == UO_Extension) |
| 2934 |
return EmitLValue(E->getSubExpr()); |
| 2935 |
|
| 2936 |
QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType()); |
| 2937 |
switch (E->getOpcode()) { |
| 2938 |
default: llvm_unreachable("Unknown unary operator lvalue!"); |
| 2939 |
case UO_Deref: { |
| 2940 |
QualType T = E->getSubExpr()->getType()->getPointeeType(); |
| 2941 |
assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type"); |
| 2942 |
|
| 2943 |
LValueBaseInfo BaseInfo; |
| 2944 |
TBAAAccessInfo TBAAInfo; |
| 2945 |
Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &BaseInfo, |
| 2946 |
&TBAAInfo); |
| 2947 |
LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo); |
| 2948 |
LV.getQuals().setAddressSpace(ExprTy.getAddressSpace()); |
| 2949 |
|
| 2950 |
// We should not generate __weak write barrier on indirect reference |
| 2951 |
// of a pointer to object; as in void foo (__weak id *param); *param = 0; |
| 2952 |
// But, we continue to generate __strong write barrier on indirect write |
| 2953 |
// into a pointer to object. |
| 2954 |
if (getLangOpts().ObjC && |
| 2955 |
getLangOpts().getGC() != LangOptions::NonGC && |
| 2956 |
LV.isObjCWeak()) |
| 2957 |
LV.setNonGC(!E->isOBJCGCCandidate(getContext())); |
| 2958 |
return LV; |
| 2959 |
} |
| 2960 |
case UO_Real: |
| 2961 |
case UO_Imag: { |
| 2962 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 2963 |
assert(LV.isSimple() && "real/imag on non-ordinary l-value"); |
| 2964 |
|
| 2965 |
// __real is valid on scalars. This is a faster way of testing that. |
| 2966 |
// __imag can only produce an rvalue on scalars. |
| 2967 |
if (E->getOpcode() == UO_Real && |
| 2968 |
!LV.getAddress(*this).getElementType()->isStructTy()) { |
| 2969 |
assert(E->getSubExpr()->getType()->isArithmeticType()); |
| 2970 |
return LV; |
| 2971 |
} |
| 2972 |
|
| 2973 |
QualType T = ExprTy->castAs<ComplexType>()->getElementType(); |
| 2974 |
|
| 2975 |
Address Component = |
| 2976 |
(E->getOpcode() == UO_Real |
| 2977 |
? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType()) |
| 2978 |
: emitAddrOfImagComponent(LV.getAddress(*this), LV.getType())); |
| 2979 |
LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(), |
| 2980 |
CGM.getTBAAInfoForSubobject(LV, T)); |
| 2981 |
ElemLV.getQuals().addQualifiers(LV.getQuals()); |
| 2982 |
return ElemLV; |
| 2983 |
} |
| 2984 |
case UO_PreInc: |
| 2985 |
case UO_PreDec: { |
| 2986 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 2987 |
bool isInc = E->getOpcode() == UO_PreInc; |
| 2988 |
|
| 2989 |
if (E->getType()->isAnyComplexType()) |
| 2990 |
EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/); |
| 2991 |
else |
| 2992 |
EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/); |
| 2993 |
return LV; |
| 2994 |
} |
| 2995 |
} |
| 2996 |
} |
| 2997 |
|
| 2998 |
LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) { |
| 2999 |
return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E), |
| 3000 |
E->getType(), AlignmentSource::Decl); |
| 3001 |
} |
| 3002 |
|
| 3003 |
LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) { |
| 3004 |
return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E), |
| 3005 |
E->getType(), AlignmentSource::Decl); |
| 3006 |
} |
| 3007 |
|
| 3008 |
LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) { |
| 3009 |
auto SL = E->getFunctionName(); |
| 3010 |
assert(SL != nullptr && "No StringLiteral name in PredefinedExpr"); |
| 3011 |
StringRef FnName = CurFn->getName(); |
| 3012 |
if (FnName.startswith("\01")) |
| 3013 |
FnName = FnName.substr(1); |
| 3014 |
StringRef NameItems[] = { |
| 3015 |
PredefinedExpr::getIdentKindName(E->getIdentKind()), FnName}; |
| 3016 |
std::string GVName = llvm::join(NameItems, NameItems + 2, "."); |
| 3017 |
if (auto *BD = dyn_cast_or_null<BlockDecl>(CurCodeDecl)) { |
| 3018 |
std::string Name = std::string(SL->getString()); |
| 3019 |
if (!Name.empty()) { |
| 3020 |
unsigned Discriminator = |
| 3021 |
CGM.getCXXABI().getMangleContext().getBlockId(BD, true); |
| 3022 |
if (Discriminator) |
| 3023 |
Name += "_" + Twine(Discriminator + 1).str(); |
| 3024 |
auto C = CGM.GetAddrOfConstantCString(Name, GVName.c_str()); |
| 3025 |
return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); |
| 3026 |
} else { |
| 3027 |
auto C = |
| 3028 |
CGM.GetAddrOfConstantCString(std::string(FnName), GVName.c_str()); |
| 3029 |
return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); |
| 3030 |
} |
| 3031 |
} |
| 3032 |
auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName); |
| 3033 |
return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl); |
| 3034 |
} |
| 3035 |
|
| 3036 |
/// Emit a type description suitable for use by a runtime sanitizer library. The |
| 3037 |
/// format of a type descriptor is |
| 3038 |
/// |
| 3039 |
/// \code |
| 3040 |
/// { i16 TypeKind, i16 TypeInfo } |
| 3041 |
/// \endcode |
| 3042 |
/// |
| 3043 |
/// followed by an array of i8 containing the type name. TypeKind is 0 for an |
| 3044 |
/// integer, 1 for a floating point value, and -1 for anything else. |
| 3045 |
llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) { |
| 3046 |
// Only emit each type's descriptor once. |
| 3047 |
if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T)) |
| 3048 |
return C; |
| 3049 |
|
| 3050 |
uint16_t TypeKind = -1; |
| 3051 |
uint16_t TypeInfo = 0; |
| 3052 |
|
| 3053 |
if (T->isIntegerType()) { |
| 3054 |
TypeKind = 0; |
| 3055 |
TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) | |
| 3056 |
(T->isSignedIntegerType() ? 1 : 0); |
| 3057 |
} else if (T->isFloatingType()) { |
| 3058 |
TypeKind = 1; |
| 3059 |
TypeInfo = getContext().getTypeSize(T); |
| 3060 |
} |
| 3061 |
|
| 3062 |
// Format the type name as if for a diagnostic, including quotes and |
| 3063 |
// optionally an 'aka'. |
| 3064 |
SmallString<32> Buffer; |
| 3065 |
CGM.getDiags().ConvertArgToString( |
| 3066 |
DiagnosticsEngine::ak_qualtype, (intptr_t)T.getAsOpaquePtr(), StringRef(), |
| 3067 |
StringRef(), std::nullopt, Buffer, std::nullopt); |
| 3068 |
|
| 3069 |
llvm::Constant *Components[] = { |
| 3070 |
Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo), |
| 3071 |
llvm::ConstantDataArray::getString(getLLVMContext(), Buffer) |
| 3072 |
}; |
| 3073 |
llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components); |
| 3074 |
|
| 3075 |
auto *GV = new llvm::GlobalVariable( |
| 3076 |
CGM.getModule(), Descriptor->getType(), |
| 3077 |
/*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor); |
| 3078 |
GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); |
| 3079 |
CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV); |
| 3080 |
|
| 3081 |
// Remember the descriptor for this type. |
| 3082 |
CGM.setTypeDescriptorInMap(T, GV); |
| 3083 |
|
| 3084 |
return GV; |
| 3085 |
} |
| 3086 |
|
| 3087 |
llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) { |
| 3088 |
llvm::Type *TargetTy = IntPtrTy; |
| 3089 |
|
| 3090 |
if (V->getType() == TargetTy) |
| 3091 |
return V; |
| 3092 |
|
| 3093 |
// Floating-point types which fit into intptr_t are bitcast to integers |
| 3094 |
// and then passed directly (after zero-extension, if necessary). |
| 3095 |
if (V->getType()->isFloatingPointTy()) { |
| 3096 |
unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue(); |
| 3097 |
if (Bits <= TargetTy->getIntegerBitWidth()) |
| 3098 |
V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(), |
| 3099 |
Bits)); |
| 3100 |
} |
| 3101 |
|
| 3102 |
// Integers which fit in intptr_t are zero-extended and passed directly. |
| 3103 |
if (V->getType()->isIntegerTy() && |
| 3104 |
V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth()) |
| 3105 |
return Builder.CreateZExt(V, TargetTy); |
| 3106 |
|
| 3107 |
// Pointers are passed directly, everything else is passed by address. |
| 3108 |
if (!V->getType()->isPointerTy()) { |
| 3109 |
Address Ptr = CreateDefaultAlignTempAlloca(V->getType()); |
| 3110 |
Builder.CreateStore(V, Ptr); |
| 3111 |
V = Ptr.getPointer(); |
| 3112 |
} |
| 3113 |
return Builder.CreatePtrToInt(V, TargetTy); |
| 3114 |
} |
| 3115 |
|
| 3116 |
/// Emit a representation of a SourceLocation for passing to a handler |
| 3117 |
/// in a sanitizer runtime library. The format for this data is: |
| 3118 |
/// \code |
| 3119 |
/// struct SourceLocation { |
| 3120 |
/// const char *Filename; |
| 3121 |
/// int32_t Line, Column; |
| 3122 |
/// }; |
| 3123 |
/// \endcode |
| 3124 |
/// For an invalid SourceLocation, the Filename pointer is null. |
| 3125 |
llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) { |
| 3126 |
llvm::Constant *Filename; |
| 3127 |
int Line, Column; |
| 3128 |
|
| 3129 |
PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc); |
| 3130 |
if (PLoc.isValid()) { |
| 3131 |
StringRef FilenameString = PLoc.getFilename(); |
| 3132 |
|
| 3133 |
int PathComponentsToStrip = |
| 3134 |
CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip; |
| 3135 |
if (PathComponentsToStrip < 0) { |
| 3136 |
assert(PathComponentsToStrip != INT_MIN); |
| 3137 |
int PathComponentsToKeep = -PathComponentsToStrip; |
| 3138 |
auto I = llvm::sys::path::rbegin(FilenameString); |
| 3139 |
auto E = llvm::sys::path::rend(FilenameString); |
| 3140 |
while (I != E && --PathComponentsToKeep) |
| 3141 |
++I; |
| 3142 |
|
| 3143 |
FilenameString = FilenameString.substr(I - E); |
| 3144 |
} else if (PathComponentsToStrip > 0) { |
| 3145 |
auto I = llvm::sys::path::begin(FilenameString); |
| 3146 |
auto E = llvm::sys::path::end(FilenameString); |
| 3147 |
while (I != E && PathComponentsToStrip--) |
| 3148 |
++I; |
| 3149 |
|
| 3150 |
if (I != E) |
| 3151 |
FilenameString = |
| 3152 |
FilenameString.substr(I - llvm::sys::path::begin(FilenameString)); |
| 3153 |
else |
| 3154 |
FilenameString = llvm::sys::path::filename(FilenameString); |
| 3155 |
} |
| 3156 |
|
| 3157 |
auto FilenameGV = |
| 3158 |
CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src"); |
| 3159 |
CGM.getSanitizerMetadata()->disableSanitizerForGlobal( |
| 3160 |
cast<llvm::GlobalVariable>( |
| 3161 |
FilenameGV.getPointer()->stripPointerCasts())); |
| 3162 |
Filename = FilenameGV.getPointer(); |
| 3163 |
Line = PLoc.getLine(); |
| 3164 |
Column = PLoc.getColumn(); |
| 3165 |
} else { |
| 3166 |
Filename = llvm::Constant::getNullValue(Int8PtrTy); |
| 3167 |
Line = Column = 0; |
| 3168 |
} |
| 3169 |
|
| 3170 |
llvm::Constant *Data[] = {Filename, Builder.getInt32(Line), |
| 3171 |
Builder.getInt32(Column)}; |
| 3172 |
|
| 3173 |
return llvm::ConstantStruct::getAnon(Data); |
| 3174 |
} |
| 3175 |
|
| 3176 |
namespace { |
| 3177 |
/// Specify under what conditions this check can be recovered |
| 3178 |
enum class CheckRecoverableKind { |
| 3179 |
/// Always terminate program execution if this check fails. |
| 3180 |
Unrecoverable, |
| 3181 |
/// Check supports recovering, runtime has both fatal (noreturn) and |
| 3182 |
/// non-fatal handlers for this check. |
| 3183 |
Recoverable, |
| 3184 |
/// Runtime conditionally aborts, always need to support recovery. |
| 3185 |
AlwaysRecoverable |
| 3186 |
}; |
| 3187 |
} |
| 3188 |
|
| 3189 |
static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) { |
| 3190 |
assert(Kind.countPopulation() == 1); |
| 3191 |
if (Kind == SanitizerKind::Function || Kind == SanitizerKind::Vptr) |
| 3192 |
return CheckRecoverableKind::AlwaysRecoverable; |
| 3193 |
else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable) |
| 3194 |
return CheckRecoverableKind::Unrecoverable; |
| 3195 |
else |
| 3196 |
return CheckRecoverableKind::Recoverable; |
| 3197 |
} |
| 3198 |
|
| 3199 |
namespace { |
| 3200 |
struct SanitizerHandlerInfo { |
| 3201 |
char const *const Name; |
| 3202 |
unsigned Version; |
| 3203 |
}; |
| 3204 |
} |
| 3205 |
|
| 3206 |
const SanitizerHandlerInfo SanitizerHandlers[] = { |
| 3207 |
#define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version}, |
| 3208 |
LIST_SANITIZER_CHECKS |
| 3209 |
#undef SANITIZER_CHECK |
| 3210 |
}; |
| 3211 |
|
| 3212 |
static void emitCheckHandlerCall(CodeGenFunction &CGF, |
| 3213 |
llvm::FunctionType *FnType, |
| 3214 |
ArrayRef<llvm::Value *> FnArgs, |
| 3215 |
SanitizerHandler CheckHandler, |
| 3216 |
CheckRecoverableKind RecoverKind, bool IsFatal, |
| 3217 |
llvm::BasicBlock *ContBB) { |
| 3218 |
assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable); |
| 3219 |
std::optional<ApplyDebugLocation> DL; |
| 3220 |
if (!CGF.Builder.getCurrentDebugLocation()) { |
| 3221 |
// Ensure that the call has at least an artificial debug location. |
| 3222 |
DL.emplace(CGF, SourceLocation()); |
| 3223 |
} |
| 3224 |
bool NeedsAbortSuffix = |
| 3225 |
IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable; |
| 3226 |
bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime; |
| 3227 |
const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler]; |
| 3228 |
const StringRef CheckName = CheckInfo.Name; |
| 3229 |
std::string FnName = "__ubsan_handle_" + CheckName.str(); |
| 3230 |
if (CheckInfo.Version && !MinimalRuntime) |
| 3231 |
FnName += "_v" + llvm::utostr(CheckInfo.Version); |
| 3232 |
if (MinimalRuntime) |
| 3233 |
FnName += "_minimal"; |
| 3234 |
if (NeedsAbortSuffix) |
| 3235 |
FnName += "_abort"; |
| 3236 |
bool MayReturn = |
| 3237 |
!IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable; |
| 3238 |
|
| 3239 |
llvm::AttrBuilder B(CGF.getLLVMContext()); |
| 3240 |
if (!MayReturn) { |
| 3241 |
B.addAttribute(llvm::Attribute::NoReturn) |
| 3242 |
.addAttribute(llvm::Attribute::NoUnwind); |
| 3243 |
} |
| 3244 |
B.addUWTableAttr(llvm::UWTableKind::Default); |
| 3245 |
|
| 3246 |
llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction( |
| 3247 |
FnType, FnName, |
| 3248 |
llvm::AttributeList::get(CGF.getLLVMContext(), |
| 3249 |
llvm::AttributeList::FunctionIndex, B), |
| 3250 |
/*Local=*/true); |
| 3251 |
llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs); |
| 3252 |
if (!MayReturn) { |
| 3253 |
HandlerCall->setDoesNotReturn(); |
| 3254 |
CGF.Builder.CreateUnreachable(); |
| 3255 |
} else { |
| 3256 |
CGF.Builder.CreateBr(ContBB); |
| 3257 |
} |
| 3258 |
} |
| 3259 |
|
| 3260 |
void CodeGenFunction::EmitCheck( |
| 3261 |
ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked, |
| 3262 |
SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs, |
| 3263 |
ArrayRef<llvm::Value *> DynamicArgs) { |
| 3264 |
assert(IsSanitizerScope); |
| 3265 |
assert(Checked.size() > 0); |
| 3266 |
assert(CheckHandler >= 0 && |
| 3267 |
size_t(CheckHandler) < std::size(SanitizerHandlers)); |
| 3268 |
const StringRef CheckName = SanitizerHandlers[CheckHandler].Name; |
| 3269 |
|
| 3270 |
llvm::Value *FatalCond = nullptr; |
| 3271 |
llvm::Value *RecoverableCond = nullptr; |
| 3272 |
llvm::Value *TrapCond = nullptr; |
| 3273 |
for (int i = 0, n = Checked.size(); i < n; ++i) { |
| 3274 |
llvm::Value *Check = Checked[i].first; |
| 3275 |
// -fsanitize-trap= overrides -fsanitize-recover=. |
| 3276 |
llvm::Value *&Cond = |
| 3277 |
CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second) |
| 3278 |
? TrapCond |
| 3279 |
: CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second) |
| 3280 |
? RecoverableCond |
| 3281 |
: FatalCond; |
| 3282 |
Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check; |
| 3283 |
} |
| 3284 |
|
| 3285 |
if (TrapCond) |
| 3286 |
EmitTrapCheck(TrapCond, CheckHandler); |
| 3287 |
if (!FatalCond && !RecoverableCond) |
| 3288 |
return; |
| 3289 |
|
| 3290 |
llvm::Value *JointCond; |
| 3291 |
if (FatalCond && RecoverableCond) |
| 3292 |
JointCond = Builder.CreateAnd(FatalCond, RecoverableCond); |
| 3293 |
else |
| 3294 |
JointCond = FatalCond ? FatalCond : RecoverableCond; |
| 3295 |
assert(JointCond); |
| 3296 |
|
| 3297 |
CheckRecoverableKind RecoverKind = getRecoverableKind(Checked[0].second); |
| 3298 |
assert(SanOpts.has(Checked[0].second)); |
| 3299 |
#ifndef NDEBUG |
| 3300 |
for (int i = 1, n = Checked.size(); i < n; ++i) { |
| 3301 |
assert(RecoverKind == getRecoverableKind(Checked[i].second) && |
| 3302 |
"All recoverable kinds in a single check must be same!"); |
| 3303 |
assert(SanOpts.has(Checked[i].second)); |
| 3304 |
} |
| 3305 |
#endif |
| 3306 |
|
| 3307 |
llvm::BasicBlock *Cont = createBasicBlock("cont"); |
| 3308 |
llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName); |
| 3309 |
llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers); |
| 3310 |
// Give hint that we very much don't expect to execute the handler |
| 3311 |
// Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp |
| 3312 |
llvm::MDBuilder MDHelper(getLLVMContext()); |
| 3313 |
llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1); |
| 3314 |
Branch->setMetadata(llvm::LLVMContext::MD_prof, Node); |
| 3315 |
EmitBlock(Handlers); |
| 3316 |
|
| 3317 |
// Handler functions take an i8* pointing to the (handler-specific) static |
| 3318 |
// information block, followed by a sequence of intptr_t arguments |
| 3319 |
// representing operand values. |
| 3320 |
SmallVector<llvm::Value *, 4> Args; |
| 3321 |
SmallVector<llvm::Type *, 4> ArgTypes; |
| 3322 |
if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) { |
| 3323 |
Args.reserve(DynamicArgs.size() + 1); |
| 3324 |
ArgTypes.reserve(DynamicArgs.size() + 1); |
| 3325 |
|
| 3326 |
// Emit handler arguments and create handler function type. |
| 3327 |
if (!StaticArgs.empty()) { |
| 3328 |
llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs); |
| 3329 |
auto *InfoPtr = new llvm::GlobalVariable( |
| 3330 |
CGM.getModule(), Info->getType(), false, |
| 3331 |
llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr, |
| 3332 |
llvm::GlobalVariable::NotThreadLocal, |
| 3333 |
CGM.getDataLayout().getDefaultGlobalsAddressSpace()); |
| 3334 |
InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); |
| 3335 |
CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr); |
| 3336 |
Args.push_back(EmitCastToVoidPtr(InfoPtr)); |
| 3337 |
ArgTypes.push_back(Args.back()->getType()); |
| 3338 |
} |
| 3339 |
|
| 3340 |
for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) { |
| 3341 |
Args.push_back(EmitCheckValue(DynamicArgs[i])); |
| 3342 |
ArgTypes.push_back(IntPtrTy); |
| 3343 |
} |
| 3344 |
} |
| 3345 |
|
| 3346 |
llvm::FunctionType *FnType = |
| 3347 |
llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false); |
| 3348 |
|
| 3349 |
if (!FatalCond || !RecoverableCond) { |
| 3350 |
// Simple case: we need to generate a single handler call, either |
| 3351 |
// fatal, or non-fatal. |
| 3352 |
emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, |
| 3353 |
(FatalCond != nullptr), Cont); |
| 3354 |
} else { |
| 3355 |
// Emit two handler calls: first one for set of unrecoverable checks, |
| 3356 |
// another one for recoverable. |
| 3357 |
llvm::BasicBlock *NonFatalHandlerBB = |
| 3358 |
createBasicBlock("non_fatal." + CheckName); |
| 3359 |
llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName); |
| 3360 |
Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB); |
| 3361 |
EmitBlock(FatalHandlerBB); |
| 3362 |
emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true, |
| 3363 |
NonFatalHandlerBB); |
| 3364 |
EmitBlock(NonFatalHandlerBB); |
| 3365 |
emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false, |
| 3366 |
Cont); |
| 3367 |
} |
| 3368 |
|
| 3369 |
EmitBlock(Cont); |
| 3370 |
} |
| 3371 |
|
| 3372 |
void CodeGenFunction::EmitCfiSlowPathCheck( |
| 3373 |
SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId, |
| 3374 |
llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) { |
| 3375 |
llvm::BasicBlock *Cont = createBasicBlock("cfi.cont"); |
| 3376 |
|
| 3377 |
llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath"); |
| 3378 |
llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB); |
| 3379 |
|
| 3380 |
llvm::MDBuilder MDHelper(getLLVMContext()); |
| 3381 |
llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1); |
| 3382 |
BI->setMetadata(llvm::LLVMContext::MD_prof, Node); |
| 3383 |
|
| 3384 |
EmitBlock(CheckBB); |
| 3385 |
|
| 3386 |
bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Kind); |
| 3387 |
|
| 3388 |
llvm::CallInst *CheckCall; |
| 3389 |
llvm::FunctionCallee SlowPathFn; |
| 3390 |
if (WithDiag) { |
| 3391 |
llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs); |
| 3392 |
auto *InfoPtr = |
| 3393 |
new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false, |
| 3394 |
llvm::GlobalVariable::PrivateLinkage, Info); |
| 3395 |
InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); |
| 3396 |
CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr); |
| 3397 |
|
| 3398 |
SlowPathFn = CGM.getModule().getOrInsertFunction( |
| 3399 |
"__cfi_slowpath_diag", |
| 3400 |
llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, |
| 3401 |
false)); |
| 3402 |
CheckCall = Builder.CreateCall( |
| 3403 |
SlowPathFn, {TypeId, Ptr, Builder.CreateBitCast(InfoPtr, Int8PtrTy)}); |
| 3404 |
} else { |
| 3405 |
SlowPathFn = CGM.getModule().getOrInsertFunction( |
| 3406 |
"__cfi_slowpath", |
| 3407 |
llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false)); |
| 3408 |
CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr}); |
| 3409 |
} |
| 3410 |
|
| 3411 |
CGM.setDSOLocal( |
| 3412 |
cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts())); |
| 3413 |
CheckCall->setDoesNotThrow(); |
| 3414 |
|
| 3415 |
EmitBlock(Cont); |
| 3416 |
} |
| 3417 |
|
| 3418 |
// Emit a stub for __cfi_check function so that the linker knows about this |
| 3419 |
// symbol in LTO mode. |
| 3420 |
void CodeGenFunction::EmitCfiCheckStub() { |
| 3421 |
llvm::Module *M = &CGM.getModule(); |
| 3422 |
auto &Ctx = M->getContext(); |
| 3423 |
llvm::Function *F = llvm::Function::Create( |
| 3424 |
llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false), |
| 3425 |
llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M); |
| 3426 |
CGM.setDSOLocal(F); |
| 3427 |
llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F); |
| 3428 |
// FIXME: consider emitting an intrinsic call like |
| 3429 |
// call void @llvm.cfi_check(i64 %0, i8* %1, i8* %2) |
| 3430 |
// which can be lowered in CrossDSOCFI pass to the actual contents of |
| 3431 |
// __cfi_check. This would allow inlining of __cfi_check calls. |
| 3432 |
llvm::CallInst::Create( |
| 3433 |
llvm::Intrinsic::getDeclaration(M, llvm::Intrinsic::trap), "", BB); |
| 3434 |
llvm::ReturnInst::Create(Ctx, nullptr, BB); |
| 3435 |
} |
| 3436 |
|
| 3437 |
// This function is basically a switch over the CFI failure kind, which is |
| 3438 |
// extracted from CFICheckFailData (1st function argument). Each case is either |
| 3439 |
// llvm.trap or a call to one of the two runtime handlers, based on |
| 3440 |
// -fsanitize-trap and -fsanitize-recover settings. Default case (invalid |
| 3441 |
// failure kind) traps, but this should really never happen. CFICheckFailData |
| 3442 |
// can be nullptr if the calling module has -fsanitize-trap behavior for this |
| 3443 |
// check kind; in this case __cfi_check_fail traps as well. |
| 3444 |
void CodeGenFunction::EmitCfiCheckFail() { |
| 3445 |
SanitizerScope SanScope(this); |
| 3446 |
FunctionArgList Args; |
| 3447 |
ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy, |
| 3448 |
ImplicitParamDecl::Other); |
| 3449 |
ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy, |
| 3450 |
ImplicitParamDecl::Other); |
| 3451 |
Args.push_back(&ArgData); |
| 3452 |
Args.push_back(&ArgAddr); |
| 3453 |
|
| 3454 |
const CGFunctionInfo &FI = |
| 3455 |
CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, Args); |
| 3456 |
|
| 3457 |
llvm::Function *F = llvm::Function::Create( |
| 3458 |
llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false), |
| 3459 |
llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule()); |
| 3460 |
|
| 3461 |
CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false); |
| 3462 |
CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F); |
| 3463 |
F->setVisibility(llvm::GlobalValue::HiddenVisibility); |
| 3464 |
|
| 3465 |
StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args, |
| 3466 |
SourceLocation()); |
| 3467 |
|
| 3468 |
// This function is not affected by NoSanitizeList. This function does |
| 3469 |
// not have a source location, but "src:*" would still apply. Revert any |
| 3470 |
// changes to SanOpts made in StartFunction. |
| 3471 |
SanOpts = CGM.getLangOpts().Sanitize; |
| 3472 |
|
| 3473 |
llvm::Value *Data = |
| 3474 |
EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false, |
| 3475 |
CGM.getContext().VoidPtrTy, ArgData.getLocation()); |
| 3476 |
llvm::Value *Addr = |
| 3477 |
EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false, |
| 3478 |
CGM.getContext().VoidPtrTy, ArgAddr.getLocation()); |
| 3479 |
|
| 3480 |
// Data == nullptr means the calling module has trap behaviour for this check. |
| 3481 |
llvm::Value *DataIsNotNullPtr = |
| 3482 |
Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy)); |
| 3483 |
EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail); |
| 3484 |
|
| 3485 |
llvm::StructType *SourceLocationTy = |
| 3486 |
llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty); |
| 3487 |
llvm::StructType *CfiCheckFailDataTy = |
| 3488 |
llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy); |
| 3489 |
|
| 3490 |
llvm::Value *V = Builder.CreateConstGEP2_32( |
| 3491 |
CfiCheckFailDataTy, |
| 3492 |
Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0, |
| 3493 |
0); |
| 3494 |
|
| 3495 |
Address CheckKindAddr(V, Int8Ty, getIntAlign()); |
| 3496 |
llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr); |
| 3497 |
|
| 3498 |
llvm::Value *AllVtables = llvm::MetadataAsValue::get( |
| 3499 |
CGM.getLLVMContext(), |
| 3500 |
llvm::MDString::get(CGM.getLLVMContext(), "all-vtables")); |
| 3501 |
llvm::Value *ValidVtable = Builder.CreateZExt( |
| 3502 |
Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test), |
| 3503 |
{Addr, AllVtables}), |
| 3504 |
IntPtrTy); |
| 3505 |
|
| 3506 |
const std::pair<int, SanitizerMask> CheckKinds[] = { |
| 3507 |
{CFITCK_VCall, SanitizerKind::CFIVCall}, |
| 3508 |
{CFITCK_NVCall, SanitizerKind::CFINVCall}, |
| 3509 |
{CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast}, |
| 3510 |
{CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast}, |
| 3511 |
{CFITCK_ICall, SanitizerKind::CFIICall}}; |
| 3512 |
|
| 3513 |
SmallVector<std::pair<llvm::Value *, SanitizerMask>, 5> Checks; |
| 3514 |
for (auto CheckKindMaskPair : CheckKinds) { |
| 3515 |
int Kind = CheckKindMaskPair.first; |
| 3516 |
SanitizerMask Mask = CheckKindMaskPair.second; |
| 3517 |
llvm::Value *Cond = |
| 3518 |
Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind)); |
| 3519 |
if (CGM.getLangOpts().Sanitize.has(Mask)) |
| 3520 |
EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {}, |
| 3521 |
{Data, Addr, ValidVtable}); |
| 3522 |
else |
| 3523 |
EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail); |
| 3524 |
} |
| 3525 |
|
| 3526 |
FinishFunction(); |
| 3527 |
// The only reference to this function will be created during LTO link. |
| 3528 |
// Make sure it survives until then. |
| 3529 |
CGM.addUsedGlobal(F); |
| 3530 |
} |
| 3531 |
|
| 3532 |
void CodeGenFunction::EmitUnreachable(SourceLocation Loc) { |
| 3533 |
if (SanOpts.has(SanitizerKind::Unreachable)) { |
| 3534 |
SanitizerScope SanScope(this); |
| 3535 |
EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()), |
| 3536 |
SanitizerKind::Unreachable), |
| 3537 |
SanitizerHandler::BuiltinUnreachable, |
| 3538 |
EmitCheckSourceLocation(Loc), std::nullopt); |
| 3539 |
} |
| 3540 |
Builder.CreateUnreachable(); |
| 3541 |
} |
| 3542 |
|
| 3543 |
void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked, |
| 3544 |
SanitizerHandler CheckHandlerID) { |
| 3545 |
llvm::BasicBlock *Cont = createBasicBlock("cont"); |
| 3546 |
|
| 3547 |
// If we're optimizing, collapse all calls to trap down to just one per |
| 3548 |
// check-type per function to save on code size. |
| 3549 |
if (TrapBBs.size() <= CheckHandlerID) |
| 3550 |
TrapBBs.resize(CheckHandlerID + 1); |
| 3551 |
llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID]; |
| 3552 |
|
| 3553 |
if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB || |
| 3554 |
(CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>())) { |
| 3555 |
TrapBB = createBasicBlock("trap"); |
| 3556 |
Builder.CreateCondBr(Checked, Cont, TrapBB); |
| 3557 |
EmitBlock(TrapBB); |
| 3558 |
|
| 3559 |
llvm::CallInst *TrapCall = |
| 3560 |
Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::ubsantrap), |
| 3561 |
llvm::ConstantInt::get(CGM.Int8Ty, CheckHandlerID)); |
| 3562 |
|
| 3563 |
if (!CGM.getCodeGenOpts().TrapFuncName.empty()) { |
| 3564 |
auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name", |
| 3565 |
CGM.getCodeGenOpts().TrapFuncName); |
| 3566 |
TrapCall->addFnAttr(A); |
| 3567 |
} |
| 3568 |
TrapCall->setDoesNotReturn(); |
| 3569 |
TrapCall->setDoesNotThrow(); |
| 3570 |
Builder.CreateUnreachable(); |
| 3571 |
} else { |
| 3572 |
auto Call = TrapBB->begin(); |
| 3573 |
assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB"); |
| 3574 |
|
| 3575 |
Call->applyMergedLocation(Call->getDebugLoc(), |
| 3576 |
Builder.getCurrentDebugLocation()); |
| 3577 |
Builder.CreateCondBr(Checked, Cont, TrapBB); |
| 3578 |
} |
| 3579 |
|
| 3580 |
EmitBlock(Cont); |
| 3581 |
} |
| 3582 |
|
| 3583 |
llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) { |
| 3584 |
llvm::CallInst *TrapCall = |
| 3585 |
Builder.CreateCall(CGM.getIntrinsic(IntrID)); |
| 3586 |
|
| 3587 |
if (!CGM.getCodeGenOpts().TrapFuncName.empty()) { |
| 3588 |
auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name", |
| 3589 |
CGM.getCodeGenOpts().TrapFuncName); |
| 3590 |
TrapCall->addFnAttr(A); |
| 3591 |
} |
| 3592 |
|
| 3593 |
return TrapCall; |
| 3594 |
} |
| 3595 |
|
| 3596 |
Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E, |
| 3597 |
LValueBaseInfo *BaseInfo, |
| 3598 |
TBAAAccessInfo *TBAAInfo) { |
| 3599 |
assert(E->getType()->isArrayType() && |
| 3600 |
"Array to pointer decay must have array source type!"); |
| 3601 |
|
| 3602 |
// Expressions of array type can't be bitfields or vector elements. |
| 3603 |
LValue LV = EmitLValue(E); |
| 3604 |
Address Addr = LV.getAddress(*this); |
| 3605 |
|
| 3606 |
// If the array type was an incomplete type, we need to make sure |
| 3607 |
// the decay ends up being the right type. |
| 3608 |
llvm::Type *NewTy = ConvertType(E->getType()); |
| 3609 |
Addr = Builder.CreateElementBitCast(Addr, NewTy); |
| 3610 |
|
| 3611 |
// Note that VLA pointers are always decayed, so we don't need to do |
| 3612 |
// anything here. |
| 3613 |
if (!E->getType()->isVariableArrayType()) { |
| 3614 |
assert(isa<llvm::ArrayType>(Addr.getElementType()) && |
| 3615 |
"Expected pointer to array"); |
| 3616 |
Addr = Builder.CreateConstArrayGEP(Addr, 0, "arraydecay"); |
| 3617 |
} |
| 3618 |
|
| 3619 |
// The result of this decay conversion points to an array element within the |
| 3620 |
// base lvalue. However, since TBAA currently does not support representing |
| 3621 |
// accesses to elements of member arrays, we conservatively represent accesses |
| 3622 |
// to the pointee object as if it had no any base lvalue specified. |
| 3623 |
// TODO: Support TBAA for member arrays. |
| 3624 |
QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType(); |
| 3625 |
if (BaseInfo) *BaseInfo = LV.getBaseInfo(); |
| 3626 |
if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType); |
| 3627 |
|
| 3628 |
return Builder.CreateElementBitCast(Addr, ConvertTypeForMem(EltType)); |
| 3629 |
} |
| 3630 |
|
| 3631 |
/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an |
| 3632 |
/// array to pointer, return the array subexpression. |
| 3633 |
static const Expr *isSimpleArrayDecayOperand(const Expr *E) { |
| 3634 |
// If this isn't just an array->pointer decay, bail out. |
| 3635 |
const auto *CE = dyn_cast<CastExpr>(E); |
| 3636 |
if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay) |
| 3637 |
return nullptr; |
| 3638 |
|
| 3639 |
// If this is a decay from variable width array, bail out. |
| 3640 |
const Expr *SubExpr = CE->getSubExpr(); |
| 3641 |
if (SubExpr->getType()->isVariableArrayType()) |
| 3642 |
return nullptr; |
| 3643 |
|
| 3644 |
return SubExpr; |
| 3645 |
} |
| 3646 |
|
| 3647 |
static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF, |
| 3648 |
llvm::Type *elemType, |
| 3649 |
llvm::Value *ptr, |
| 3650 |
ArrayRef<llvm::Value*> indices, |
| 3651 |
bool inbounds, |
| 3652 |
bool signedIndices, |
| 3653 |
SourceLocation loc, |
| 3654 |
const llvm::Twine &name = "arrayidx") { |
| 3655 |
if (inbounds) { |
| 3656 |
return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices, |
| 3657 |
CodeGenFunction::NotSubtraction, loc, |
| 3658 |
name); |
| 3659 |
} else { |
| 3660 |
return CGF.Builder.CreateGEP(elemType, ptr, indices, name); |
| 3661 |
} |
| 3662 |
} |
| 3663 |
|
| 3664 |
static CharUnits getArrayElementAlign(CharUnits arrayAlign, |
| 3665 |
llvm::Value *idx, |
| 3666 |
CharUnits eltSize) { |
| 3667 |
// If we have a constant index, we can use the exact offset of the |
| 3668 |
// element we're accessing. |
| 3669 |
if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) { |
| 3670 |
CharUnits offset = constantIdx->getZExtValue() * eltSize; |
| 3671 |
return arrayAlign.alignmentAtOffset(offset); |
| 3672 |
|
| 3673 |
// Otherwise, use the worst-case alignment for any element. |
| 3674 |
} else { |
| 3675 |
return arrayAlign.alignmentOfArrayElement(eltSize); |
| 3676 |
} |
| 3677 |
} |
| 3678 |
|
| 3679 |
static QualType getFixedSizeElementType(const ASTContext &ctx, |
| 3680 |
const VariableArrayType *vla) { |
| 3681 |
QualType eltType; |
| 3682 |
do { |
| 3683 |
eltType = vla->getElementType(); |
| 3684 |
} while ((vla = ctx.getAsVariableArrayType(eltType))); |
| 3685 |
return eltType; |
| 3686 |
} |
| 3687 |
|
| 3688 |
/// Given an array base, check whether its member access belongs to a record |
| 3689 |
/// with preserve_access_index attribute or not. |
| 3690 |
static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) { |
| 3691 |
if (!ArrayBase || !CGF.getDebugInfo()) |
| 3692 |
return false; |
| 3693 |
|
| 3694 |
// Only support base as either a MemberExpr or DeclRefExpr. |
| 3695 |
// DeclRefExpr to cover cases like: |
| 3696 |
// struct s { int a; int b[10]; }; |
| 3697 |
// struct s *p; |
| 3698 |
// p[1].a |
| 3699 |
// p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr. |
| 3700 |
// p->b[5] is a MemberExpr example. |
| 3701 |
const Expr *E = ArrayBase->IgnoreImpCasts(); |
| 3702 |
if (const auto *ME = dyn_cast<MemberExpr>(E)) |
| 3703 |
return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>(); |
| 3704 |
|
| 3705 |
if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) { |
| 3706 |
const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl()); |
| 3707 |
if (!VarDef) |
| 3708 |
return false; |
| 3709 |
|
| 3710 |
const auto *PtrT = VarDef->getType()->getAs<PointerType>(); |
| 3711 |
if (!PtrT) |
| 3712 |
return false; |
| 3713 |
|
| 3714 |
const auto *PointeeT = PtrT->getPointeeType() |
| 3715 |
->getUnqualifiedDesugaredType(); |
| 3716 |
if (const auto *RecT = dyn_cast<RecordType>(PointeeT)) |
| 3717 |
return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>(); |
| 3718 |
return false; |
| 3719 |
} |
| 3720 |
|
| 3721 |
return false; |
| 3722 |
} |
| 3723 |
|
| 3724 |
static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr, |
| 3725 |
ArrayRef<llvm::Value *> indices, |
| 3726 |
QualType eltType, bool inbounds, |
| 3727 |
bool signedIndices, SourceLocation loc, |
| 3728 |
QualType *arrayType = nullptr, |
| 3729 |
const Expr *Base = nullptr, |
| 3730 |
const llvm::Twine &name = "arrayidx") { |
| 3731 |
// All the indices except that last must be zero. |
| 3732 |
#ifndef NDEBUG |
| 3733 |
for (auto *idx : indices.drop_back()) |
| 3734 |
assert(isa<llvm::ConstantInt>(idx) && |
| 3735 |
cast<llvm::ConstantInt>(idx)->isZero()); |
| 3736 |
#endif |
| 3737 |
|
| 3738 |
// Determine the element size of the statically-sized base. This is |
| 3739 |
// the thing that the indices are expressed in terms of. |
| 3740 |
if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) { |
| 3741 |
eltType = getFixedSizeElementType(CGF.getContext(), vla); |
| 3742 |
} |
| 3743 |
|
| 3744 |
// We can use that to compute the best alignment of the element. |
| 3745 |
CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType); |
| 3746 |
CharUnits eltAlign = |
| 3747 |
getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); |
| 3748 |
|
| 3749 |
llvm::Value *eltPtr; |
| 3750 |
auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back()); |
| 3751 |
if (!LastIndex || |
| 3752 |
(!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) { |
| 3753 |
eltPtr = emitArraySubscriptGEP( |
| 3754 |
CGF, addr.getElementType(), addr.getPointer(), indices, inbounds, |
| 3755 |
signedIndices, loc, name); |
| 3756 |
} else { |
| 3757 |
// Remember the original array subscript for bpf target |
| 3758 |
unsigned idx = LastIndex->getZExtValue(); |
| 3759 |
llvm::DIType *DbgInfo = nullptr; |
| 3760 |
if (arrayType) |
| 3761 |
DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc); |
| 3762 |
eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(), |
| 3763 |
addr.getPointer(), |
| 3764 |
indices.size() - 1, |
| 3765 |
idx, DbgInfo); |
| 3766 |
} |
| 3767 |
|
| 3768 |
return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign); |
| 3769 |
} |
| 3770 |
|
| 3771 |
LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E, |
| 3772 |
bool Accessed) { |
| 3773 |
// The index must always be an integer, which is not an aggregate. Emit it |
| 3774 |
// in lexical order (this complexity is, sadly, required by C++17). |
| 3775 |
llvm::Value *IdxPre = |
| 3776 |
(E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr; |
| 3777 |
bool SignedIndices = false; |
| 3778 |
auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * { |
| 3779 |
auto *Idx = IdxPre; |
| 3780 |
if (E->getLHS() != E->getIdx()) { |
| 3781 |
assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS"); |
| 3782 |
Idx = EmitScalarExpr(E->getIdx()); |
| 3783 |
} |
| 3784 |
|
| 3785 |
QualType IdxTy = E->getIdx()->getType(); |
| 3786 |
bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType(); |
| 3787 |
SignedIndices |= IdxSigned; |
| 3788 |
|
| 3789 |
if (SanOpts.has(SanitizerKind::ArrayBounds)) |
| 3790 |
EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed); |
| 3791 |
|
| 3792 |
// Extend or truncate the index type to 32 or 64-bits. |
| 3793 |
if (Promote && Idx->getType() != IntPtrTy) |
| 3794 |
Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom"); |
| 3795 |
|
| 3796 |
return Idx; |
| 3797 |
}; |
| 3798 |
IdxPre = nullptr; |
| 3799 |
|
| 3800 |
// If the base is a vector type, then we are forming a vector element lvalue |
| 3801 |
// with this subscript. |
| 3802 |
if (E->getBase()->getType()->isVectorType() && |
| 3803 |
!isa<ExtVectorElementExpr>(E->getBase())) { |
| 3804 |
// Emit the vector as an lvalue to get its address. |
| 3805 |
LValue LHS = EmitLValue(E->getBase()); |
| 3806 |
auto *Idx = EmitIdxAfterBase(/*Promote*/false); |
| 3807 |
assert(LHS.isSimple() && "Can only subscript lvalue vectors here!"); |
| 3808 |
return LValue::MakeVectorElt(LHS.getAddress(*this), Idx, |
| 3809 |
E->getBase()->getType(), LHS.getBaseInfo(), |
| 3810 |
TBAAAccessInfo()); |
| 3811 |
} |
| 3812 |
|
| 3813 |
// All the other cases basically behave like simple offsetting. |
| 3814 |
|
| 3815 |
// Handle the extvector case we ignored above. |
| 3816 |
if (isa<ExtVectorElementExpr>(E->getBase())) { |
| 3817 |
LValue LV = EmitLValue(E->getBase()); |
| 3818 |
auto *Idx = EmitIdxAfterBase(/*Promote*/true); |
| 3819 |
Address Addr = EmitExtVectorElementLValue(LV); |
| 3820 |
|
| 3821 |
QualType EltType = LV.getType()->castAs<VectorType>()->getElementType(); |
| 3822 |
Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true, |
| 3823 |
SignedIndices, E->getExprLoc()); |
| 3824 |
return MakeAddrLValue(Addr, EltType, LV.getBaseInfo(), |
| 3825 |
CGM.getTBAAInfoForSubobject(LV, EltType)); |
| 3826 |
} |
| 3827 |
|
| 3828 |
LValueBaseInfo EltBaseInfo; |
| 3829 |
TBAAAccessInfo EltTBAAInfo; |
| 3830 |
Address Addr = Address::invalid(); |
| 3831 |
if (const VariableArrayType *vla = |
| 3832 |
getContext().getAsVariableArrayType(E->getType())) { |
| 3833 |
// The base must be a pointer, which is not an aggregate. Emit |
| 3834 |
// it. It needs to be emitted first in case it's what captures |
| 3835 |
// the VLA bounds. |
| 3836 |
Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo); |
| 3837 |
auto *Idx = EmitIdxAfterBase(/*Promote*/true); |
| 3838 |
|
| 3839 |
// The element count here is the total number of non-VLA elements. |
| 3840 |
llvm::Value *numElements = getVLASize(vla).NumElts; |
| 3841 |
|
| 3842 |
// Effectively, the multiply by the VLA size is part of the GEP. |
| 3843 |
// GEP indexes are signed, and scaling an index isn't permitted to |
| 3844 |
// signed-overflow, so we use the same semantics for our explicit |
| 3845 |
// multiply. We suppress this if overflow is not undefined behavior. |
| 3846 |
if (getLangOpts().isSignedOverflowDefined()) { |
| 3847 |
Idx = Builder.CreateMul(Idx, numElements); |
| 3848 |
} else { |
| 3849 |
Idx = Builder.CreateNSWMul(Idx, numElements); |
| 3850 |
} |
| 3851 |
|
| 3852 |
Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(), |
| 3853 |
!getLangOpts().isSignedOverflowDefined(), |
| 3854 |
SignedIndices, E->getExprLoc()); |
| 3855 |
|
| 3856 |
} else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){ |
| 3857 |
// Indexing over an interface, as in "NSString *P; P[4];" |
| 3858 |
|
| 3859 |
// Emit the base pointer. |
| 3860 |
Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo); |
| 3861 |
auto *Idx = EmitIdxAfterBase(/*Promote*/true); |
| 3862 |
|
| 3863 |
CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT); |
| 3864 |
llvm::Value *InterfaceSizeVal = |
| 3865 |
llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity()); |
| 3866 |
|
| 3867 |
llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal); |
| 3868 |
|
| 3869 |
// We don't necessarily build correct LLVM struct types for ObjC |
| 3870 |
// interfaces, so we can't rely on GEP to do this scaling |
| 3871 |
// correctly, so we need to cast to i8*. FIXME: is this actually |
| 3872 |
// true? A lot of other things in the fragile ABI would break... |
| 3873 |
llvm::Type *OrigBaseElemTy = Addr.getElementType(); |
| 3874 |
Addr = Builder.CreateElementBitCast(Addr, Int8Ty); |
| 3875 |
|
| 3876 |
// Do the GEP. |
| 3877 |
CharUnits EltAlign = |
| 3878 |
getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize); |
| 3879 |
llvm::Value *EltPtr = |
| 3880 |
emitArraySubscriptGEP(*this, Addr.getElementType(), Addr.getPointer(), |
| 3881 |
ScaledIdx, false, SignedIndices, E->getExprLoc()); |
| 3882 |
Addr = Address(EltPtr, Addr.getElementType(), EltAlign); |
| 3883 |
|
| 3884 |
// Cast back. |
| 3885 |
Addr = Builder.CreateElementBitCast(Addr, OrigBaseElemTy); |
| 3886 |
} else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) { |
| 3887 |
// If this is A[i] where A is an array, the frontend will have decayed the |
| 3888 |
// base to be a ArrayToPointerDecay implicit cast. While correct, it is |
| 3889 |
// inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a |
| 3890 |
// "gep x, i" here. Emit one "gep A, 0, i". |
| 3891 |
assert(Array->getType()->isArrayType() && |
| 3892 |
"Array to pointer decay must have array source type!"); |
| 3893 |
LValue ArrayLV; |
| 3894 |
// For simple multidimensional array indexing, set the 'accessed' flag for |
| 3895 |
// better bounds-checking of the base expression. |
| 3896 |
if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array)) |
| 3897 |
ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true); |
| 3898 |
else |
| 3899 |
ArrayLV = EmitLValue(Array); |
| 3900 |
auto *Idx = EmitIdxAfterBase(/*Promote*/true); |
| 3901 |
|
| 3902 |
// Propagate the alignment from the array itself to the result. |
| 3903 |
QualType arrayType = Array->getType(); |
| 3904 |
Addr = emitArraySubscriptGEP( |
| 3905 |
*this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx}, |
| 3906 |
E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices, |
| 3907 |
E->getExprLoc(), &arrayType, E->getBase()); |
| 3908 |
EltBaseInfo = ArrayLV.getBaseInfo(); |
| 3909 |
EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType()); |
| 3910 |
} else { |
| 3911 |
// The base must be a pointer; emit it with an estimate of its alignment. |
| 3912 |
Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo); |
| 3913 |
auto *Idx = EmitIdxAfterBase(/*Promote*/true); |
| 3914 |
QualType ptrType = E->getBase()->getType(); |
| 3915 |
Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(), |
| 3916 |
!getLangOpts().isSignedOverflowDefined(), |
| 3917 |
SignedIndices, E->getExprLoc(), &ptrType, |
| 3918 |
E->getBase()); |
| 3919 |
} |
| 3920 |
|
| 3921 |
LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo); |
| 3922 |
|
| 3923 |
if (getLangOpts().ObjC && |
| 3924 |
getLangOpts().getGC() != LangOptions::NonGC) { |
| 3925 |
LV.setNonGC(!E->isOBJCGCCandidate(getContext())); |
| 3926 |
setObjCGCLValueClass(getContext(), E, LV); |
| 3927 |
} |
| 3928 |
return LV; |
| 3929 |
} |
| 3930 |
|
| 3931 |
LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) { |
| 3932 |
assert( |
| 3933 |
!E->isIncomplete() && |
| 3934 |
"incomplete matrix subscript expressions should be rejected during Sema"); |
| 3935 |
LValue Base = EmitLValue(E->getBase()); |
| 3936 |
llvm::Value *RowIdx = EmitScalarExpr(E->getRowIdx()); |
| 3937 |
llvm::Value *ColIdx = EmitScalarExpr(E->getColumnIdx()); |
| 3938 |
llvm::Value *NumRows = Builder.getIntN( |
| 3939 |
RowIdx->getType()->getScalarSizeInBits(), |
| 3940 |
E->getBase()->getType()->castAs<ConstantMatrixType>()->getNumRows()); |
| 3941 |
llvm::Value *FinalIdx = |
| 3942 |
Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx); |
| 3943 |
return LValue::MakeMatrixElt( |
| 3944 |
MaybeConvertMatrixAddress(Base.getAddress(*this), *this), FinalIdx, |
| 3945 |
E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo()); |
| 3946 |
} |
| 3947 |
|
| 3948 |
static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base, |
| 3949 |
LValueBaseInfo &BaseInfo, |
| 3950 |
TBAAAccessInfo &TBAAInfo, |
| 3951 |
QualType BaseTy, QualType ElTy, |
| 3952 |
bool IsLowerBound) { |
| 3953 |
LValue BaseLVal; |
| 3954 |
if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) { |
| 3955 |
BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound); |
| 3956 |
if (BaseTy->isArrayType()) { |
| 3957 |
Address Addr = BaseLVal.getAddress(CGF); |
| 3958 |
BaseInfo = BaseLVal.getBaseInfo(); |
| 3959 |
|
| 3960 |
// If the array type was an incomplete type, we need to make sure |
| 3961 |
// the decay ends up being the right type. |
| 3962 |
llvm::Type *NewTy = CGF.ConvertType(BaseTy); |
| 3963 |
Addr = CGF.Builder.CreateElementBitCast(Addr, NewTy); |
| 3964 |
|
| 3965 |
// Note that VLA pointers are always decayed, so we don't need to do |
| 3966 |
// anything here. |
| 3967 |
if (!BaseTy->isVariableArrayType()) { |
| 3968 |
assert(isa<llvm::ArrayType>(Addr.getElementType()) && |
| 3969 |
"Expected pointer to array"); |
| 3970 |
Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay"); |
| 3971 |
} |
| 3972 |
|
| 3973 |
return CGF.Builder.CreateElementBitCast(Addr, |
| 3974 |
CGF.ConvertTypeForMem(ElTy)); |
| 3975 |
} |
| 3976 |
LValueBaseInfo TypeBaseInfo; |
| 3977 |
TBAAAccessInfo TypeTBAAInfo; |
| 3978 |
CharUnits Align = |
| 3979 |
CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo); |
| 3980 |
BaseInfo.mergeForCast(TypeBaseInfo); |
| 3981 |
TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo); |
| 3982 |
return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)), |
| 3983 |
CGF.ConvertTypeForMem(ElTy), Align); |
| 3984 |
} |
| 3985 |
return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo); |
| 3986 |
} |
| 3987 |
|
| 3988 |
LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, |
| 3989 |
bool IsLowerBound) { |
| 3990 |
QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(E->getBase()); |
| 3991 |
QualType ResultExprTy; |
| 3992 |
if (auto *AT = getContext().getAsArrayType(BaseTy)) |
| 3993 |
ResultExprTy = AT->getElementType(); |
| 3994 |
else |
| 3995 |
ResultExprTy = BaseTy->getPointeeType(); |
| 3996 |
llvm::Value *Idx = nullptr; |
| 3997 |
if (IsLowerBound || E->getColonLocFirst().isInvalid()) { |
| 3998 |
// Requesting lower bound or upper bound, but without provided length and |
| 3999 |
// without ':' symbol for the default length -> length = 1. |
| 4000 |
// Idx = LowerBound ?: 0; |
| 4001 |
if (auto *LowerBound = E->getLowerBound()) { |
| 4002 |
Idx = Builder.CreateIntCast( |
| 4003 |
EmitScalarExpr(LowerBound), IntPtrTy, |
| 4004 |
LowerBound->getType()->hasSignedIntegerRepresentation()); |
| 4005 |
} else |
| 4006 |
Idx = llvm::ConstantInt::getNullValue(IntPtrTy); |
| 4007 |
} else { |
| 4008 |
// Try to emit length or lower bound as constant. If this is possible, 1 |
| 4009 |
// is subtracted from constant length or lower bound. Otherwise, emit LLVM |
| 4010 |
// IR (LB + Len) - 1. |
| 4011 |
auto &C = CGM.getContext(); |
| 4012 |
auto *Length = E->getLength(); |
| 4013 |
llvm::APSInt ConstLength; |
| 4014 |
if (Length) { |
| 4015 |
// Idx = LowerBound + Length - 1; |
| 4016 |
if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) { |
| 4017 |
ConstLength = CL->zextOrTrunc(PointerWidthInBits); |
| 4018 |
Length = nullptr; |
| 4019 |
} |
| 4020 |
auto *LowerBound = E->getLowerBound(); |
| 4021 |
llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false); |
| 4022 |
if (LowerBound) { |
| 4023 |
if (std::optional<llvm::APSInt> LB = |
| 4024 |
LowerBound->getIntegerConstantExpr(C)) { |
| 4025 |
ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits); |
| 4026 |
LowerBound = nullptr; |
| 4027 |
} |
| 4028 |
} |
| 4029 |
if (!Length) |
| 4030 |
--ConstLength; |
| 4031 |
else if (!LowerBound) |
| 4032 |
--ConstLowerBound; |
| 4033 |
|
| 4034 |
if (Length || LowerBound) { |
| 4035 |
auto *LowerBoundVal = |
| 4036 |
LowerBound |
| 4037 |
? Builder.CreateIntCast( |
| 4038 |
EmitScalarExpr(LowerBound), IntPtrTy, |
| 4039 |
LowerBound->getType()->hasSignedIntegerRepresentation()) |
| 4040 |
: llvm::ConstantInt::get(IntPtrTy, ConstLowerBound); |
| 4041 |
auto *LengthVal = |
| 4042 |
Length |
| 4043 |
? Builder.CreateIntCast( |
| 4044 |
EmitScalarExpr(Length), IntPtrTy, |
| 4045 |
Length->getType()->hasSignedIntegerRepresentation()) |
| 4046 |
: llvm::ConstantInt::get(IntPtrTy, ConstLength); |
| 4047 |
Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len", |
| 4048 |
/*HasNUW=*/false, |
| 4049 |
!getLangOpts().isSignedOverflowDefined()); |
| 4050 |
if (Length && LowerBound) { |
| 4051 |
Idx = Builder.CreateSub( |
| 4052 |
Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1", |
| 4053 |
/*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined()); |
| 4054 |
} |
| 4055 |
} else |
| 4056 |
Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound); |
| 4057 |
} else { |
| 4058 |
// Idx = ArraySize - 1; |
| 4059 |
QualType ArrayTy = BaseTy->isPointerType() |
| 4060 |
? E->getBase()->IgnoreParenImpCasts()->getType() |
| 4061 |
: BaseTy; |
| 4062 |
if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) { |
| 4063 |
Length = VAT->getSizeExpr(); |
| 4064 |
if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) { |
| 4065 |
ConstLength = *L; |
| 4066 |
Length = nullptr; |
| 4067 |
} |
| 4068 |
} else { |
| 4069 |
auto *CAT = C.getAsConstantArrayType(ArrayTy); |
| 4070 |
ConstLength = CAT->getSize(); |
| 4071 |
} |
| 4072 |
if (Length) { |
| 4073 |
auto *LengthVal = Builder.CreateIntCast( |
| 4074 |
EmitScalarExpr(Length), IntPtrTy, |
| 4075 |
Length->getType()->hasSignedIntegerRepresentation()); |
| 4076 |
Idx = Builder.CreateSub( |
| 4077 |
LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1", |
| 4078 |
/*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined()); |
| 4079 |
} else { |
| 4080 |
ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits); |
| 4081 |
--ConstLength; |
| 4082 |
Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength); |
| 4083 |
} |
| 4084 |
} |
| 4085 |
} |
| 4086 |
assert(Idx); |
| 4087 |
|
| 4088 |
Address EltPtr = Address::invalid(); |
| 4089 |
LValueBaseInfo BaseInfo; |
| 4090 |
TBAAAccessInfo TBAAInfo; |
| 4091 |
if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) { |
| 4092 |
// The base must be a pointer, which is not an aggregate. Emit |
| 4093 |
// it. It needs to be emitted first in case it's what captures |
| 4094 |
// the VLA bounds. |
| 4095 |
Address Base = |
| 4096 |
emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo, |
| 4097 |
BaseTy, VLA->getElementType(), IsLowerBound); |
| 4098 |
// The element count here is the total number of non-VLA elements. |
| 4099 |
llvm::Value *NumElements = getVLASize(VLA).NumElts; |
| 4100 |
|
| 4101 |
// Effectively, the multiply by the VLA size is part of the GEP. |
| 4102 |
// GEP indexes are signed, and scaling an index isn't permitted to |
| 4103 |
// signed-overflow, so we use the same semantics for our explicit |
| 4104 |
// multiply. We suppress this if overflow is not undefined behavior. |
| 4105 |
if (getLangOpts().isSignedOverflowDefined()) |
| 4106 |
Idx = Builder.CreateMul(Idx, NumElements); |
| 4107 |
else |
| 4108 |
Idx = Builder.CreateNSWMul(Idx, NumElements); |
| 4109 |
EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(), |
| 4110 |
!getLangOpts().isSignedOverflowDefined(), |
| 4111 |
/*signedIndices=*/false, E->getExprLoc()); |
| 4112 |
} else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) { |
| 4113 |
// If this is A[i] where A is an array, the frontend will have decayed the |
| 4114 |
// base to be a ArrayToPointerDecay implicit cast. While correct, it is |
| 4115 |
// inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a |
| 4116 |
// "gep x, i" here. Emit one "gep A, 0, i". |
| 4117 |
assert(Array->getType()->isArrayType() && |
| 4118 |
"Array to pointer decay must have array source type!"); |
| 4119 |
LValue ArrayLV; |
| 4120 |
// For simple multidimensional array indexing, set the 'accessed' flag for |
| 4121 |
// better bounds-checking of the base expression. |
| 4122 |
if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array)) |
| 4123 |
ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true); |
| 4124 |
else |
| 4125 |
ArrayLV = EmitLValue(Array); |
| 4126 |
|
| 4127 |
// Propagate the alignment from the array itself to the result. |
| 4128 |
EltPtr = emitArraySubscriptGEP( |
| 4129 |
*this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx}, |
| 4130 |
ResultExprTy, !getLangOpts().isSignedOverflowDefined(), |
| 4131 |
/*signedIndices=*/false, E->getExprLoc()); |
| 4132 |
BaseInfo = ArrayLV.getBaseInfo(); |
| 4133 |
TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy); |
| 4134 |
} else { |
| 4135 |
Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, |
| 4136 |
TBAAInfo, BaseTy, ResultExprTy, |
| 4137 |
IsLowerBound); |
| 4138 |
EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy, |
| 4139 |
!getLangOpts().isSignedOverflowDefined(), |
| 4140 |
/*signedIndices=*/false, E->getExprLoc()); |
| 4141 |
} |
| 4142 |
|
| 4143 |
return MakeAddrLValue(EltPtr, ResultExprTy, BaseInfo, TBAAInfo); |
| 4144 |
} |
| 4145 |
|
| 4146 |
LValue CodeGenFunction:: |
| 4147 |
EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { |
| 4148 |
// Emit the base vector as an l-value. |
| 4149 |
LValue Base; |
| 4150 |
|
| 4151 |
// ExtVectorElementExpr's base can either be a vector or pointer to vector. |
| 4152 |
if (E->isArrow()) { |
| 4153 |
// If it is a pointer to a vector, emit the address and form an lvalue with |
| 4154 |
// it. |
| 4155 |
LValueBaseInfo BaseInfo; |
| 4156 |
TBAAAccessInfo TBAAInfo; |
| 4157 |
Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo); |
| 4158 |
const auto *PT = E->getBase()->getType()->castAs<PointerType>(); |
| 4159 |
Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo); |
| 4160 |
Base.getQuals().removeObjCGCAttr(); |
| 4161 |
} else if (E->getBase()->isGLValue()) { |
| 4162 |
// Otherwise, if the base is an lvalue ( as in the case of foo.x.x), |
| 4163 |
// emit the base as an lvalue. |
| 4164 |
assert(E->getBase()->getType()->isVectorType()); |
| 4165 |
Base = EmitLValue(E->getBase()); |
| 4166 |
} else { |
| 4167 |
// Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such. |
| 4168 |
assert(E->getBase()->getType()->isVectorType() && |
| 4169 |
"Result must be a vector"); |
| 4170 |
llvm::Value *Vec = EmitScalarExpr(E->getBase()); |
| 4171 |
|
| 4172 |
// Store the vector to memory (because LValue wants an address). |
| 4173 |
Address VecMem = CreateMemTemp(E->getBase()->getType()); |
| 4174 |
Builder.CreateStore(Vec, VecMem); |
| 4175 |
Base = MakeAddrLValue(VecMem, E->getBase()->getType(), |
| 4176 |
AlignmentSource::Decl); |
| 4177 |
} |
| 4178 |
|
| 4179 |
QualType type = |
| 4180 |
E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers()); |
| 4181 |
|
| 4182 |
// Encode the element access list into a vector of unsigned indices. |
| 4183 |
SmallVector<uint32_t, 4> Indices; |
| 4184 |
E->getEncodedElementAccess(Indices); |
| 4185 |
|
| 4186 |
if (Base.isSimple()) { |
| 4187 |
llvm::Constant *CV = |
| 4188 |
llvm::ConstantDataVector::get(getLLVMContext(), Indices); |
| 4189 |
return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type, |
| 4190 |
Base.getBaseInfo(), TBAAAccessInfo()); |
| 4191 |
} |
| 4192 |
assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!"); |
| 4193 |
|
| 4194 |
llvm::Constant *BaseElts = Base.getExtVectorElts(); |
| 4195 |
SmallVector<llvm::Constant *, 4> CElts; |
| 4196 |
|
| 4197 |
for (unsigned i = 0, e = Indices.size(); i != e; ++i) |
| 4198 |
CElts.push_back(BaseElts->getAggregateElement(Indices[i])); |
| 4199 |
llvm::Constant *CV = llvm::ConstantVector::get(CElts); |
| 4200 |
return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type, |
| 4201 |
Base.getBaseInfo(), TBAAAccessInfo()); |
| 4202 |
} |
| 4203 |
|
| 4204 |
LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) { |
| 4205 |
if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, E)) { |
| 4206 |
EmitIgnoredExpr(E->getBase()); |
| 4207 |
return EmitDeclRefLValue(DRE); |
| 4208 |
} |
| 4209 |
|
| 4210 |
Expr *BaseExpr = E->getBase(); |
| 4211 |
// If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar. |
| 4212 |
LValue BaseLV; |
| 4213 |
if (E->isArrow()) { |
| 4214 |
LValueBaseInfo BaseInfo; |
| 4215 |
TBAAAccessInfo TBAAInfo; |
| 4216 |
Address Addr = EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo); |
| 4217 |
QualType PtrTy = BaseExpr->getType()->getPointeeType(); |
| 4218 |
SanitizerSet SkippedChecks; |
| 4219 |
bool IsBaseCXXThis = IsWrappedCXXThis(BaseExpr); |
| 4220 |
if (IsBaseCXXThis) |
| 4221 |
SkippedChecks.set(SanitizerKind::Alignment, true); |
| 4222 |
if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr)) |
| 4223 |
SkippedChecks.set(SanitizerKind::Null, true); |
| 4224 |
EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy, |
| 4225 |
/*Alignment=*/CharUnits::Zero(), SkippedChecks); |
| 4226 |
BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo); |
| 4227 |
} else |
| 4228 |
BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess); |
| 4229 |
|
| 4230 |
NamedDecl *ND = E->getMemberDecl(); |
| 4231 |
if (auto *Field = dyn_cast<FieldDecl>(ND)) { |
| 4232 |
LValue LV = EmitLValueForField(BaseLV, Field); |
| 4233 |
setObjCGCLValueClass(getContext(), E, LV); |
| 4234 |
if (getLangOpts().OpenMP) { |
| 4235 |
// If the member was explicitly marked as nontemporal, mark it as |
| 4236 |
// nontemporal. If the base lvalue is marked as nontemporal, mark access |
| 4237 |
// to children as nontemporal too. |
| 4238 |
if ((IsWrappedCXXThis(BaseExpr) && |
| 4239 |
CGM.getOpenMPRuntime().isNontemporalDecl(Field)) || |
| 4240 |
BaseLV.isNontemporal()) |
| 4241 |
LV.setNontemporal(/*Value=*/true); |
| 4242 |
} |
| 4243 |
return LV; |
| 4244 |
} |
| 4245 |
|
| 4246 |
if (const auto *FD = dyn_cast<FunctionDecl>(ND)) |
| 4247 |
return EmitFunctionDeclLValue(*this, E, FD); |
| 4248 |
|
| 4249 |
llvm_unreachable("Unhandled member declaration!"); |
| 4250 |
} |
| 4251 |
|
| 4252 |
/// Given that we are currently emitting a lambda, emit an l-value for |
| 4253 |
/// one of its members. |
| 4254 |
LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) { |
| 4255 |
if (CurCodeDecl) { |
| 4256 |
assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent()->isLambda()); |
| 4257 |
assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent() == Field->getParent()); |
| 4258 |
} |
| 4259 |
QualType LambdaTagType = |
| 4260 |
getContext().getTagDeclType(Field->getParent()); |
| 4261 |
LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue, LambdaTagType); |
| 4262 |
return EmitLValueForField(LambdaLV, Field); |
| 4263 |
} |
| 4264 |
|
| 4265 |
/// Get the field index in the debug info. The debug info structure/union |
| 4266 |
/// will ignore the unnamed bitfields. |
| 4267 |
unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec, |
| 4268 |
unsigned FieldIndex) { |
| 4269 |
unsigned I = 0, Skipped = 0; |
| 4270 |
|
| 4271 |
for (auto *F : Rec->getDefinition()->fields()) { |
| 4272 |
if (I == FieldIndex) |
| 4273 |
break; |
| 4274 |
if (F->isUnnamedBitfield()) |
| 4275 |
Skipped++; |
| 4276 |
I++; |
| 4277 |
} |
| 4278 |
|
| 4279 |
return FieldIndex - Skipped; |
| 4280 |
} |
| 4281 |
|
| 4282 |
/// Get the address of a zero-sized field within a record. The resulting |
| 4283 |
/// address doesn't necessarily have the right type. |
| 4284 |
static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base, |
| 4285 |
const FieldDecl *Field) { |
| 4286 |
CharUnits Offset = CGF.getContext().toCharUnitsFromBits( |
| 4287 |
CGF.getContext().getFieldOffset(Field)); |
| 4288 |
if (Offset.isZero()) |
| 4289 |
return Base; |
| 4290 |
Base = CGF.Builder.CreateElementBitCast(Base, CGF.Int8Ty); |
| 4291 |
return CGF.Builder.CreateConstInBoundsByteGEP(Base, Offset); |
| 4292 |
} |
| 4293 |
|
| 4294 |
/// Drill down to the storage of a field without walking into |
| 4295 |
/// reference types. |
| 4296 |
/// |
| 4297 |
/// The resulting address doesn't necessarily have the right type. |
| 4298 |
static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base, |
| 4299 |
const FieldDecl *field) { |
| 4300 |
if (field->isZeroSize(CGF.getContext())) |
| 4301 |
return emitAddrOfZeroSizeField(CGF, base, field); |
| 4302 |
|
| 4303 |
const RecordDecl *rec = field->getParent(); |
| 4304 |
|
| 4305 |
unsigned idx = |
| 4306 |
CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); |
| 4307 |
|
| 4308 |
return CGF.Builder.CreateStructGEP(base, idx, field->getName()); |
| 4309 |
} |
| 4310 |
|
| 4311 |
static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base, |
| 4312 |
Address addr, const FieldDecl *field) { |
| 4313 |
const RecordDecl *rec = field->getParent(); |
| 4314 |
llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType( |
| 4315 |
base.getType(), rec->getLocation()); |
| 4316 |
|
| 4317 |
unsigned idx = |
| 4318 |
CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); |
| 4319 |
|
| 4320 |
return CGF.Builder.CreatePreserveStructAccessIndex( |
| 4321 |
addr, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo); |
| 4322 |
} |
| 4323 |
|
| 4324 |
static bool hasAnyVptr(const QualType Type, const ASTContext &Context) { |
| 4325 |
const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl(); |
| 4326 |
if (!RD) |
| 4327 |
return false; |
| 4328 |
|
| 4329 |
if (RD->isDynamicClass()) |
| 4330 |
return true; |
| 4331 |
|
| 4332 |
for (const auto &Base : RD->bases()) |
| 4333 |
if (hasAnyVptr(Base.getType(), Context)) |
| 4334 |
return true; |
| 4335 |
|
| 4336 |
for (const FieldDecl *Field : RD->fields()) |
| 4337 |
if (hasAnyVptr(Field->getType(), Context)) |
| 4338 |
return true; |
| 4339 |
|
| 4340 |
return false; |
| 4341 |
} |
| 4342 |
|
| 4343 |
LValue CodeGenFunction::EmitLValueForField(LValue base, |
| 4344 |
const FieldDecl *field) { |
| 4345 |
LValueBaseInfo BaseInfo = base.getBaseInfo(); |
| 4346 |
|
| 4347 |
if (field->isBitField()) { |
| 4348 |
const CGRecordLayout &RL = |
| 4349 |
CGM.getTypes().getCGRecordLayout(field->getParent()); |
| 4350 |
const CGBitFieldInfo &Info = RL.getBitFieldInfo(field); |
| 4351 |
const bool UseVolatile = isAAPCS(CGM.getTarget()) && |
| 4352 |
CGM.getCodeGenOpts().AAPCSBitfieldWidth && |
| 4353 |
Info.VolatileStorageSize != 0 && |
| 4354 |
field->getType() |
| 4355 |
.withCVRQualifiers(base.getVRQualifiers()) |
| 4356 |
.isVolatileQualified(); |
| 4357 |
Address Addr = base.getAddress(*this); |
| 4358 |
unsigned Idx = RL.getLLVMFieldNo(field); |
| 4359 |
const RecordDecl *rec = field->getParent(); |
| 4360 |
if (!UseVolatile) { |
| 4361 |
if (!IsInPreservedAIRegion && |
| 4362 |
(!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) { |
| 4363 |
if (Idx != 0) |
| 4364 |
// For structs, we GEP to the field that the record layout suggests. |
| 4365 |
Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); |
| 4366 |
} else { |
| 4367 |
llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType( |
| 4368 |
getContext().getRecordType(rec), rec->getLocation()); |
| 4369 |
Addr = Builder.CreatePreserveStructAccessIndex( |
| 4370 |
Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()), |
| 4371 |
DbgInfo); |
| 4372 |
} |
| 4373 |
} |
| 4374 |
const unsigned SS = |
| 4375 |
UseVolatile ? Info.VolatileStorageSize : Info.StorageSize; |
| 4376 |
// Get the access type. |
| 4377 |
llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS); |
| 4378 |
if (Addr.getElementType() != FieldIntTy) |
| 4379 |
Addr = Builder.CreateElementBitCast(Addr, FieldIntTy); |
| 4380 |
if (UseVolatile) { |
| 4381 |
const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity(); |
| 4382 |
if (VolatileOffset) |
| 4383 |
Addr = Builder.CreateConstInBoundsGEP(Addr, VolatileOffset); |
| 4384 |
} |
| 4385 |
|
| 4386 |
QualType fieldType = |
| 4387 |
field->getType().withCVRQualifiers(base.getVRQualifiers()); |
| 4388 |
// TODO: Support TBAA for bit fields. |
| 4389 |
LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource()); |
| 4390 |
return LValue::MakeBitfield(Addr, Info, fieldType, FieldBaseInfo, |
| 4391 |
TBAAAccessInfo()); |
| 4392 |
} |
| 4393 |
|
| 4394 |
// Fields of may-alias structures are may-alias themselves. |
| 4395 |
// FIXME: this should get propagated down through anonymous structs |
| 4396 |
// and unions. |
| 4397 |
QualType FieldType = field->getType(); |
| 4398 |
const RecordDecl *rec = field->getParent(); |
| 4399 |
AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource(); |
| 4400 |
LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource)); |
| 4401 |
TBAAAccessInfo FieldTBAAInfo; |
| 4402 |
if (base.getTBAAInfo().isMayAlias() || |
| 4403 |
rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) { |
| 4404 |
FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo(); |
| 4405 |
} else if (rec->isUnion()) { |
| 4406 |
// TODO: Support TBAA for unions. |
| 4407 |
FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo(); |
| 4408 |
} else { |
| 4409 |
// If no base type been assigned for the base access, then try to generate |
| 4410 |
// one for this base lvalue. |
| 4411 |
FieldTBAAInfo = base.getTBAAInfo(); |
| 4412 |
if (!FieldTBAAInfo.BaseType) { |
| 4413 |
FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(base.getType()); |
| 4414 |
assert(!FieldTBAAInfo.Offset && |
| 4415 |
"Nonzero offset for an access with no base type!"); |
| 4416 |
} |
| 4417 |
|
| 4418 |
// Adjust offset to be relative to the base type. |
| 4419 |
const ASTRecordLayout &Layout = |
| 4420 |
getContext().getASTRecordLayout(field->getParent()); |
| 4421 |
unsigned CharWidth = getContext().getCharWidth(); |
| 4422 |
if (FieldTBAAInfo.BaseType) |
| 4423 |
FieldTBAAInfo.Offset += |
| 4424 |
Layout.getFieldOffset(field->getFieldIndex()) / CharWidth; |
| 4425 |
|
| 4426 |
// Update the final access type and size. |
| 4427 |
FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(FieldType); |
| 4428 |
FieldTBAAInfo.Size = |
| 4429 |
getContext().getTypeSizeInChars(FieldType).getQuantity(); |
| 4430 |
} |
| 4431 |
|
| 4432 |
Address addr = base.getAddress(*this); |
| 4433 |
if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) { |
| 4434 |
if (CGM.getCodeGenOpts().StrictVTablePointers && |
| 4435 |
ClassDef->isDynamicClass()) { |
| 4436 |
// Getting to any field of dynamic object requires stripping dynamic |
| 4437 |
// information provided by invariant.group. This is because accessing |
| 4438 |
// fields may leak the real address of dynamic object, which could result |
| 4439 |
// in miscompilation when leaked pointer would be compared. |
| 4440 |
auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer()); |
| 4441 |
addr = Address(stripped, addr.getElementType(), addr.getAlignment()); |
| 4442 |
} |
| 4443 |
} |
| 4444 |
|
| 4445 |
unsigned RecordCVR = base.getVRQualifiers(); |
| 4446 |
if (rec->isUnion()) { |
| 4447 |
// For unions, there is no pointer adjustment. |
| 4448 |
if (CGM.getCodeGenOpts().StrictVTablePointers && |
| 4449 |
hasAnyVptr(FieldType, getContext())) |
| 4450 |
// Because unions can easily skip invariant.barriers, we need to add |
| 4451 |
// a barrier every time CXXRecord field with vptr is referenced. |
| 4452 |
addr = Builder.CreateLaunderInvariantGroup(addr); |
| 4453 |
|
| 4454 |
if (IsInPreservedAIRegion || |
| 4455 |
(getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) { |
| 4456 |
// Remember the original union field index |
| 4457 |
llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(), |
| 4458 |
rec->getLocation()); |
| 4459 |
addr = Address( |
| 4460 |
Builder.CreatePreserveUnionAccessIndex( |
| 4461 |
addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), |
| 4462 |
addr.getElementType(), addr.getAlignment()); |
| 4463 |
} |
| 4464 |
|
| 4465 |
if (FieldType->isReferenceType()) |
| 4466 |
addr = Builder.CreateElementBitCast( |
| 4467 |
addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName()); |
| 4468 |
} else { |
| 4469 |
if (!IsInPreservedAIRegion && |
| 4470 |
(!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) |
| 4471 |
// For structs, we GEP to the field that the record layout suggests. |
| 4472 |
addr = emitAddrOfFieldStorage(*this, addr, field); |
| 4473 |
else |
| 4474 |
// Remember the original struct field index |
| 4475 |
addr = emitPreserveStructAccess(*this, base, addr, field); |
| 4476 |
} |
| 4477 |
|
| 4478 |
// If this is a reference field, load the reference right now. |
| 4479 |
if (FieldType->isReferenceType()) { |
| 4480 |
LValue RefLVal = |
| 4481 |
MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo); |
| 4482 |
if (RecordCVR & Qualifiers::Volatile) |
| 4483 |
RefLVal.getQuals().addVolatile(); |
| 4484 |
addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo); |
| 4485 |
|
| 4486 |
// Qualifiers on the struct don't apply to the referencee. |
| 4487 |
RecordCVR = 0; |
| 4488 |
FieldType = FieldType->getPointeeType(); |
| 4489 |
} |
| 4490 |
|
| 4491 |
// Make sure that the address is pointing to the right type. This is critical |
| 4492 |
// for both unions and structs. A union needs a bitcast, a struct element |
| 4493 |
// will need a bitcast if the LLVM type laid out doesn't match the desired |
| 4494 |
// type. |
| 4495 |
addr = Builder.CreateElementBitCast( |
| 4496 |
addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName()); |
| 4497 |
|
| 4498 |
if (field->hasAttr<AnnotateAttr>()) |
| 4499 |
addr = EmitFieldAnnotations(field, addr); |
| 4500 |
|
| 4501 |
LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo); |
| 4502 |
LV.getQuals().addCVRQualifiers(RecordCVR); |
| 4503 |
|
| 4504 |
// __weak attribute on a field is ignored. |
| 4505 |
if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak) |
| 4506 |
LV.getQuals().removeObjCGCAttr(); |
| 4507 |
|
| 4508 |
return LV; |
| 4509 |
} |
| 4510 |
|
| 4511 |
LValue |
| 4512 |
CodeGenFunction::EmitLValueForFieldInitialization(LValue Base, |
| 4513 |
const FieldDecl *Field) { |
| 4514 |
QualType FieldType = Field->getType(); |
| 4515 |
|
| 4516 |
if (!FieldType->isReferenceType()) |
| 4517 |
return EmitLValueForField(Base, Field); |
| 4518 |
|
| 4519 |
Address V = emitAddrOfFieldStorage(*this, Base.getAddress(*this), Field); |
| 4520 |
|
| 4521 |
// Make sure that the address is pointing to the right type. |
| 4522 |
llvm::Type *llvmType = ConvertTypeForMem(FieldType); |
| 4523 |
V = Builder.CreateElementBitCast(V, llvmType, Field->getName()); |
| 4524 |
|
| 4525 |
// TODO: Generate TBAA information that describes this access as a structure |
| 4526 |
// member access and not just an access to an object of the field's type. This |
| 4527 |
// should be similar to what we do in EmitLValueForField(). |
| 4528 |
LValueBaseInfo BaseInfo = Base.getBaseInfo(); |
| 4529 |
AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource(); |
| 4530 |
LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(FieldAlignSource)); |
| 4531 |
return MakeAddrLValue(V, FieldType, FieldBaseInfo, |
| 4532 |
CGM.getTBAAInfoForSubobject(Base, FieldType)); |
| 4533 |
} |
| 4534 |
|
| 4535 |
LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){ |
| 4536 |
if (E->isFileScope()) { |
| 4537 |
ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E); |
| 4538 |
return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl); |
| 4539 |
} |
| 4540 |
if (E->getType()->isVariablyModifiedType()) |
| 4541 |
// make sure to emit the VLA size. |
| 4542 |
EmitVariablyModifiedType(E->getType()); |
| 4543 |
|
| 4544 |
Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral"); |
| 4545 |
const Expr *InitExpr = E->getInitializer(); |
| 4546 |
LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl); |
| 4547 |
|
| 4548 |
EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(), |
| 4549 |
/*Init*/ true); |
| 4550 |
|
| 4551 |
// Block-scope compound literals are destroyed at the end of the enclosing |
| 4552 |
// scope in C. |
| 4553 |
if (!getLangOpts().CPlusPlus) |
| 4554 |
if (QualType::DestructionKind DtorKind = E->getType().isDestructedType()) |
| 4555 |
pushLifetimeExtendedDestroy(getCleanupKind(DtorKind), DeclPtr, |
| 4556 |
E->getType(), getDestroyer(DtorKind), |
| 4557 |
DtorKind & EHCleanup); |
| 4558 |
|
| 4559 |
return Result; |
| 4560 |
} |
| 4561 |
|
| 4562 |
LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) { |
| 4563 |
if (!E->isGLValue()) |
| 4564 |
// Initializing an aggregate temporary in C++11: T{...}. |
| 4565 |
return EmitAggExprToLValue(E); |
| 4566 |
|
| 4567 |
// An lvalue initializer list must be initializing a reference. |
| 4568 |
assert(E->isTransparent() && "non-transparent glvalue init list"); |
| 4569 |
return EmitLValue(E->getInit(0)); |
| 4570 |
} |
| 4571 |
|
| 4572 |
/// Emit the operand of a glvalue conditional operator. This is either a glvalue |
| 4573 |
/// or a (possibly-parenthesized) throw-expression. If this is a throw, no |
| 4574 |
/// LValue is returned and the current block has been terminated. |
| 4575 |
static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF, |
| 4576 |
const Expr *Operand) { |
| 4577 |
if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Operand->IgnoreParens())) { |
| 4578 |
CGF.EmitCXXThrowExpr(ThrowExpr, /*KeepInsertionPoint*/false); |
| 4579 |
return std::nullopt; |
| 4580 |
} |
| 4581 |
|
| 4582 |
return CGF.EmitLValue(Operand); |
| 4583 |
} |
| 4584 |
|
| 4585 |
namespace { |
| 4586 |
// Handle the case where the condition is a constant evaluatable simple integer, |
| 4587 |
// which means we don't have to separately handle the true/false blocks. |
| 4588 |
std::optional<LValue> HandleConditionalOperatorLValueSimpleCase( |
| 4589 |
CodeGenFunction &CGF, const AbstractConditionalOperator *E) { |
| 4590 |
const Expr *condExpr = E->getCond(); |
| 4591 |
bool CondExprBool; |
| 4592 |
if (CGF.ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) { |
| 4593 |
const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr(); |
| 4594 |
if (!CondExprBool) |
| 4595 |
std::swap(Live, Dead); |
| 4596 |
|
| 4597 |
if (!CGF.ContainsLabel(Dead)) { |
| 4598 |
// If the true case is live, we need to track its region. |
| 4599 |
if (CondExprBool) |
| 4600 |
CGF.incrementProfileCounter(E); |
| 4601 |
// If a throw expression we emit it and return an undefined lvalue |
| 4602 |
// because it can't be used. |
| 4603 |
if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Live->IgnoreParens())) { |
| 4604 |
CGF.EmitCXXThrowExpr(ThrowExpr); |
| 4605 |
llvm::Type *ElemTy = CGF.ConvertType(Dead->getType()); |
| 4606 |
llvm::Type *Ty = llvm::PointerType::getUnqual(ElemTy); |
| 4607 |
return CGF.MakeAddrLValue( |
| 4608 |
Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()), |
| 4609 |
Dead->getType()); |
| 4610 |
} |
| 4611 |
return CGF.EmitLValue(Live); |
| 4612 |
} |
| 4613 |
} |
| 4614 |
return std::nullopt; |
| 4615 |
} |
| 4616 |
struct ConditionalInfo { |
| 4617 |
llvm::BasicBlock *lhsBlock, *rhsBlock; |
| 4618 |
std::optional<LValue> LHS, RHS; |
| 4619 |
}; |
| 4620 |
|
| 4621 |
// Create and generate the 3 blocks for a conditional operator. |
| 4622 |
// Leaves the 'current block' in the continuation basic block. |
| 4623 |
template<typename FuncTy> |
| 4624 |
ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF, |
| 4625 |
const AbstractConditionalOperator *E, |
| 4626 |
const FuncTy &BranchGenFunc) { |
| 4627 |
ConditionalInfo Info{CGF.createBasicBlock("cond.true"), |
| 4628 |
CGF.createBasicBlock("cond.false"), std::nullopt, |
| 4629 |
std::nullopt}; |
| 4630 |
llvm::BasicBlock *endBlock = CGF.createBasicBlock("cond.end"); |
| 4631 |
|
| 4632 |
CodeGenFunction::ConditionalEvaluation eval(CGF); |
| 4633 |
CGF.EmitBranchOnBoolExpr(E->getCond(), Info.lhsBlock, Info.rhsBlock, |
| 4634 |
CGF.getProfileCount(E)); |
| 4635 |
|
| 4636 |
// Any temporaries created here are conditional. |
| 4637 |
CGF.EmitBlock(Info.lhsBlock); |
| 4638 |
CGF.incrementProfileCounter(E); |
| 4639 |
eval.begin(CGF); |
| 4640 |
Info.LHS = BranchGenFunc(CGF, E->getTrueExpr()); |
| 4641 |
eval.end(CGF); |
| 4642 |
Info.lhsBlock = CGF.Builder.GetInsertBlock(); |
| 4643 |
|
| 4644 |
if (Info.LHS) |
| 4645 |
CGF.Builder.CreateBr(endBlock); |
| 4646 |
|
| 4647 |
// Any temporaries created here are conditional. |
| 4648 |
CGF.EmitBlock(Info.rhsBlock); |
| 4649 |
eval.begin(CGF); |
| 4650 |
Info.RHS = BranchGenFunc(CGF, E->getFalseExpr()); |
| 4651 |
eval.end(CGF); |
| 4652 |
Info.rhsBlock = CGF.Builder.GetInsertBlock(); |
| 4653 |
CGF.EmitBlock(endBlock); |
| 4654 |
|
| 4655 |
return Info; |
| 4656 |
} |
| 4657 |
} // namespace |
| 4658 |
|
| 4659 |
void CodeGenFunction::EmitIgnoredConditionalOperator( |
| 4660 |
const AbstractConditionalOperator *E) { |
| 4661 |
if (!E->isGLValue()) { |
| 4662 |
// ?: here should be an aggregate. |
| 4663 |
assert(hasAggregateEvaluationKind(E->getType()) && |
| 4664 |
"Unexpected conditional operator!"); |
| 4665 |
return (void)EmitAggExprToLValue(E); |
| 4666 |
} |
| 4667 |
|
| 4668 |
OpaqueValueMapping binding(*this, E); |
| 4669 |
if (HandleConditionalOperatorLValueSimpleCase(*this, E)) |
| 4670 |
return; |
| 4671 |
|
| 4672 |
EmitConditionalBlocks(*this, E, [](CodeGenFunction &CGF, const Expr *E) { |
| 4673 |
CGF.EmitIgnoredExpr(E); |
| 4674 |
return LValue{}; |
| 4675 |
}); |
| 4676 |
} |
| 4677 |
LValue CodeGenFunction::EmitConditionalOperatorLValue( |
| 4678 |
const AbstractConditionalOperator *expr) { |
| 4679 |
if (!expr->isGLValue()) { |
| 4680 |
// ?: here should be an aggregate. |
| 4681 |
assert(hasAggregateEvaluationKind(expr->getType()) && |
| 4682 |
"Unexpected conditional operator!"); |
| 4683 |
return EmitAggExprToLValue(expr); |
| 4684 |
} |
| 4685 |
|
| 4686 |
OpaqueValueMapping binding(*this, expr); |
| 4687 |
if (std::optional<LValue> Res = |
| 4688 |
HandleConditionalOperatorLValueSimpleCase(*this, expr)) |
| 4689 |
return *Res; |
| 4690 |
|
| 4691 |
ConditionalInfo Info = EmitConditionalBlocks( |
| 4692 |
*this, expr, [](CodeGenFunction &CGF, const Expr *E) { |
| 4693 |
return EmitLValueOrThrowExpression(CGF, E); |
| 4694 |
}); |
| 4695 |
|
| 4696 |
if ((Info.LHS && !Info.LHS->isSimple()) || |
| 4697 |
(Info.RHS && !Info.RHS->isSimple())) |
| 4698 |
return EmitUnsupportedLValue(expr, "conditional operator"); |
| 4699 |
|
| 4700 |
if (Info.LHS && Info.RHS) { |
| 4701 |
Address lhsAddr = Info.LHS->getAddress(*this); |
| 4702 |
Address rhsAddr = Info.RHS->getAddress(*this); |
| 4703 |
llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue"); |
| 4704 |
phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock); |
| 4705 |
phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock); |
| 4706 |
Address result(phi, lhsAddr.getElementType(), |
| 4707 |
std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment())); |
| 4708 |
AlignmentSource alignSource = |
| 4709 |
std::max(Info.LHS->getBaseInfo().getAlignmentSource(), |
| 4710 |
Info.RHS->getBaseInfo().getAlignmentSource()); |
| 4711 |
TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator( |
| 4712 |
Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo()); |
| 4713 |
return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource), |
| 4714 |
TBAAInfo); |
| 4715 |
} else { |
| 4716 |
assert((Info.LHS || Info.RHS) && |
| 4717 |
"both operands of glvalue conditional are throw-expressions?"); |
| 4718 |
return Info.LHS ? *Info.LHS : *Info.RHS; |
| 4719 |
} |
| 4720 |
} |
| 4721 |
|
| 4722 |
/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference |
| 4723 |
/// type. If the cast is to a reference, we can have the usual lvalue result, |
| 4724 |
/// otherwise if a cast is needed by the code generator in an lvalue context, |
| 4725 |
/// then it must mean that we need the address of an aggregate in order to |
| 4726 |
/// access one of its members. This can happen for all the reasons that casts |
| 4727 |
/// are permitted with aggregate result, including noop aggregate casts, and |
| 4728 |
/// cast from scalar to union. |
| 4729 |
LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) { |
| 4730 |
switch (E->getCastKind()) { |
| 4731 |
case CK_ToVoid: |
| 4732 |
case CK_BitCast: |
| 4733 |
case CK_LValueToRValueBitCast: |
| 4734 |
case CK_ArrayToPointerDecay: |
| 4735 |
case CK_FunctionToPointerDecay: |
| 4736 |
case CK_NullToMemberPointer: |
| 4737 |
case CK_NullToPointer: |
| 4738 |
case CK_IntegralToPointer: |
| 4739 |
case CK_PointerToIntegral: |
| 4740 |
case CK_PointerToBoolean: |
| 4741 |
case CK_VectorSplat: |
| 4742 |
case CK_IntegralCast: |
| 4743 |
case CK_BooleanToSignedIntegral: |
| 4744 |
case CK_IntegralToBoolean: |
| 4745 |
case CK_IntegralToFloating: |
| 4746 |
case CK_FloatingToIntegral: |
| 4747 |
case CK_FloatingToBoolean: |
| 4748 |
case CK_FloatingCast: |
| 4749 |
case CK_FloatingRealToComplex: |
| 4750 |
case CK_FloatingComplexToReal: |
| 4751 |
case CK_FloatingComplexToBoolean: |
| 4752 |
case CK_FloatingComplexCast: |
| 4753 |
case CK_FloatingComplexToIntegralComplex: |
| 4754 |
case CK_IntegralRealToComplex: |
| 4755 |
case CK_IntegralComplexToReal: |
| 4756 |
case CK_IntegralComplexToBoolean: |
| 4757 |
case CK_IntegralComplexCast: |
| 4758 |
case CK_IntegralComplexToFloatingComplex: |
| 4759 |
case CK_DerivedToBaseMemberPointer: |
| 4760 |
case CK_BaseToDerivedMemberPointer: |
| 4761 |
case CK_MemberPointerToBoolean: |
| 4762 |
case CK_ReinterpretMemberPointer: |
| 4763 |
case CK_AnyPointerToBlockPointerCast: |
| 4764 |
case CK_ARCProduceObject: |
| 4765 |
case CK_ARCConsumeObject: |
| 4766 |
case CK_ARCReclaimReturnedObject: |
| 4767 |
case CK_ARCExtendBlockObject: |
| 4768 |
case CK_CopyAndAutoreleaseBlockObject: |
| 4769 |
case CK_IntToOCLSampler: |
| 4770 |
case CK_FloatingToFixedPoint: |
| 4771 |
case CK_FixedPointToFloating: |
| 4772 |
case CK_FixedPointCast: |
| 4773 |
case CK_FixedPointToBoolean: |
| 4774 |
case CK_FixedPointToIntegral: |
| 4775 |
case CK_IntegralToFixedPoint: |
| 4776 |
case CK_MatrixCast: |
| 4777 |
return EmitUnsupportedLValue(E, "unexpected cast lvalue"); |
| 4778 |
|
| 4779 |
case CK_Dependent: |
| 4780 |
llvm_unreachable("dependent cast kind in IR gen!"); |
| 4781 |
|
| 4782 |
case CK_BuiltinFnToFnPtr: |
| 4783 |
llvm_unreachable("builtin functions are handled elsewhere"); |
| 4784 |
|
| 4785 |
// These are never l-values; just use the aggregate emission code. |
| 4786 |
case CK_NonAtomicToAtomic: |
| 4787 |
case CK_AtomicToNonAtomic: |
| 4788 |
return EmitAggExprToLValue(E); |
| 4789 |
|
| 4790 |
case CK_Dynamic: { |
| 4791 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 4792 |
Address V = LV.getAddress(*this); |
| 4793 |
const auto *DCE = cast<CXXDynamicCastExpr>(E); |
| 4794 |
return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType()); |
| 4795 |
} |
| 4796 |
|
| 4797 |
case CK_ConstructorConversion: |
| 4798 |
case CK_UserDefinedConversion: |
| 4799 |
case CK_CPointerToObjCPointerCast: |
| 4800 |
case CK_BlockPointerToObjCPointerCast: |
| 4801 |
case CK_LValueToRValue: |
| 4802 |
return EmitLValue(E->getSubExpr()); |
| 4803 |
|
| 4804 |
case CK_NoOp: { |
| 4805 |
// CK_NoOp can model a qualification conversion, which can remove an array |
| 4806 |
// bound and change the IR type. |
| 4807 |
// FIXME: Once pointee types are removed from IR, remove this. |
| 4808 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 4809 |
if (LV.isSimple()) { |
| 4810 |
Address V = LV.getAddress(*this); |
| 4811 |
if (V.isValid()) { |
| 4812 |
llvm::Type *T = ConvertTypeForMem(E->getType()); |
| 4813 |
if (V.getElementType() != T) |
| 4814 |
LV.setAddress(Builder.CreateElementBitCast(V, T)); |
| 4815 |
} |
| 4816 |
} |
| 4817 |
return LV; |
| 4818 |
} |
| 4819 |
|
| 4820 |
case CK_UncheckedDerivedToBase: |
| 4821 |
case CK_DerivedToBase: { |
| 4822 |
const auto *DerivedClassTy = |
| 4823 |
E->getSubExpr()->getType()->castAs<RecordType>(); |
| 4824 |
auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl()); |
| 4825 |
|
| 4826 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 4827 |
Address This = LV.getAddress(*this); |
| 4828 |
|
| 4829 |
// Perform the derived-to-base conversion |
| 4830 |
Address Base = GetAddressOfBaseClass( |
| 4831 |
This, DerivedClassDecl, E->path_begin(), E->path_end(), |
| 4832 |
/*NullCheckValue=*/false, E->getExprLoc()); |
| 4833 |
|
| 4834 |
// TODO: Support accesses to members of base classes in TBAA. For now, we |
| 4835 |
// conservatively pretend that the complete object is of the base class |
| 4836 |
// type. |
| 4837 |
return MakeAddrLValue(Base, E->getType(), LV.getBaseInfo(), |
| 4838 |
CGM.getTBAAInfoForSubobject(LV, E->getType())); |
| 4839 |
} |
| 4840 |
case CK_ToUnion: |
| 4841 |
return EmitAggExprToLValue(E); |
| 4842 |
case CK_BaseToDerived: { |
| 4843 |
const auto *DerivedClassTy = E->getType()->castAs<RecordType>(); |
| 4844 |
auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl()); |
| 4845 |
|
| 4846 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 4847 |
|
| 4848 |
// Perform the base-to-derived conversion |
| 4849 |
Address Derived = GetAddressOfDerivedClass( |
| 4850 |
LV.getAddress(*this), DerivedClassDecl, E->path_begin(), E->path_end(), |
| 4851 |
/*NullCheckValue=*/false); |
| 4852 |
|
| 4853 |
// C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is |
| 4854 |
// performed and the object is not of the derived type. |
| 4855 |
if (sanitizePerformTypeCheck()) |
| 4856 |
EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(), |
| 4857 |
Derived.getPointer(), E->getType()); |
| 4858 |
|
| 4859 |
if (SanOpts.has(SanitizerKind::CFIDerivedCast)) |
| 4860 |
EmitVTablePtrCheckForCast(E->getType(), Derived, |
| 4861 |
/*MayBeNull=*/false, CFITCK_DerivedCast, |
| 4862 |
E->getBeginLoc()); |
| 4863 |
|
| 4864 |
return MakeAddrLValue(Derived, E->getType(), LV.getBaseInfo(), |
| 4865 |
CGM.getTBAAInfoForSubobject(LV, E->getType())); |
| 4866 |
} |
| 4867 |
case CK_LValueBitCast: { |
| 4868 |
// This must be a reinterpret_cast (or c-style equivalent). |
| 4869 |
const auto *CE = cast<ExplicitCastExpr>(E); |
| 4870 |
|
| 4871 |
CGM.EmitExplicitCastExprType(CE, this); |
| 4872 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 4873 |
Address V = Builder.CreateElementBitCast( |
| 4874 |
LV.getAddress(*this), |
| 4875 |
ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType())); |
| 4876 |
|
| 4877 |
if (SanOpts.has(SanitizerKind::CFIUnrelatedCast)) |
| 4878 |
EmitVTablePtrCheckForCast(E->getType(), V, |
| 4879 |
/*MayBeNull=*/false, CFITCK_UnrelatedCast, |
| 4880 |
E->getBeginLoc()); |
| 4881 |
|
| 4882 |
return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(), |
| 4883 |
CGM.getTBAAInfoForSubobject(LV, E->getType())); |
| 4884 |
} |
| 4885 |
case CK_AddressSpaceConversion: { |
| 4886 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 4887 |
QualType DestTy = getContext().getPointerType(E->getType()); |
| 4888 |
llvm::Value *V = getTargetHooks().performAddrSpaceCast( |
| 4889 |
*this, LV.getPointer(*this), |
| 4890 |
E->getSubExpr()->getType().getAddressSpace(), |
| 4891 |
E->getType().getAddressSpace(), ConvertType(DestTy)); |
| 4892 |
return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()), |
| 4893 |
LV.getAddress(*this).getAlignment()), |
| 4894 |
E->getType(), LV.getBaseInfo(), LV.getTBAAInfo()); |
| 4895 |
} |
| 4896 |
case CK_ObjCObjectLValueCast: { |
| 4897 |
LValue LV = EmitLValue(E->getSubExpr()); |
| 4898 |
Address V = Builder.CreateElementBitCast(LV.getAddress(*this), |
| 4899 |
ConvertType(E->getType())); |
| 4900 |
return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(), |
| 4901 |
CGM.getTBAAInfoForSubobject(LV, E->getType())); |
| 4902 |
} |
| 4903 |
case CK_ZeroToOCLOpaqueType: |
| 4904 |
llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid"); |
| 4905 |
} |
| 4906 |
|
| 4907 |
llvm_unreachable("Unhandled lvalue cast kind?"); |
| 4908 |
} |
| 4909 |
|
| 4910 |
LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) { |
| 4911 |
assert(OpaqueValueMappingData::shouldBindAsLValue(e)); |
| 4912 |
return getOrCreateOpaqueLValueMapping(e); |
| 4913 |
} |
| 4914 |
|
| 4915 |
LValue |
| 4916 |
CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) { |
| 4917 |
assert(OpaqueValueMapping::shouldBindAsLValue(e)); |
| 4918 |
|
| 4919 |
llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator |
| 4920 |
it = OpaqueLValues.find(e); |
| 4921 |
|
| 4922 |
if (it != OpaqueLValues.end()) |
| 4923 |
return it->second; |
| 4924 |
|
| 4925 |
assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted"); |
| 4926 |
return EmitLValue(e->getSourceExpr()); |
| 4927 |
} |
| 4928 |
|
| 4929 |
RValue |
| 4930 |
CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) { |
| 4931 |
assert(!OpaqueValueMapping::shouldBindAsLValue(e)); |
| 4932 |
|
| 4933 |
llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator |
| 4934 |
it = OpaqueRValues.find(e); |
| 4935 |
|
| 4936 |
if (it != OpaqueRValues.end()) |
| 4937 |
return it->second; |
| 4938 |
|
| 4939 |
assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted"); |
| 4940 |
return EmitAnyExpr(e->getSourceExpr()); |
| 4941 |
} |
| 4942 |
|
| 4943 |
RValue CodeGenFunction::EmitRValueForField(LValue LV, |
| 4944 |
const FieldDecl *FD, |
| 4945 |
SourceLocation Loc) { |
| 4946 |
QualType FT = FD->getType(); |
| 4947 |
LValue FieldLV = EmitLValueForField(LV, FD); |
| 4948 |
switch (getEvaluationKind(FT)) { |
| 4949 |
case TEK_Complex: |
| 4950 |
return RValue::getComplex(EmitLoadOfComplex(FieldLV, Loc)); |
| 4951 |
case TEK_Aggregate: |
| 4952 |
return FieldLV.asAggregateRValue(*this); |
| 4953 |
case TEK_Scalar: |
| 4954 |
// This routine is used to load fields one-by-one to perform a copy, so |
| 4955 |
// don't load reference fields. |
| 4956 |
if (FD->getType()->isReferenceType()) |
| 4957 |
return RValue::get(FieldLV.getPointer(*this)); |
| 4958 |
// Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a |
| 4959 |
// primitive load. |
| 4960 |
if (FieldLV.isBitField()) |
| 4961 |
return EmitLoadOfLValue(FieldLV, Loc); |
| 4962 |
return RValue::get(EmitLoadOfScalar(FieldLV, Loc)); |
| 4963 |
} |
| 4964 |
llvm_unreachable("bad evaluation kind"); |
| 4965 |
} |
| 4966 |
|
| 4967 |
//===--------------------------------------------------------------------===// |
| 4968 |
// Expression Emission |
| 4969 |
//===--------------------------------------------------------------------===// |
| 4970 |
|
| 4971 |
RValue CodeGenFunction::EmitCallExpr(const CallExpr *E, |
| 4972 |
ReturnValueSlot ReturnValue) { |
| 4973 |
// Builtins never have block type. |
| 4974 |
if (E->getCallee()->getType()->isBlockPointerType()) |
| 4975 |
return EmitBlockCallExpr(E, ReturnValue); |
| 4976 |
|
| 4977 |
if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E)) |
| 4978 |
return EmitCXXMemberCallExpr(CE, ReturnValue); |
| 4979 |
|
| 4980 |
if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E)) |
| 4981 |
return EmitCUDAKernelCallExpr(CE, ReturnValue); |
| 4982 |
|
| 4983 |
if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E)) |
| 4984 |
if (const CXXMethodDecl *MD = |
| 4985 |
dyn_cast_or_null<CXXMethodDecl>(CE->getCalleeDecl())) |
| 4986 |
return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue); |
| 4987 |
|
| 4988 |
CGCallee callee = EmitCallee(E->getCallee()); |
| 4989 |
|
| 4990 |
if (callee.isBuiltin()) { |
| 4991 |
return EmitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(), |
| 4992 |
E, ReturnValue); |
| 4993 |
} |
| 4994 |
|
| 4995 |
if (callee.isPseudoDestructor()) { |
| 4996 |
return EmitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr()); |
| 4997 |
} |
| 4998 |
|
| 4999 |
return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue); |
| 5000 |
} |
| 5001 |
|
| 5002 |
/// Emit a CallExpr without considering whether it might be a subclass. |
| 5003 |
RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E, |
| 5004 |
ReturnValueSlot ReturnValue) { |
| 5005 |
CGCallee Callee = EmitCallee(E->getCallee()); |
| 5006 |
return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue); |
| 5007 |
} |
| 5008 |
|
| 5009 |
// Detect the unusual situation where an inline version is shadowed by a |
| 5010 |
// non-inline version. In that case we should pick the external one |
| 5011 |
// everywhere. That's GCC behavior too. |
| 5012 |
static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) { |
| 5013 |
for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl()) |
| 5014 |
if (!PD->isInlineBuiltinDeclaration()) |
| 5015 |
return false; |
| 5016 |
return true; |
| 5017 |
} |
| 5018 |
|
| 5019 |
static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) { |
| 5020 |
const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); |
| 5021 |
|
| 5022 |
if (auto builtinID = FD->getBuiltinID()) { |
| 5023 |
std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str(); |
| 5024 |
std::string NoBuiltins = "no-builtins"; |
| 5025 |
|
| 5026 |
StringRef Ident = CGF.CGM.getMangledName(GD); |
| 5027 |
std::string FDInlineName = (Ident + ".inline").str(); |
| 5028 |
|
| 5029 |
bool IsPredefinedLibFunction = |
| 5030 |
CGF.getContext().BuiltinInfo.isPredefinedLibFunction(builtinID); |
| 5031 |
bool HasAttributeNoBuiltin = |
| 5032 |
CGF.CurFn->getAttributes().hasFnAttr(NoBuiltinFD) || |
| 5033 |
CGF.CurFn->getAttributes().hasFnAttr(NoBuiltins); |
| 5034 |
|
| 5035 |
// When directing calling an inline builtin, call it through it's mangled |
| 5036 |
// name to make it clear it's not the actual builtin. |
| 5037 |
if (CGF.CurFn->getName() != FDInlineName && |
| 5038 |
OnlyHasInlineBuiltinDeclaration(FD)) { |
| 5039 |
llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD); |
| 5040 |
llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr); |
| 5041 |
llvm::Module *M = Fn->getParent(); |
| 5042 |
llvm::Function *Clone = M->getFunction(FDInlineName); |
| 5043 |
if (!Clone) { |
| 5044 |
Clone = llvm::Function::Create(Fn->getFunctionType(), |
| 5045 |
llvm::GlobalValue::InternalLinkage, |
| 5046 |
Fn->getAddressSpace(), FDInlineName, M); |
| 5047 |
Clone->addFnAttr(llvm::Attribute::AlwaysInline); |
| 5048 |
} |
| 5049 |
return CGCallee::forDirect(Clone, GD); |
| 5050 |
} |
| 5051 |
|
| 5052 |
// Replaceable builtins provide their own implementation of a builtin. If we |
| 5053 |
// are in an inline builtin implementation, avoid trivial infinite |
| 5054 |
// recursion. Honor __attribute__((no_builtin("foo"))) or |
| 5055 |
// __attribute__((no_builtin)) on the current function unless foo is |
| 5056 |
// not a predefined library function which means we must generate the |
| 5057 |
// builtin no matter what. |
| 5058 |
else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin) |
| 5059 |
return CGCallee::forBuiltin(builtinID, FD); |
| 5060 |
} |
| 5061 |
|
| 5062 |
llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD); |
| 5063 |
if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice && |
| 5064 |
FD->hasAttr<CUDAGlobalAttr>()) |
| 5065 |
CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub( |
| 5066 |
cast<llvm::GlobalValue>(CalleePtr->stripPointerCasts())); |
| 5067 |
|
| 5068 |
return CGCallee::forDirect(CalleePtr, GD); |
| 5069 |
} |
| 5070 |
|
| 5071 |
CGCallee CodeGenFunction::EmitCallee(const Expr *E) { |
| 5072 |
E = E->IgnoreParens(); |
| 5073 |
|
| 5074 |
// Look through function-to-pointer decay. |
| 5075 |
if (auto ICE = dyn_cast<ImplicitCastExpr>(E)) { |
| 5076 |
if (ICE->getCastKind() == CK_FunctionToPointerDecay || |
| 5077 |
ICE->getCastKind() == CK_BuiltinFnToFnPtr) { |
| 5078 |
return EmitCallee(ICE->getSubExpr()); |
| 5079 |
} |
| 5080 |
|
| 5081 |
// Resolve direct calls. |
| 5082 |
} else if (auto DRE = dyn_cast<DeclRefExpr>(E)) { |
| 5083 |
if (auto FD = dyn_cast<FunctionDecl>(DRE->getDecl())) { |
| 5084 |
return EmitDirectCallee(*this, FD); |
| 5085 |
} |
| 5086 |
} else if (auto ME = dyn_cast<MemberExpr>(E)) { |
| 5087 |
if (auto FD = dyn_cast<FunctionDecl>(ME->getMemberDecl())) { |
| 5088 |
EmitIgnoredExpr(ME->getBase()); |
| 5089 |
return EmitDirectCallee(*this, FD); |
| 5090 |
} |
| 5091 |
|
| 5092 |
// Look through template substitutions. |
| 5093 |
} else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) { |
| 5094 |
return EmitCallee(NTTP->getReplacement()); |
| 5095 |
|
| 5096 |
// Treat pseudo-destructor calls differently. |
| 5097 |
} else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(E)) { |
| 5098 |
return CGCallee::forPseudoDestructor(PDE); |
| 5099 |
} |
| 5100 |
|
| 5101 |
// Otherwise, we have an indirect reference. |
| 5102 |
llvm::Value *calleePtr; |
| 5103 |
QualType functionType; |
| 5104 |
if (auto ptrType = E->getType()->getAs<PointerType>()) { |
| 5105 |
calleePtr = EmitScalarExpr(E); |
| 5106 |
functionType = ptrType->getPointeeType(); |
| 5107 |
} else { |
| 5108 |
functionType = E->getType(); |
| 5109 |
calleePtr = EmitLValue(E).getPointer(*this); |
| 5110 |
} |
| 5111 |
assert(functionType->isFunctionType()); |
| 5112 |
|
| 5113 |
GlobalDecl GD; |
| 5114 |
if (const auto *VD = |
| 5115 |
dyn_cast_or_null<VarDecl>(E->getReferencedDeclOfCallee())) |
| 5116 |
GD = GlobalDecl(VD); |
| 5117 |
|
| 5118 |
CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD); |
| 5119 |
CGCallee callee(calleeInfo, calleePtr); |
| 5120 |
return callee; |
| 5121 |
} |
| 5122 |
|
| 5123 |
LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) { |
| 5124 |
// Comma expressions just emit their LHS then their RHS as an l-value. |
| 5125 |
if (E->getOpcode() == BO_Comma) { |
| 5126 |
EmitIgnoredExpr(E->getLHS()); |
| 5127 |
EnsureInsertPoint(); |
| 5128 |
return EmitLValue(E->getRHS()); |
| 5129 |
} |
| 5130 |
|
| 5131 |
if (E->getOpcode() == BO_PtrMemD || |
| 5132 |
E->getOpcode() == BO_PtrMemI) |
| 5133 |
return EmitPointerToDataMemberBinaryExpr(E); |
| 5134 |
|
| 5135 |
assert(E->getOpcode() == BO_Assign && "unexpected binary l-value"); |
| 5136 |
|
| 5137 |
// Note that in all of these cases, __block variables need the RHS |
| 5138 |
// evaluated first just in case the variable gets moved by the RHS. |
| 5139 |
|
| 5140 |
switch (getEvaluationKind(E->getType())) { |
| 5141 |
case TEK_Scalar: { |
| 5142 |
switch (E->getLHS()->getType().getObjCLifetime()) { |
| 5143 |
case Qualifiers::OCL_Strong: |
| 5144 |
return EmitARCStoreStrong(E, /*ignored*/ false).first; |
| 5145 |
|
| 5146 |
case Qualifiers::OCL_Autoreleasing: |
| 5147 |
return EmitARCStoreAutoreleasing(E).first; |
| 5148 |
|
| 5149 |
// No reason to do any of these differently. |
| 5150 |
case Qualifiers::OCL_None: |
| 5151 |
case Qualifiers::OCL_ExplicitNone: |
| 5152 |
case Qualifiers::OCL_Weak: |
| 5153 |
break; |
| 5154 |
} |
| 5155 |
|
| 5156 |
RValue RV = EmitAnyExpr(E->getRHS()); |
| 5157 |
LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store); |
| 5158 |
if (RV.isScalar()) |
| 5159 |
EmitNullabilityCheck(LV, RV.getScalarVal(), E->getExprLoc()); |
| 5160 |
EmitStoreThroughLValue(RV, LV); |
| 5161 |
if (getLangOpts().OpenMP) |
| 5162 |
CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this, |
| 5163 |
E->getLHS()); |
| 5164 |
return LV; |
| 5165 |
} |
| 5166 |
|
| 5167 |
case TEK_Complex: |
| 5168 |
return EmitComplexAssignmentLValue(E); |
| 5169 |
|
| 5170 |
case TEK_Aggregate: |
| 5171 |
return EmitAggExprToLValue(E); |
| 5172 |
} |
| 5173 |
llvm_unreachable("bad evaluation kind"); |
| 5174 |
} |
| 5175 |
|
| 5176 |
LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) { |
| 5177 |
RValue RV = EmitCallExpr(E); |
| 5178 |
|
| 5179 |
if (!RV.isScalar()) |
| 5180 |
return MakeAddrLValue(RV.getAggregateAddress(), E->getType(), |
| 5181 |
AlignmentSource::Decl); |
| 5182 |
|
| 5183 |
assert(E->getCallReturnType(getContext())->isReferenceType() && |
| 5184 |
"Can't have a scalar return unless the return type is a " |
| 5185 |
"reference type!"); |
| 5186 |
|
| 5187 |
return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType()); |
| 5188 |
} |
| 5189 |
|
| 5190 |
LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) { |
| 5191 |
// FIXME: This shouldn't require another copy. |
| 5192 |
return EmitAggExprToLValue(E); |
| 5193 |
} |
| 5194 |
|
| 5195 |
LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) { |
| 5196 |
assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor() |
| 5197 |
&& "binding l-value to type which needs a temporary"); |
| 5198 |
AggValueSlot Slot = CreateAggTemp(E->getType()); |
| 5199 |
EmitCXXConstructExpr(E, Slot); |
| 5200 |
return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl); |
| 5201 |
} |
| 5202 |
|
| 5203 |
LValue |
| 5204 |
CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) { |
| 5205 |
return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType()); |
| 5206 |
} |
| 5207 |
|
| 5208 |
Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) { |
| 5209 |
return Builder.CreateElementBitCast(CGM.GetAddrOfMSGuidDecl(E->getGuidDecl()), |
| 5210 |
ConvertType(E->getType())); |
| 5211 |
} |
| 5212 |
|
| 5213 |
LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) { |
| 5214 |
return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(), |
| 5215 |
AlignmentSource::Decl); |
| 5216 |
} |
| 5217 |
|
| 5218 |
LValue |
| 5219 |
CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) { |
| 5220 |
AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue"); |
| 5221 |
Slot.setExternallyDestructed(); |
| 5222 |
EmitAggExpr(E->getSubExpr(), Slot); |
| 5223 |
EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddress()); |
| 5224 |
return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl); |
| 5225 |
} |
| 5226 |
|
| 5227 |
LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) { |
| 5228 |
RValue RV = EmitObjCMessageExpr(E); |
| 5229 |
|
| 5230 |
if (!RV.isScalar()) |
| 5231 |
return MakeAddrLValue(RV.getAggregateAddress(), E->getType(), |
| 5232 |
AlignmentSource::Decl); |
| 5233 |
|
| 5234 |
assert(E->getMethodDecl()->getReturnType()->isReferenceType() && |
| 5235 |
"Can't have a scalar return unless the return type is a " |
| 5236 |
"reference type!"); |
| 5237 |
|
| 5238 |
return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType()); |
| 5239 |
} |
| 5240 |
|
| 5241 |
LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) { |
| 5242 |
Address V = |
| 5243 |
CGM.getObjCRuntime().GetAddrOfSelector(*this, E->getSelector()); |
| 5244 |
return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl); |
| 5245 |
} |
| 5246 |
|
| 5247 |
llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface, |
| 5248 |
const ObjCIvarDecl *Ivar) { |
| 5249 |
return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar); |
| 5250 |
} |
| 5251 |
|
| 5252 |
llvm::Value * |
| 5253 |
CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface, |
| 5254 |
const ObjCIvarDecl *Ivar) { |
| 5255 |
llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar); |
| 5256 |
QualType PointerDiffType = getContext().getPointerDiffType(); |
| 5257 |
return Builder.CreateZExtOrTrunc(OffsetValue, |
| 5258 |
getTypes().ConvertType(PointerDiffType)); |
| 5259 |
} |
| 5260 |
|
| 5261 |
LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy, |
| 5262 |
llvm::Value *BaseValue, |
| 5263 |
const ObjCIvarDecl *Ivar, |
| 5264 |
unsigned CVRQualifiers) { |
| 5265 |
return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue, |
| 5266 |
Ivar, CVRQualifiers); |
| 5267 |
} |
| 5268 |
|
| 5269 |
LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) { |
| 5270 |
// FIXME: A lot of the code below could be shared with EmitMemberExpr. |
| 5271 |
llvm::Value *BaseValue = nullptr; |
| 5272 |
const Expr *BaseExpr = E->getBase(); |
| 5273 |
Qualifiers BaseQuals; |
| 5274 |
QualType ObjectTy; |
| 5275 |
if (E->isArrow()) { |
| 5276 |
BaseValue = EmitScalarExpr(BaseExpr); |
| 5277 |
ObjectTy = BaseExpr->getType()->getPointeeType(); |
| 5278 |
BaseQuals = ObjectTy.getQualifiers(); |
| 5279 |
} else { |
| 5280 |
LValue BaseLV = EmitLValue(BaseExpr); |
| 5281 |
BaseValue = BaseLV.getPointer(*this); |
| 5282 |
ObjectTy = BaseExpr->getType(); |
| 5283 |
BaseQuals = ObjectTy.getQualifiers(); |
| 5284 |
} |
| 5285 |
|
| 5286 |
LValue LV = |
| 5287 |
EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(), |
| 5288 |
BaseQuals.getCVRQualifiers()); |
| 5289 |
setObjCGCLValueClass(getContext(), E, LV); |
| 5290 |
return LV; |
| 5291 |
} |
| 5292 |
|
| 5293 |
LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) { |
| 5294 |
// Can only get l-value for message expression returning aggregate type |
| 5295 |
RValue RV = EmitAnyExprToTemp(E); |
| 5296 |
return MakeAddrLValue(RV.getAggregateAddress(), E->getType(), |
| 5297 |
AlignmentSource::Decl); |
| 5298 |
} |
| 5299 |
|
| 5300 |
RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee, |
| 5301 |
const CallExpr *E, ReturnValueSlot ReturnValue, |
| 5302 |
llvm::Value *Chain) { |
| 5303 |
// Get the actual function type. The callee type will always be a pointer to |
| 5304 |
// function type or a block pointer type. |
| 5305 |
assert(CalleeType->isFunctionPointerType() && |
| 5306 |
"Call must have function pointer type!"); |
| 5307 |
|
| 5308 |
const Decl *TargetDecl = |
| 5309 |
OrigCallee.getAbstractInfo().getCalleeDecl().getDecl(); |
| 5310 |
|
| 5311 |
CalleeType = getContext().getCanonicalType(CalleeType); |
| 5312 |
|
| 5313 |
auto PointeeType = cast<PointerType>(CalleeType)->getPointeeType(); |
| 5314 |
|
| 5315 |
CGCallee Callee = OrigCallee; |
| 5316 |
|
| 5317 |
if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function) && |
| 5318 |
(!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { |
| 5319 |
if (llvm::Constant *PrefixSig = |
| 5320 |
CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) { |
| 5321 |
SanitizerScope SanScope(this); |
| 5322 |
// Remove any (C++17) exception specifications, to allow calling e.g. a |
| 5323 |
// noexcept function through a non-noexcept pointer. |
| 5324 |
auto ProtoTy = |
| 5325 |
getContext().getFunctionTypeWithExceptionSpec(PointeeType, EST_None); |
| 5326 |
llvm::Constant *FTRTTIConst = |
| 5327 |
CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true); |
| 5328 |
llvm::Type *PrefixSigType = PrefixSig->getType(); |
| 5329 |
llvm::StructType *PrefixStructTy = llvm::StructType::get( |
| 5330 |
CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true); |
| 5331 |
|
| 5332 |
llvm::Value *CalleePtr = Callee.getFunctionPointer(); |
| 5333 |
|
| 5334 |
llvm::Value *CalleePrefixStruct = Builder.CreateBitCast( |
| 5335 |
CalleePtr, llvm::PointerType::getUnqual(PrefixStructTy)); |
| 5336 |
llvm::Value *CalleeSigPtr = |
| 5337 |
Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0); |
| 5338 |
llvm::Value *CalleeSig = |
| 5339 |
Builder.CreateAlignedLoad(PrefixSigType, CalleeSigPtr, getIntAlign()); |
| 5340 |
llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig); |
| 5341 |
|
| 5342 |
llvm::BasicBlock *Cont = createBasicBlock("cont"); |
| 5343 |
llvm::BasicBlock *TypeCheck = createBasicBlock("typecheck"); |
| 5344 |
Builder.CreateCondBr(CalleeSigMatch, TypeCheck, Cont); |
| 5345 |
|
| 5346 |
EmitBlock(TypeCheck); |
| 5347 |
llvm::Value *CalleeRTTIPtr = |
| 5348 |
Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1); |
| 5349 |
llvm::Value *CalleeRTTIEncoded = |
| 5350 |
Builder.CreateAlignedLoad(Int32Ty, CalleeRTTIPtr, getPointerAlign()); |
| 5351 |
llvm::Value *CalleeRTTI = |
| 5352 |
DecodeAddrUsedInPrologue(CalleePtr, CalleeRTTIEncoded); |
| 5353 |
llvm::Value *CalleeRTTIMatch = |
| 5354 |
Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst); |
| 5355 |
llvm::Constant *StaticData[] = {EmitCheckSourceLocation(E->getBeginLoc()), |
| 5356 |
EmitCheckTypeDescriptor(CalleeType)}; |
| 5357 |
EmitCheck(std::make_pair(CalleeRTTIMatch, SanitizerKind::Function), |
| 5358 |
SanitizerHandler::FunctionTypeMismatch, StaticData, |
| 5359 |
{CalleePtr, CalleeRTTI, FTRTTIConst}); |
| 5360 |
|
| 5361 |
Builder.CreateBr(Cont); |
| 5362 |
EmitBlock(Cont); |
| 5363 |
} |
| 5364 |
} |
| 5365 |
|
| 5366 |
const auto *FnType = cast<FunctionType>(PointeeType); |
| 5367 |
|
| 5368 |
// If we are checking indirect calls and this call is indirect, check that the |
| 5369 |
// function pointer is a member of the bit set for the function type. |
| 5370 |
if (SanOpts.has(SanitizerKind::CFIICall) && |
| 5371 |
(!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { |
| 5372 |
SanitizerScope SanScope(this); |
| 5373 |
EmitSanitizerStatReport(llvm::SanStat_CFI_ICall); |
| 5374 |
|
| 5375 |
llvm::Metadata *MD; |
| 5376 |
if (CGM.getCodeGenOpts().SanitizeCfiICallGeneralizePointers) |
| 5377 |
MD = CGM.CreateMetadataIdentifierGeneralized(QualType(FnType, 0)); |
| 5378 |
else |
| 5379 |
MD = CGM.CreateMetadataIdentifierForType(QualType(FnType, 0)); |
| 5380 |
|
| 5381 |
llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD); |
| 5382 |
|
| 5383 |
llvm::Value *CalleePtr = Callee.getFunctionPointer(); |
| 5384 |
llvm::Value *CastedCallee = Builder.CreateBitCast(CalleePtr, Int8PtrTy); |
| 5385 |
llvm::Value *TypeTest = Builder.CreateCall( |
| 5386 |
CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedCallee, TypeId}); |
| 5387 |
|
| 5388 |
auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD); |
| 5389 |
llvm::Constant *StaticData[] = { |
| 5390 |
llvm::ConstantInt::get(Int8Ty, CFITCK_ICall), |
| 5391 |
EmitCheckSourceLocation(E->getBeginLoc()), |
| 5392 |
EmitCheckTypeDescriptor(QualType(FnType, 0)), |
| 5393 |
}; |
| 5394 |
if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) { |
| 5395 |
EmitCfiSlowPathCheck(SanitizerKind::CFIICall, TypeTest, CrossDsoTypeId, |
| 5396 |
CastedCallee, StaticData); |
| 5397 |
} else { |
| 5398 |
EmitCheck(std::make_pair(TypeTest, SanitizerKind::CFIICall), |
| 5399 |
SanitizerHandler::CFICheckFail, StaticData, |
| 5400 |
{CastedCallee, llvm::UndefValue::get(IntPtrTy)}); |
| 5401 |
} |
| 5402 |
} |
| 5403 |
|
| 5404 |
CallArgList Args; |
| 5405 |
if (Chain) |
| 5406 |
Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)), |
| 5407 |
CGM.getContext().VoidPtrTy); |
| 5408 |
|
| 5409 |
// C++17 requires that we evaluate arguments to a call using assignment syntax |
| 5410 |
// right-to-left, and that we evaluate arguments to certain other operators |
| 5411 |
// left-to-right. Note that we allow this to override the order dictated by |
| 5412 |
// the calling convention on the MS ABI, which means that parameter |
| 5413 |
// destruction order is not necessarily reverse construction order. |
| 5414 |
// FIXME: Revisit this based on C++ committee response to unimplementability. |
| 5415 |
EvaluationOrder Order = EvaluationOrder::Default; |
| 5416 |
if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) { |
| 5417 |
if (OCE->isAssignmentOp()) |
| 5418 |
Order = EvaluationOrder::ForceRightToLeft; |
| 5419 |
else { |
| 5420 |
switch (OCE->getOperator()) { |
| 5421 |
case OO_LessLess: |
| 5422 |
case OO_GreaterGreater: |
| 5423 |
case OO_AmpAmp: |
| 5424 |
case OO_PipePipe: |
| 5425 |
case OO_Comma: |
| 5426 |
case OO_ArrowStar: |
| 5427 |
Order = EvaluationOrder::ForceLeftToRight; |
| 5428 |
break; |
| 5429 |
default: |
| 5430 |
break; |
| 5431 |
} |
| 5432 |
} |
| 5433 |
} |
| 5434 |
|
| 5435 |
EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(), |
| 5436 |
E->getDirectCallee(), /*ParamsToSkip*/ 0, Order); |
| 5437 |
|
| 5438 |
const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall( |
| 5439 |
Args, FnType, /*ChainCall=*/Chain); |
| 5440 |
|
| 5441 |
// C99 6.5.2.2p6: |
| 5442 |
// If the expression that denotes the called function has a type |
| 5443 |
// that does not include a prototype, [the default argument |
| 5444 |
// promotions are performed]. If the number of arguments does not |
| 5445 |
// equal the number of parameters, the behavior is undefined. If |
| 5446 |
// the function is defined with a type that includes a prototype, |
| 5447 |
// and either the prototype ends with an ellipsis (, ...) or the |
| 5448 |
// types of the arguments after promotion are not compatible with |
| 5449 |
// the types of the parameters, the behavior is undefined. If the |
| 5450 |
// function is defined with a type that does not include a |
| 5451 |
// prototype, and the types of the arguments after promotion are |
| 5452 |
// not compatible with those of the parameters after promotion, |
| 5453 |
// the behavior is undefined [except in some trivial cases]. |
| 5454 |
// That is, in the general case, we should assume that a call |
| 5455 |
// through an unprototyped function type works like a *non-variadic* |
| 5456 |
// call. The way we make this work is to cast to the exact type |
| 5457 |
// of the promoted arguments. |
| 5458 |
// |
| 5459 |
// Chain calls use this same code path to add the invisible chain parameter |
| 5460 |
// to the function type. |
| 5461 |
if (isa<FunctionNoProtoType>(FnType) || Chain) { |
| 5462 |
llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo); |
| 5463 |
int AS = Callee.getFunctionPointer()->getType()->getPointerAddressSpace(); |
| 5464 |
CalleeTy = CalleeTy->getPointerTo(AS); |
| 5465 |
|
| 5466 |
llvm::Value *CalleePtr = Callee.getFunctionPointer(); |
| 5467 |
CalleePtr = Builder.CreateBitCast(CalleePtr, CalleeTy, "callee.knr.cast"); |
| 5468 |
Callee.setFunctionPointer(CalleePtr); |
| 5469 |
} |
| 5470 |
|
| 5471 |
// HIP function pointer contains kernel handle when it is used in triple |
| 5472 |
// chevron. The kernel stub needs to be loaded from kernel handle and used |
| 5473 |
// as callee. |
| 5474 |
if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice && |
| 5475 |
isa<CUDAKernelCallExpr>(E) && |
| 5476 |
(!TargetDecl || !isa<FunctionDecl>(TargetDecl))) { |
| 5477 |
llvm::Value *Handle = Callee.getFunctionPointer(); |
| 5478 |
auto *Cast = |
| 5479 |
Builder.CreateBitCast(Handle, Handle->getType()->getPointerTo()); |
| 5480 |
auto *Stub = Builder.CreateLoad( |
| 5481 |
Address(Cast, Handle->getType(), CGM.getPointerAlign())); |
| 5482 |
Callee.setFunctionPointer(Stub); |
| 5483 |
} |
| 5484 |
llvm::CallBase *CallOrInvoke = nullptr; |
| 5485 |
RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &CallOrInvoke, |
| 5486 |
E == MustTailCall, E->getExprLoc()); |
| 5487 |
|
| 5488 |
// Generate function declaration DISuprogram in order to be used |
| 5489 |
// in debug info about call sites. |
| 5490 |
if (CGDebugInfo *DI = getDebugInfo()) { |
| 5491 |
if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) { |
| 5492 |
FunctionArgList Args; |
| 5493 |
QualType ResTy = BuildFunctionArgList(CalleeDecl, Args); |
| 5494 |
DI->EmitFuncDeclForCallSite(CallOrInvoke, |
| 5495 |
DI->getFunctionType(CalleeDecl, ResTy, Args), |
| 5496 |
CalleeDecl); |
| 5497 |
} |
| 5498 |
} |
| 5499 |
|
| 5500 |
return Call; |
| 5501 |
} |
| 5502 |
|
| 5503 |
LValue CodeGenFunction:: |
| 5504 |
EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) { |
| 5505 |
Address BaseAddr = Address::invalid(); |
| 5506 |
if (E->getOpcode() == BO_PtrMemI) { |
| 5507 |
BaseAddr = EmitPointerWithAlignment(E->getLHS()); |
| 5508 |
} else { |
| 5509 |
BaseAddr = EmitLValue(E->getLHS()).getAddress(*this); |
| 5510 |
} |
| 5511 |
|
| 5512 |
llvm::Value *OffsetV = EmitScalarExpr(E->getRHS()); |
| 5513 |
const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>(); |
| 5514 |
|
| 5515 |
LValueBaseInfo BaseInfo; |
| 5516 |
TBAAAccessInfo TBAAInfo; |
| 5517 |
Address MemberAddr = |
| 5518 |
EmitCXXMemberDataPointerAddress(E, BaseAddr, OffsetV, MPT, &BaseInfo, |
| 5519 |
&TBAAInfo); |
| 5520 |
|
| 5521 |
return MakeAddrLValue(MemberAddr, MPT->getPointeeType(), BaseInfo, TBAAInfo); |
| 5522 |
} |
| 5523 |
|
| 5524 |
/// Given the address of a temporary variable, produce an r-value of |
| 5525 |
/// its type. |
| 5526 |
RValue CodeGenFunction::convertTempToRValue(Address addr, |
| 5527 |
QualType type, |
| 5528 |
SourceLocation loc) { |
| 5529 |
LValue lvalue = MakeAddrLValue(addr, type, AlignmentSource::Decl); |
| 5530 |
switch (getEvaluationKind(type)) { |
| 5531 |
case TEK_Complex: |
| 5532 |
return RValue::getComplex(EmitLoadOfComplex(lvalue, loc)); |
| 5533 |
case TEK_Aggregate: |
| 5534 |
return lvalue.asAggregateRValue(*this); |
| 5535 |
case TEK_Scalar: |
| 5536 |
return RValue::get(EmitLoadOfScalar(lvalue, loc)); |
| 5537 |
} |
| 5538 |
llvm_unreachable("bad evaluation kind"); |
| 5539 |
} |
| 5540 |
|
| 5541 |
void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) { |
| 5542 |
assert(Val->getType()->isFPOrFPVectorTy()); |
| 5543 |
if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val)) |
| 5544 |
return; |
| 5545 |
|
| 5546 |
llvm::MDBuilder MDHelper(getLLVMContext()); |
| 5547 |
llvm::MDNode *Node = MDHelper.createFPMath(Accuracy); |
| 5548 |
|
| 5549 |
cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node); |
| 5550 |
} |
| 5551 |
|
| 5552 |
namespace { |
| 5553 |
struct LValueOrRValue { |
| 5554 |
LValue LV; |
| 5555 |
RValue RV; |
| 5556 |
}; |
| 5557 |
} |
| 5558 |
|
| 5559 |
static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF, |
| 5560 |
const PseudoObjectExpr *E, |
| 5561 |
bool forLValue, |
| 5562 |
AggValueSlot slot) { |
| 5563 |
SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques; |
| 5564 |
|
| 5565 |
// Find the result expression, if any. |
| 5566 |
const Expr *resultExpr = E->getResultExpr(); |
| 5567 |
LValueOrRValue result; |
| 5568 |
|
| 5569 |
for (PseudoObjectExpr::const_semantics_iterator |
| 5570 |
i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) { |
| 5571 |
const Expr *semantic = *i; |
| 5572 |
|
| 5573 |
// If this semantic expression is an opaque value, bind it |
| 5574 |
// to the result of its source expression. |
| 5575 |
if (const auto *ov = dyn_cast<OpaqueValueExpr>(semantic)) { |
| 5576 |
// Skip unique OVEs. |
| 5577 |
if (ov->isUnique()) { |
| 5578 |
assert(ov != resultExpr && |
| 5579 |
"A unique OVE cannot be used as the result expression"); |
| 5580 |
continue; |
| 5581 |
} |
| 5582 |
|
| 5583 |
// If this is the result expression, we may need to evaluate |
| 5584 |
// directly into the slot. |
| 5585 |
typedef CodeGenFunction::OpaqueValueMappingData OVMA; |
| 5586 |
OVMA opaqueData; |
| 5587 |
if (ov == resultExpr && ov->isPRValue() && !forLValue && |
| 5588 |
CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) { |
| 5589 |
CGF.EmitAggExpr(ov->getSourceExpr(), slot); |
| 5590 |
LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(), |
| 5591 |
AlignmentSource::Decl); |
| 5592 |
opaqueData = OVMA::bind(CGF, ov, LV); |
| 5593 |
result.RV = slot.asRValue(); |
| 5594 |
|
| 5595 |
// Otherwise, emit as normal. |
| 5596 |
} else { |
| 5597 |
opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr()); |
| 5598 |
|
| 5599 |
// If this is the result, also evaluate the result now. |
| 5600 |
if (ov == resultExpr) { |
| 5601 |
if (forLValue) |
| 5602 |
result.LV = CGF.EmitLValue(ov); |
| 5603 |
else |
| 5604 |
result.RV = CGF.EmitAnyExpr(ov, slot); |
| 5605 |
} |
| 5606 |
} |
| 5607 |
|
| 5608 |
opaques.push_back(opaqueData); |
| 5609 |
|
| 5610 |
// Otherwise, if the expression is the result, evaluate it |
| 5611 |
// and remember the result. |
| 5612 |
} else if (semantic == resultExpr) { |
| 5613 |
if (forLValue) |
| 5614 |
result.LV = CGF.EmitLValue(semantic); |
| 5615 |
else |
| 5616 |
result.RV = CGF.EmitAnyExpr(semantic, slot); |
| 5617 |
|
| 5618 |
// Otherwise, evaluate the expression in an ignored context. |
| 5619 |
} else { |
| 5620 |
CGF.EmitIgnoredExpr(semantic); |
| 5621 |
} |
| 5622 |
} |
| 5623 |
|
| 5624 |
// Unbind all the opaques now. |
| 5625 |
for (unsigned i = 0, e = opaques.size(); i != e; ++i) |
| 5626 |
opaques[i].unbind(CGF); |
| 5627 |
|
| 5628 |
return result; |
| 5629 |
} |
| 5630 |
|
| 5631 |
RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E, |
| 5632 |
AggValueSlot slot) { |
| 5633 |
return emitPseudoObjectExpr(*this, E, false, slot).RV; |
| 5634 |
} |
| 5635 |
|
| 5636 |
LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) { |
| 5637 |
return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV; |
| 5638 |
} |