| 1 |
//===----- CGCoroutine.cpp - Emit LLVM Code for C++ coroutines ------------===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// This contains code dealing with C++ code generation of coroutines. |
| 10 |
// |
| 11 |
//===----------------------------------------------------------------------===// |
| 12 |
|
| 13 |
#include "CGCleanup.h" |
| 14 |
#include "CodeGenFunction.h" |
| 15 |
#include "llvm/ADT/ScopeExit.h" |
| 16 |
#include "clang/AST/StmtCXX.h" |
| 17 |
#include "clang/AST/StmtVisitor.h" |
| 18 |
|
| 19 |
using namespace clang; |
| 20 |
using namespace CodeGen; |
| 21 |
|
| 22 |
using llvm::Value; |
| 23 |
using llvm::BasicBlock; |
| 24 |
|
| 25 |
namespace { |
| 26 |
enum class AwaitKind { Init, Normal, Yield, Final }; |
| 27 |
static constexpr llvm::StringLiteral AwaitKindStr[] = {"init", "await", "yield", |
| 28 |
"final"}; |
| 29 |
} |
| 30 |
|
| 31 |
struct clang::CodeGen::CGCoroData { |
| 32 |
// What is the current await expression kind and how many |
| 33 |
// await/yield expressions were encountered so far. |
| 34 |
// These are used to generate pretty labels for await expressions in LLVM IR. |
| 35 |
AwaitKind CurrentAwaitKind = AwaitKind::Init; |
| 36 |
unsigned AwaitNum = 0; |
| 37 |
unsigned YieldNum = 0; |
| 38 |
|
| 39 |
// How many co_return statements are in the coroutine. Used to decide whether |
| 40 |
// we need to add co_return; equivalent at the end of the user authored body. |
| 41 |
unsigned CoreturnCount = 0; |
| 42 |
|
| 43 |
// A branch to this block is emitted when coroutine needs to suspend. |
| 44 |
llvm::BasicBlock *SuspendBB = nullptr; |
| 45 |
|
| 46 |
// The promise type's 'unhandled_exception' handler, if it defines one. |
| 47 |
Stmt *ExceptionHandler = nullptr; |
| 48 |
|
| 49 |
// A temporary i1 alloca that stores whether 'await_resume' threw an |
| 50 |
// exception. If it did, 'true' is stored in this variable, and the coroutine |
| 51 |
// body must be skipped. If the promise type does not define an exception |
| 52 |
// handler, this is null. |
| 53 |
llvm::Value *ResumeEHVar = nullptr; |
| 54 |
|
| 55 |
// Stores the jump destination just before the coroutine memory is freed. |
| 56 |
// This is the destination that every suspend point jumps to for the cleanup |
| 57 |
// branch. |
| 58 |
CodeGenFunction::JumpDest CleanupJD; |
| 59 |
|
| 60 |
// Stores the jump destination just before the final suspend. The co_return |
| 61 |
// statements jumps to this point after calling return_xxx promise member. |
| 62 |
CodeGenFunction::JumpDest FinalJD; |
| 63 |
|
| 64 |
// Stores the llvm.coro.id emitted in the function so that we can supply it |
| 65 |
// as the first argument to coro.begin, coro.alloc and coro.free intrinsics. |
| 66 |
// Note: llvm.coro.id returns a token that cannot be directly expressed in a |
| 67 |
// builtin. |
| 68 |
llvm::CallInst *CoroId = nullptr; |
| 69 |
|
| 70 |
// Stores the llvm.coro.begin emitted in the function so that we can replace |
| 71 |
// all coro.frame intrinsics with direct SSA value of coro.begin that returns |
| 72 |
// the address of the coroutine frame of the current coroutine. |
| 73 |
llvm::CallInst *CoroBegin = nullptr; |
| 74 |
|
| 75 |
// Stores the last emitted coro.free for the deallocate expressions, we use it |
| 76 |
// to wrap dealloc code with if(auto mem = coro.free) dealloc(mem). |
| 77 |
llvm::CallInst *LastCoroFree = nullptr; |
| 78 |
|
| 79 |
// If coro.id came from the builtin, remember the expression to give better |
| 80 |
// diagnostic. If CoroIdExpr is nullptr, the coro.id was created by |
| 81 |
// EmitCoroutineBody. |
| 82 |
CallExpr const *CoroIdExpr = nullptr; |
| 83 |
}; |
| 84 |
|
| 85 |
// Defining these here allows to keep CGCoroData private to this file. |
| 86 |
clang::CodeGen::CodeGenFunction::CGCoroInfo::CGCoroInfo() {} |
| 87 |
CodeGenFunction::CGCoroInfo::~CGCoroInfo() {} |
| 88 |
|
| 89 |
static void createCoroData(CodeGenFunction &CGF, |
| 90 |
CodeGenFunction::CGCoroInfo &CurCoro, |
| 91 |
llvm::CallInst *CoroId, |
| 92 |
CallExpr const *CoroIdExpr = nullptr) { |
| 93 |
if (CurCoro.Data) { |
| 94 |
if (CurCoro.Data->CoroIdExpr) |
| 95 |
CGF.CGM.Error(CoroIdExpr->getBeginLoc(), |
| 96 |
"only one __builtin_coro_id can be used in a function"); |
| 97 |
else if (CoroIdExpr) |
| 98 |
CGF.CGM.Error(CoroIdExpr->getBeginLoc(), |
| 99 |
"__builtin_coro_id shall not be used in a C++ coroutine"); |
| 100 |
else |
| 101 |
llvm_unreachable("EmitCoroutineBodyStatement called twice?"); |
| 102 |
|
| 103 |
return; |
| 104 |
} |
| 105 |
|
| 106 |
CurCoro.Data = std::unique_ptr<CGCoroData>(new CGCoroData); |
| 107 |
CurCoro.Data->CoroId = CoroId; |
| 108 |
CurCoro.Data->CoroIdExpr = CoroIdExpr; |
| 109 |
} |
| 110 |
|
| 111 |
// Synthesize a pretty name for a suspend point. |
| 112 |
static SmallString<32> buildSuspendPrefixStr(CGCoroData &Coro, AwaitKind Kind) { |
| 113 |
unsigned No = 0; |
| 114 |
switch (Kind) { |
| 115 |
case AwaitKind::Init: |
| 116 |
case AwaitKind::Final: |
| 117 |
break; |
| 118 |
case AwaitKind::Normal: |
| 119 |
No = ++Coro.AwaitNum; |
| 120 |
break; |
| 121 |
case AwaitKind::Yield: |
| 122 |
No = ++Coro.YieldNum; |
| 123 |
break; |
| 124 |
} |
| 125 |
SmallString<32> Prefix(AwaitKindStr[static_cast<unsigned>(Kind)]); |
| 126 |
if (No > 1) { |
| 127 |
Twine(No).toVector(Prefix); |
| 128 |
} |
| 129 |
return Prefix; |
| 130 |
} |
| 131 |
|
| 132 |
static bool memberCallExpressionCanThrow(const Expr *E) { |
| 133 |
if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E)) |
| 134 |
if (const auto *Proto = |
| 135 |
CE->getMethodDecl()->getType()->getAs<FunctionProtoType>()) |
| 136 |
if (isNoexceptExceptionSpec(Proto->getExceptionSpecType()) && |
| 137 |
Proto->canThrow() == CT_Cannot) |
| 138 |
return false; |
| 139 |
return true; |
| 140 |
} |
| 141 |
|
| 142 |
// Emit suspend expression which roughly looks like: |
| 143 |
// |
| 144 |
// auto && x = CommonExpr(); |
| 145 |
// if (!x.await_ready()) { |
| 146 |
// llvm_coro_save(); |
| 147 |
// x.await_suspend(...); (*) |
| 148 |
// llvm_coro_suspend(); (**) |
| 149 |
// } |
| 150 |
// x.await_resume(); |
| 151 |
// |
| 152 |
// where the result of the entire expression is the result of x.await_resume() |
| 153 |
// |
| 154 |
// (*) If x.await_suspend return type is bool, it allows to veto a suspend: |
| 155 |
// if (x.await_suspend(...)) |
| 156 |
// llvm_coro_suspend(); |
| 157 |
// |
| 158 |
// (**) llvm_coro_suspend() encodes three possible continuations as |
| 159 |
// a switch instruction: |
| 160 |
// |
| 161 |
// %where-to = call i8 @llvm.coro.suspend(...) |
| 162 |
// switch i8 %where-to, label %coro.ret [ ; jump to epilogue to suspend |
| 163 |
// i8 0, label %yield.ready ; go here when resumed |
| 164 |
// i8 1, label %yield.cleanup ; go here when destroyed |
| 165 |
// ] |
| 166 |
// |
| 167 |
// See llvm's docs/Coroutines.rst for more details. |
| 168 |
// |
| 169 |
namespace { |
| 170 |
struct LValueOrRValue { |
| 171 |
LValue LV; |
| 172 |
RValue RV; |
| 173 |
}; |
| 174 |
} |
| 175 |
static LValueOrRValue emitSuspendExpression(CodeGenFunction &CGF, CGCoroData &Coro, |
| 176 |
CoroutineSuspendExpr const &S, |
| 177 |
AwaitKind Kind, AggValueSlot aggSlot, |
| 178 |
bool ignoreResult, bool forLValue) { |
| 179 |
auto *E = S.getCommonExpr(); |
| 180 |
|
| 181 |
auto Binder = |
| 182 |
CodeGenFunction::OpaqueValueMappingData::bind(CGF, S.getOpaqueValue(), E); |
| 183 |
auto UnbindOnExit = llvm::make_scope_exit([&] { Binder.unbind(CGF); }); |
| 184 |
|
| 185 |
auto Prefix = buildSuspendPrefixStr(Coro, Kind); |
| 186 |
BasicBlock *ReadyBlock = CGF.createBasicBlock(Prefix + Twine(".ready")); |
| 187 |
BasicBlock *SuspendBlock = CGF.createBasicBlock(Prefix + Twine(".suspend")); |
| 188 |
BasicBlock *CleanupBlock = CGF.createBasicBlock(Prefix + Twine(".cleanup")); |
| 189 |
|
| 190 |
// If expression is ready, no need to suspend. |
| 191 |
CGF.EmitBranchOnBoolExpr(S.getReadyExpr(), ReadyBlock, SuspendBlock, 0); |
| 192 |
|
| 193 |
// Otherwise, emit suspend logic. |
| 194 |
CGF.EmitBlock(SuspendBlock); |
| 195 |
|
| 196 |
auto &Builder = CGF.Builder; |
| 197 |
llvm::Function *CoroSave = CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_save); |
| 198 |
auto *NullPtr = llvm::ConstantPointerNull::get(CGF.CGM.Int8PtrTy); |
| 199 |
auto *SaveCall = Builder.CreateCall(CoroSave, {NullPtr}); |
| 200 |
|
| 201 |
auto *SuspendRet = CGF.EmitScalarExpr(S.getSuspendExpr()); |
| 202 |
if (SuspendRet != nullptr && SuspendRet->getType()->isIntegerTy(1)) { |
| 203 |
// Veto suspension if requested by bool returning await_suspend. |
| 204 |
BasicBlock *RealSuspendBlock = |
| 205 |
CGF.createBasicBlock(Prefix + Twine(".suspend.bool")); |
| 206 |
CGF.Builder.CreateCondBr(SuspendRet, RealSuspendBlock, ReadyBlock); |
| 207 |
CGF.EmitBlock(RealSuspendBlock); |
| 208 |
} |
| 209 |
|
| 210 |
// Emit the suspend point. |
| 211 |
const bool IsFinalSuspend = (Kind == AwaitKind::Final); |
| 212 |
llvm::Function *CoroSuspend = |
| 213 |
CGF.CGM.getIntrinsic(llvm::Intrinsic::coro_suspend); |
| 214 |
auto *SuspendResult = Builder.CreateCall( |
| 215 |
CoroSuspend, {SaveCall, Builder.getInt1(IsFinalSuspend)}); |
| 216 |
|
| 217 |
// Create a switch capturing three possible continuations. |
| 218 |
auto *Switch = Builder.CreateSwitch(SuspendResult, Coro.SuspendBB, 2); |
| 219 |
Switch->addCase(Builder.getInt8(0), ReadyBlock); |
| 220 |
Switch->addCase(Builder.getInt8(1), CleanupBlock); |
| 221 |
|
| 222 |
// Emit cleanup for this suspend point. |
| 223 |
CGF.EmitBlock(CleanupBlock); |
| 224 |
CGF.EmitBranchThroughCleanup(Coro.CleanupJD); |
| 225 |
|
| 226 |
// Emit await_resume expression. |
| 227 |
CGF.EmitBlock(ReadyBlock); |
| 228 |
|
| 229 |
// Exception handling requires additional IR. If the 'await_resume' function |
| 230 |
// is marked as 'noexcept', we avoid generating this additional IR. |
| 231 |
CXXTryStmt *TryStmt = nullptr; |
| 232 |
if (Coro.ExceptionHandler && Kind == AwaitKind::Init && |
| 233 |
memberCallExpressionCanThrow(S.getResumeExpr())) { |
| 234 |
Coro.ResumeEHVar = |
| 235 |
CGF.CreateTempAlloca(Builder.getInt1Ty(), Prefix + Twine("resume.eh")); |
| 236 |
Builder.CreateFlagStore(true, Coro.ResumeEHVar); |
| 237 |
|
| 238 |
auto Loc = S.getResumeExpr()->getExprLoc(); |
| 239 |
auto *Catch = new (CGF.getContext()) |
| 240 |
CXXCatchStmt(Loc, /*exDecl=*/nullptr, Coro.ExceptionHandler); |
| 241 |
auto *TryBody = CompoundStmt::Create(CGF.getContext(), S.getResumeExpr(), |
| 242 |
FPOptionsOverride(), Loc, Loc); |
| 243 |
TryStmt = CXXTryStmt::Create(CGF.getContext(), Loc, TryBody, Catch); |
| 244 |
CGF.EnterCXXTryStmt(*TryStmt); |
| 245 |
} |
| 246 |
|
| 247 |
LValueOrRValue Res; |
| 248 |
if (forLValue) |
| 249 |
Res.LV = CGF.EmitLValue(S.getResumeExpr()); |
| 250 |
else |
| 251 |
Res.RV = CGF.EmitAnyExpr(S.getResumeExpr(), aggSlot, ignoreResult); |
| 252 |
|
| 253 |
if (TryStmt) { |
| 254 |
Builder.CreateFlagStore(false, Coro.ResumeEHVar); |
| 255 |
CGF.ExitCXXTryStmt(*TryStmt); |
| 256 |
} |
| 257 |
|
| 258 |
return Res; |
| 259 |
} |
| 260 |
|
| 261 |
RValue CodeGenFunction::EmitCoawaitExpr(const CoawaitExpr &E, |
| 262 |
AggValueSlot aggSlot, |
| 263 |
bool ignoreResult) { |
| 264 |
return emitSuspendExpression(*this, *CurCoro.Data, E, |
| 265 |
CurCoro.Data->CurrentAwaitKind, aggSlot, |
| 266 |
ignoreResult, /*forLValue*/false).RV; |
| 267 |
} |
| 268 |
RValue CodeGenFunction::EmitCoyieldExpr(const CoyieldExpr &E, |
| 269 |
AggValueSlot aggSlot, |
| 270 |
bool ignoreResult) { |
| 271 |
return emitSuspendExpression(*this, *CurCoro.Data, E, AwaitKind::Yield, |
| 272 |
aggSlot, ignoreResult, /*forLValue*/false).RV; |
| 273 |
} |
| 274 |
|
| 275 |
void CodeGenFunction::EmitCoreturnStmt(CoreturnStmt const &S) { |
| 276 |
++CurCoro.Data->CoreturnCount; |
| 277 |
const Expr *RV = S.getOperand(); |
| 278 |
if (RV && RV->getType()->isVoidType() && !isa<InitListExpr>(RV)) { |
| 279 |
// Make sure to evaluate the non initlist expression of a co_return |
| 280 |
// with a void expression for side effects. |
| 281 |
RunCleanupsScope cleanupScope(*this); |
| 282 |
EmitIgnoredExpr(RV); |
| 283 |
} |
| 284 |
EmitStmt(S.getPromiseCall()); |
| 285 |
EmitBranchThroughCleanup(CurCoro.Data->FinalJD); |
| 286 |
} |
| 287 |
|
| 288 |
|
| 289 |
#ifndef NDEBUG |
| 290 |
static QualType getCoroutineSuspendExprReturnType(const ASTContext &Ctx, |
| 291 |
const CoroutineSuspendExpr *E) { |
| 292 |
const auto *RE = E->getResumeExpr(); |
| 293 |
// Is it possible for RE to be a CXXBindTemporaryExpr wrapping |
| 294 |
// a MemberCallExpr? |
| 295 |
assert(isa<CallExpr>(RE) && "unexpected suspend expression type"); |
| 296 |
return cast<CallExpr>(RE)->getCallReturnType(Ctx); |
| 297 |
} |
| 298 |
#endif |
| 299 |
|
| 300 |
LValue |
| 301 |
CodeGenFunction::EmitCoawaitLValue(const CoawaitExpr *E) { |
| 302 |
assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && |
| 303 |
"Can't have a scalar return unless the return type is a " |
| 304 |
"reference type!"); |
| 305 |
return emitSuspendExpression(*this, *CurCoro.Data, *E, |
| 306 |
CurCoro.Data->CurrentAwaitKind, AggValueSlot::ignored(), |
| 307 |
/*ignoreResult*/false, /*forLValue*/true).LV; |
| 308 |
} |
| 309 |
|
| 310 |
LValue |
| 311 |
CodeGenFunction::EmitCoyieldLValue(const CoyieldExpr *E) { |
| 312 |
assert(getCoroutineSuspendExprReturnType(getContext(), E)->isReferenceType() && |
| 313 |
"Can't have a scalar return unless the return type is a " |
| 314 |
"reference type!"); |
| 315 |
return emitSuspendExpression(*this, *CurCoro.Data, *E, |
| 316 |
AwaitKind::Yield, AggValueSlot::ignored(), |
| 317 |
/*ignoreResult*/false, /*forLValue*/true).LV; |
| 318 |
} |
| 319 |
|
| 320 |
// Hunts for the parameter reference in the parameter copy/move declaration. |
| 321 |
namespace { |
| 322 |
struct GetParamRef : public StmtVisitor<GetParamRef> { |
| 323 |
public: |
| 324 |
DeclRefExpr *Expr = nullptr; |
| 325 |
GetParamRef() {} |
| 326 |
void VisitDeclRefExpr(DeclRefExpr *E) { |
| 327 |
assert(Expr == nullptr && "multilple declref in param move"); |
| 328 |
Expr = E; |
| 329 |
} |
| 330 |
void VisitStmt(Stmt *S) { |
| 331 |
for (auto *C : S->children()) { |
| 332 |
if (C) |
| 333 |
Visit(C); |
| 334 |
} |
| 335 |
} |
| 336 |
}; |
| 337 |
} |
| 338 |
|
| 339 |
// This class replaces references to parameters to their copies by changing |
| 340 |
// the addresses in CGF.LocalDeclMap and restoring back the original values in |
| 341 |
// its destructor. |
| 342 |
|
| 343 |
namespace { |
| 344 |
struct ParamReferenceReplacerRAII { |
| 345 |
CodeGenFunction::DeclMapTy SavedLocals; |
| 346 |
CodeGenFunction::DeclMapTy& LocalDeclMap; |
| 347 |
|
| 348 |
ParamReferenceReplacerRAII(CodeGenFunction::DeclMapTy &LocalDeclMap) |
| 349 |
: LocalDeclMap(LocalDeclMap) {} |
| 350 |
|
| 351 |
void addCopy(DeclStmt const *PM) { |
| 352 |
// Figure out what param it refers to. |
| 353 |
|
| 354 |
assert(PM->isSingleDecl()); |
| 355 |
VarDecl const*VD = static_cast<VarDecl const*>(PM->getSingleDecl()); |
| 356 |
Expr const *InitExpr = VD->getInit(); |
| 357 |
GetParamRef Visitor; |
| 358 |
Visitor.Visit(const_cast<Expr*>(InitExpr)); |
| 359 |
assert(Visitor.Expr); |
| 360 |
DeclRefExpr *DREOrig = Visitor.Expr; |
| 361 |
auto *PD = DREOrig->getDecl(); |
| 362 |
|
| 363 |
auto it = LocalDeclMap.find(PD); |
| 364 |
assert(it != LocalDeclMap.end() && "parameter is not found"); |
| 365 |
SavedLocals.insert({ PD, it->second }); |
| 366 |
|
| 367 |
auto copyIt = LocalDeclMap.find(VD); |
| 368 |
assert(copyIt != LocalDeclMap.end() && "parameter copy is not found"); |
| 369 |
it->second = copyIt->getSecond(); |
| 370 |
} |
| 371 |
|
| 372 |
~ParamReferenceReplacerRAII() { |
| 373 |
for (auto&& SavedLocal : SavedLocals) { |
| 374 |
LocalDeclMap.insert({SavedLocal.first, SavedLocal.second}); |
| 375 |
} |
| 376 |
} |
| 377 |
}; |
| 378 |
} |
| 379 |
|
| 380 |
// For WinEH exception representation backend needs to know what funclet coro.end |
| 381 |
// belongs to. That information is passed in a funclet bundle. |
| 382 |
static SmallVector<llvm::OperandBundleDef, 1> |
| 383 |
getBundlesForCoroEnd(CodeGenFunction &CGF) { |
| 384 |
SmallVector<llvm::OperandBundleDef, 1> BundleList; |
| 385 |
|
| 386 |
if (llvm::Instruction *EHPad = CGF.CurrentFuncletPad) |
| 387 |
BundleList.emplace_back("funclet", EHPad); |
| 388 |
|
| 389 |
return BundleList; |
| 390 |
} |
| 391 |
|
| 392 |
namespace { |
| 393 |
// We will insert coro.end to cut any of the destructors for objects that |
| 394 |
// do not need to be destroyed once the coroutine is resumed. |
| 395 |
// See llvm/docs/Coroutines.rst for more details about coro.end. |
| 396 |
struct CallCoroEnd final : public EHScopeStack::Cleanup { |
| 397 |
void Emit(CodeGenFunction &CGF, Flags flags) override { |
| 398 |
auto &CGM = CGF.CGM; |
| 399 |
auto *NullPtr = llvm::ConstantPointerNull::get(CGF.Int8PtrTy); |
| 400 |
llvm::Function *CoroEndFn = CGM.getIntrinsic(llvm::Intrinsic::coro_end); |
| 401 |
// See if we have a funclet bundle to associate coro.end with. (WinEH) |
| 402 |
auto Bundles = getBundlesForCoroEnd(CGF); |
| 403 |
auto *CoroEnd = CGF.Builder.CreateCall( |
| 404 |
CoroEndFn, {NullPtr, CGF.Builder.getTrue()}, Bundles); |
| 405 |
if (Bundles.empty()) { |
| 406 |
// Otherwise, (landingpad model), create a conditional branch that leads |
| 407 |
// either to a cleanup block or a block with EH resume instruction. |
| 408 |
auto *ResumeBB = CGF.getEHResumeBlock(/*isCleanup=*/true); |
| 409 |
auto *CleanupContBB = CGF.createBasicBlock("cleanup.cont"); |
| 410 |
CGF.Builder.CreateCondBr(CoroEnd, ResumeBB, CleanupContBB); |
| 411 |
CGF.EmitBlock(CleanupContBB); |
| 412 |
} |
| 413 |
} |
| 414 |
}; |
| 415 |
} |
| 416 |
|
| 417 |
namespace { |
| 418 |
// Make sure to call coro.delete on scope exit. |
| 419 |
struct CallCoroDelete final : public EHScopeStack::Cleanup { |
| 420 |
Stmt *Deallocate; |
| 421 |
|
| 422 |
// Emit "if (coro.free(CoroId, CoroBegin)) Deallocate;" |
| 423 |
|
| 424 |
// Note: That deallocation will be emitted twice: once for a normal exit and |
| 425 |
// once for exceptional exit. This usage is safe because Deallocate does not |
| 426 |
// contain any declarations. The SubStmtBuilder::makeNewAndDeleteExpr() |
| 427 |
// builds a single call to a deallocation function which is safe to emit |
| 428 |
// multiple times. |
| 429 |
void Emit(CodeGenFunction &CGF, Flags) override { |
| 430 |
// Remember the current point, as we are going to emit deallocation code |
| 431 |
// first to get to coro.free instruction that is an argument to a delete |
| 432 |
// call. |
| 433 |
BasicBlock *SaveInsertBlock = CGF.Builder.GetInsertBlock(); |
| 434 |
|
| 435 |
auto *FreeBB = CGF.createBasicBlock("coro.free"); |
| 436 |
CGF.EmitBlock(FreeBB); |
| 437 |
CGF.EmitStmt(Deallocate); |
| 438 |
|
| 439 |
auto *AfterFreeBB = CGF.createBasicBlock("after.coro.free"); |
| 440 |
CGF.EmitBlock(AfterFreeBB); |
| 441 |
|
| 442 |
// We should have captured coro.free from the emission of deallocate. |
| 443 |
auto *CoroFree = CGF.CurCoro.Data->LastCoroFree; |
| 444 |
if (!CoroFree) { |
| 445 |
CGF.CGM.Error(Deallocate->getBeginLoc(), |
| 446 |
"Deallocation expressoin does not refer to coro.free"); |
| 447 |
return; |
| 448 |
} |
| 449 |
|
| 450 |
// Get back to the block we were originally and move coro.free there. |
| 451 |
auto *InsertPt = SaveInsertBlock->getTerminator(); |
| 452 |
CoroFree->moveBefore(InsertPt); |
| 453 |
CGF.Builder.SetInsertPoint(InsertPt); |
| 454 |
|
| 455 |
// Add if (auto *mem = coro.free) Deallocate; |
| 456 |
auto *NullPtr = llvm::ConstantPointerNull::get(CGF.Int8PtrTy); |
| 457 |
auto *Cond = CGF.Builder.CreateICmpNE(CoroFree, NullPtr); |
| 458 |
CGF.Builder.CreateCondBr(Cond, FreeBB, AfterFreeBB); |
| 459 |
|
| 460 |
// No longer need old terminator. |
| 461 |
InsertPt->eraseFromParent(); |
| 462 |
CGF.Builder.SetInsertPoint(AfterFreeBB); |
| 463 |
} |
| 464 |
explicit CallCoroDelete(Stmt *DeallocStmt) : Deallocate(DeallocStmt) {} |
| 465 |
}; |
| 466 |
} |
| 467 |
|
| 468 |
static void emitBodyAndFallthrough(CodeGenFunction &CGF, |
| 469 |
const CoroutineBodyStmt &S, Stmt *Body) { |
| 470 |
CGF.EmitStmt(Body); |
| 471 |
const bool CanFallthrough = CGF.Builder.GetInsertBlock(); |
| 472 |
if (CanFallthrough) |
| 473 |
if (Stmt *OnFallthrough = S.getFallthroughHandler()) |
| 474 |
CGF.EmitStmt(OnFallthrough); |
| 475 |
} |
| 476 |
|
| 477 |
void CodeGenFunction::EmitCoroutineBody(const CoroutineBodyStmt &S) { |
| 478 |
auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getInt8PtrTy()); |
| 479 |
auto &TI = CGM.getContext().getTargetInfo(); |
| 480 |
unsigned NewAlign = TI.getNewAlign() / TI.getCharWidth(); |
| 481 |
|
| 482 |
auto *EntryBB = Builder.GetInsertBlock(); |
| 483 |
auto *AllocBB = createBasicBlock("coro.alloc"); |
| 484 |
auto *InitBB = createBasicBlock("coro.init"); |
| 485 |
auto *FinalBB = createBasicBlock("coro.final"); |
| 486 |
auto *RetBB = createBasicBlock("coro.ret"); |
| 487 |
|
| 488 |
auto *CoroId = Builder.CreateCall( |
| 489 |
CGM.getIntrinsic(llvm::Intrinsic::coro_id), |
| 490 |
{Builder.getInt32(NewAlign), NullPtr, NullPtr, NullPtr}); |
| 491 |
createCoroData(*this, CurCoro, CoroId); |
| 492 |
CurCoro.Data->SuspendBB = RetBB; |
| 493 |
assert(ShouldEmitLifetimeMarkers && |
| 494 |
"Must emit lifetime intrinsics for coroutines"); |
| 495 |
|
| 496 |
// Backend is allowed to elide memory allocations, to help it, emit |
| 497 |
// auto mem = coro.alloc() ? 0 : ... allocation code ...; |
| 498 |
auto *CoroAlloc = Builder.CreateCall( |
| 499 |
CGM.getIntrinsic(llvm::Intrinsic::coro_alloc), {CoroId}); |
| 500 |
|
| 501 |
Builder.CreateCondBr(CoroAlloc, AllocBB, InitBB); |
| 502 |
|
| 503 |
EmitBlock(AllocBB); |
| 504 |
auto *AllocateCall = EmitScalarExpr(S.getAllocate()); |
| 505 |
auto *AllocOrInvokeContBB = Builder.GetInsertBlock(); |
| 506 |
|
| 507 |
// Handle allocation failure if 'ReturnStmtOnAllocFailure' was provided. |
| 508 |
if (auto *RetOnAllocFailure = S.getReturnStmtOnAllocFailure()) { |
| 509 |
auto *RetOnFailureBB = createBasicBlock("coro.ret.on.failure"); |
| 510 |
|
| 511 |
// See if allocation was successful. |
| 512 |
auto *NullPtr = llvm::ConstantPointerNull::get(Int8PtrTy); |
| 513 |
auto *Cond = Builder.CreateICmpNE(AllocateCall, NullPtr); |
| 514 |
Builder.CreateCondBr(Cond, InitBB, RetOnFailureBB); |
| 515 |
|
| 516 |
// If not, return OnAllocFailure object. |
| 517 |
EmitBlock(RetOnFailureBB); |
| 518 |
EmitStmt(RetOnAllocFailure); |
| 519 |
} |
| 520 |
else { |
| 521 |
Builder.CreateBr(InitBB); |
| 522 |
} |
| 523 |
|
| 524 |
EmitBlock(InitBB); |
| 525 |
|
| 526 |
// Pass the result of the allocation to coro.begin. |
| 527 |
auto *Phi = Builder.CreatePHI(VoidPtrTy, 2); |
| 528 |
Phi->addIncoming(NullPtr, EntryBB); |
| 529 |
Phi->addIncoming(AllocateCall, AllocOrInvokeContBB); |
| 530 |
auto *CoroBegin = Builder.CreateCall( |
| 531 |
CGM.getIntrinsic(llvm::Intrinsic::coro_begin), {CoroId, Phi}); |
| 532 |
CurCoro.Data->CoroBegin = CoroBegin; |
| 533 |
|
| 534 |
CurCoro.Data->CleanupJD = getJumpDestInCurrentScope(RetBB); |
| 535 |
{ |
| 536 |
CGDebugInfo *DI = getDebugInfo(); |
| 537 |
ParamReferenceReplacerRAII ParamReplacer(LocalDeclMap); |
| 538 |
CodeGenFunction::RunCleanupsScope ResumeScope(*this); |
| 539 |
EHStack.pushCleanup<CallCoroDelete>(NormalAndEHCleanup, S.getDeallocate()); |
| 540 |
|
| 541 |
// Create mapping between parameters and copy-params for coroutine function. |
| 542 |
llvm::ArrayRef<const Stmt *> ParamMoves = S.getParamMoves(); |
| 543 |
assert( |
| 544 |
(ParamMoves.size() == 0 || (ParamMoves.size() == FnArgs.size())) && |
| 545 |
"ParamMoves and FnArgs should be the same size for coroutine function"); |
| 546 |
if (ParamMoves.size() == FnArgs.size() && DI) |
| 547 |
for (const auto Pair : llvm::zip(FnArgs, ParamMoves)) |
| 548 |
DI->getCoroutineParameterMappings().insert( |
| 549 |
{std::get<0>(Pair), std::get<1>(Pair)}); |
| 550 |
|
| 551 |
// Create parameter copies. We do it before creating a promise, since an |
| 552 |
// evolution of coroutine TS may allow promise constructor to observe |
| 553 |
// parameter copies. |
| 554 |
for (auto *PM : S.getParamMoves()) { |
| 555 |
EmitStmt(PM); |
| 556 |
ParamReplacer.addCopy(cast<DeclStmt>(PM)); |
| 557 |
// TODO: if(CoroParam(...)) need to surround ctor and dtor |
| 558 |
// for the copy, so that llvm can elide it if the copy is |
| 559 |
// not needed. |
| 560 |
} |
| 561 |
|
| 562 |
EmitStmt(S.getPromiseDeclStmt()); |
| 563 |
|
| 564 |
Address PromiseAddr = GetAddrOfLocalVar(S.getPromiseDecl()); |
| 565 |
auto *PromiseAddrVoidPtr = |
| 566 |
new llvm::BitCastInst(PromiseAddr.getPointer(), VoidPtrTy, "", CoroId); |
| 567 |
// Update CoroId to refer to the promise. We could not do it earlier because |
| 568 |
// promise local variable was not emitted yet. |
| 569 |
CoroId->setArgOperand(1, PromiseAddrVoidPtr); |
| 570 |
|
| 571 |
// ReturnValue should be valid as long as the coroutine's return type |
| 572 |
// is not void. The assertion could help us to reduce the check later. |
| 573 |
assert(ReturnValue.isValid() == (bool)S.getReturnStmt()); |
| 574 |
// Now we have the promise, initialize the GRO. |
| 575 |
// We need to emit `get_return_object` first. According to: |
| 576 |
// [dcl.fct.def.coroutine]p7 |
| 577 |
// The call to get_return_object is sequenced before the call to |
| 578 |
// initial_suspend and is invoked at most once. |
| 579 |
// |
| 580 |
// So we couldn't emit return value when we emit return statment, |
| 581 |
// otherwise the call to get_return_object wouldn't be in front |
| 582 |
// of initial_suspend. |
| 583 |
if (ReturnValue.isValid()) { |
| 584 |
EmitAnyExprToMem(S.getReturnValue(), ReturnValue, |
| 585 |
S.getReturnValue()->getType().getQualifiers(), |
| 586 |
/*IsInit*/ true); |
| 587 |
} |
| 588 |
|
| 589 |
EHStack.pushCleanup<CallCoroEnd>(EHCleanup); |
| 590 |
|
| 591 |
CurCoro.Data->CurrentAwaitKind = AwaitKind::Init; |
| 592 |
CurCoro.Data->ExceptionHandler = S.getExceptionHandler(); |
| 593 |
EmitStmt(S.getInitSuspendStmt()); |
| 594 |
CurCoro.Data->FinalJD = getJumpDestInCurrentScope(FinalBB); |
| 595 |
|
| 596 |
CurCoro.Data->CurrentAwaitKind = AwaitKind::Normal; |
| 597 |
|
| 598 |
if (CurCoro.Data->ExceptionHandler) { |
| 599 |
// If we generated IR to record whether an exception was thrown from |
| 600 |
// 'await_resume', then use that IR to determine whether the coroutine |
| 601 |
// body should be skipped. |
| 602 |
// If we didn't generate the IR (perhaps because 'await_resume' was marked |
| 603 |
// as 'noexcept'), then we skip this check. |
| 604 |
BasicBlock *ContBB = nullptr; |
| 605 |
if (CurCoro.Data->ResumeEHVar) { |
| 606 |
BasicBlock *BodyBB = createBasicBlock("coro.resumed.body"); |
| 607 |
ContBB = createBasicBlock("coro.resumed.cont"); |
| 608 |
Value *SkipBody = Builder.CreateFlagLoad(CurCoro.Data->ResumeEHVar, |
| 609 |
"coro.resumed.eh"); |
| 610 |
Builder.CreateCondBr(SkipBody, ContBB, BodyBB); |
| 611 |
EmitBlock(BodyBB); |
| 612 |
} |
| 613 |
|
| 614 |
auto Loc = S.getBeginLoc(); |
| 615 |
CXXCatchStmt Catch(Loc, /*exDecl=*/nullptr, |
| 616 |
CurCoro.Data->ExceptionHandler); |
| 617 |
auto *TryStmt = |
| 618 |
CXXTryStmt::Create(getContext(), Loc, S.getBody(), &Catch); |
| 619 |
|
| 620 |
EnterCXXTryStmt(*TryStmt); |
| 621 |
emitBodyAndFallthrough(*this, S, TryStmt->getTryBlock()); |
| 622 |
ExitCXXTryStmt(*TryStmt); |
| 623 |
|
| 624 |
if (ContBB) |
| 625 |
EmitBlock(ContBB); |
| 626 |
} |
| 627 |
else { |
| 628 |
emitBodyAndFallthrough(*this, S, S.getBody()); |
| 629 |
} |
| 630 |
|
| 631 |
// See if we need to generate final suspend. |
| 632 |
const bool CanFallthrough = Builder.GetInsertBlock(); |
| 633 |
const bool HasCoreturns = CurCoro.Data->CoreturnCount > 0; |
| 634 |
if (CanFallthrough || HasCoreturns) { |
| 635 |
EmitBlock(FinalBB); |
| 636 |
CurCoro.Data->CurrentAwaitKind = AwaitKind::Final; |
| 637 |
EmitStmt(S.getFinalSuspendStmt()); |
| 638 |
} else { |
| 639 |
// We don't need FinalBB. Emit it to make sure the block is deleted. |
| 640 |
EmitBlock(FinalBB, /*IsFinished=*/true); |
| 641 |
} |
| 642 |
} |
| 643 |
|
| 644 |
EmitBlock(RetBB); |
| 645 |
// Emit coro.end before getReturnStmt (and parameter destructors), since |
| 646 |
// resume and destroy parts of the coroutine should not include them. |
| 647 |
llvm::Function *CoroEnd = CGM.getIntrinsic(llvm::Intrinsic::coro_end); |
| 648 |
Builder.CreateCall(CoroEnd, {NullPtr, Builder.getFalse()}); |
| 649 |
|
| 650 |
if (Stmt *Ret = S.getReturnStmt()) { |
| 651 |
// Since we already emitted the return value above, so we shouldn't |
| 652 |
// emit it again here. |
| 653 |
cast<ReturnStmt>(Ret)->setRetValue(nullptr); |
| 654 |
EmitStmt(Ret); |
| 655 |
} |
| 656 |
|
| 657 |
// LLVM require the frontend to mark the coroutine. |
| 658 |
CurFn->setPresplitCoroutine(); |
| 659 |
} |
| 660 |
|
| 661 |
// Emit coroutine intrinsic and patch up arguments of the token type. |
| 662 |
RValue CodeGenFunction::EmitCoroutineIntrinsic(const CallExpr *E, |
| 663 |
unsigned int IID) { |
| 664 |
SmallVector<llvm::Value *, 8> Args; |
| 665 |
switch (IID) { |
| 666 |
default: |
| 667 |
break; |
| 668 |
// The coro.frame builtin is replaced with an SSA value of the coro.begin |
| 669 |
// intrinsic. |
| 670 |
case llvm::Intrinsic::coro_frame: { |
| 671 |
if (CurCoro.Data && CurCoro.Data->CoroBegin) { |
| 672 |
return RValue::get(CurCoro.Data->CoroBegin); |
| 673 |
} |
| 674 |
CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_begin " |
| 675 |
"has been used earlier in this function"); |
| 676 |
auto *NullPtr = llvm::ConstantPointerNull::get(Builder.getInt8PtrTy()); |
| 677 |
return RValue::get(NullPtr); |
| 678 |
} |
| 679 |
case llvm::Intrinsic::coro_size: { |
| 680 |
auto &Context = getContext(); |
| 681 |
CanQualType SizeTy = Context.getSizeType(); |
| 682 |
llvm::IntegerType *T = Builder.getIntNTy(Context.getTypeSize(SizeTy)); |
| 683 |
llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::coro_size, T); |
| 684 |
return RValue::get(Builder.CreateCall(F)); |
| 685 |
} |
| 686 |
case llvm::Intrinsic::coro_align: { |
| 687 |
auto &Context = getContext(); |
| 688 |
CanQualType SizeTy = Context.getSizeType(); |
| 689 |
llvm::IntegerType *T = Builder.getIntNTy(Context.getTypeSize(SizeTy)); |
| 690 |
llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::coro_align, T); |
| 691 |
return RValue::get(Builder.CreateCall(F)); |
| 692 |
} |
| 693 |
// The following three intrinsics take a token parameter referring to a token |
| 694 |
// returned by earlier call to @llvm.coro.id. Since we cannot represent it in |
| 695 |
// builtins, we patch it up here. |
| 696 |
case llvm::Intrinsic::coro_alloc: |
| 697 |
case llvm::Intrinsic::coro_begin: |
| 698 |
case llvm::Intrinsic::coro_free: { |
| 699 |
if (CurCoro.Data && CurCoro.Data->CoroId) { |
| 700 |
Args.push_back(CurCoro.Data->CoroId); |
| 701 |
break; |
| 702 |
} |
| 703 |
CGM.Error(E->getBeginLoc(), "this builtin expect that __builtin_coro_id has" |
| 704 |
" been used earlier in this function"); |
| 705 |
// Fallthrough to the next case to add TokenNone as the first argument. |
| 706 |
[[fallthrough]]; |
| 707 |
} |
| 708 |
// @llvm.coro.suspend takes a token parameter. Add token 'none' as the first |
| 709 |
// argument. |
| 710 |
case llvm::Intrinsic::coro_suspend: |
| 711 |
Args.push_back(llvm::ConstantTokenNone::get(getLLVMContext())); |
| 712 |
break; |
| 713 |
} |
| 714 |
for (const Expr *Arg : E->arguments()) |
| 715 |
Args.push_back(EmitScalarExpr(Arg)); |
| 716 |
|
| 717 |
llvm::Function *F = CGM.getIntrinsic(IID); |
| 718 |
llvm::CallInst *Call = Builder.CreateCall(F, Args); |
| 719 |
|
| 720 |
// Note: The following code is to enable to emit coro.id and coro.begin by |
| 721 |
// hand to experiment with coroutines in C. |
| 722 |
// If we see @llvm.coro.id remember it in the CoroData. We will update |
| 723 |
// coro.alloc, coro.begin and coro.free intrinsics to refer to it. |
| 724 |
if (IID == llvm::Intrinsic::coro_id) { |
| 725 |
createCoroData(*this, CurCoro, Call, E); |
| 726 |
} |
| 727 |
else if (IID == llvm::Intrinsic::coro_begin) { |
| 728 |
if (CurCoro.Data) |
| 729 |
CurCoro.Data->CoroBegin = Call; |
| 730 |
} |
| 731 |
else if (IID == llvm::Intrinsic::coro_free) { |
| 732 |
// Remember the last coro_free as we need it to build the conditional |
| 733 |
// deletion of the coroutine frame. |
| 734 |
if (CurCoro.Data) |
| 735 |
CurCoro.Data->LastCoroFree = Call; |
| 736 |
} |
| 737 |
return RValue::get(Call); |
| 738 |
} |