| 1 |
//===--- CodeGenPGO.cpp - PGO Instrumentation for LLVM CodeGen --*- C++ -*-===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// Instrumentation-based profile-guided optimization |
| 10 |
// |
| 11 |
//===----------------------------------------------------------------------===// |
| 12 |
|
| 13 |
#include "CodeGenPGO.h" |
| 14 |
#include "CodeGenFunction.h" |
| 15 |
#include "CoverageMappingGen.h" |
| 16 |
#include "clang/AST/RecursiveASTVisitor.h" |
| 17 |
#include "clang/AST/StmtVisitor.h" |
| 18 |
#include "llvm/IR/Intrinsics.h" |
| 19 |
#include "llvm/IR/MDBuilder.h" |
| 20 |
#include "llvm/Support/CommandLine.h" |
| 21 |
#include "llvm/Support/Endian.h" |
| 22 |
#include "llvm/Support/FileSystem.h" |
| 23 |
#include "llvm/Support/MD5.h" |
| 24 |
#include <optional> |
| 25 |
|
| 26 |
static llvm::cl::opt<bool> |
| 27 |
EnableValueProfiling("enable-value-profiling", |
| 28 |
llvm::cl::desc("Enable value profiling"), |
| 29 |
llvm::cl::Hidden, llvm::cl::init(false)); |
| 30 |
|
| 31 |
using namespace clang; |
| 32 |
using namespace CodeGen; |
| 33 |
|
| 34 |
void CodeGenPGO::setFuncName(StringRef Name, |
| 35 |
llvm::GlobalValue::LinkageTypes Linkage) { |
| 36 |
llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader(); |
| 37 |
FuncName = llvm::getPGOFuncName( |
| 38 |
Name, Linkage, CGM.getCodeGenOpts().MainFileName, |
| 39 |
PGOReader ? PGOReader->getVersion() : llvm::IndexedInstrProf::Version); |
| 40 |
|
| 41 |
// If we're generating a profile, create a variable for the name. |
| 42 |
if (CGM.getCodeGenOpts().hasProfileClangInstr()) |
| 43 |
FuncNameVar = llvm::createPGOFuncNameVar(CGM.getModule(), Linkage, FuncName); |
| 44 |
} |
| 45 |
|
| 46 |
void CodeGenPGO::setFuncName(llvm::Function *Fn) { |
| 47 |
setFuncName(Fn->getName(), Fn->getLinkage()); |
| 48 |
// Create PGOFuncName meta data. |
| 49 |
llvm::createPGOFuncNameMetadata(*Fn, FuncName); |
| 50 |
} |
| 51 |
|
| 52 |
/// The version of the PGO hash algorithm. |
| 53 |
enum PGOHashVersion : unsigned { |
| 54 |
PGO_HASH_V1, |
| 55 |
PGO_HASH_V2, |
| 56 |
PGO_HASH_V3, |
| 57 |
|
| 58 |
// Keep this set to the latest hash version. |
| 59 |
PGO_HASH_LATEST = PGO_HASH_V3 |
| 60 |
}; |
| 61 |
|
| 62 |
namespace { |
| 63 |
/// Stable hasher for PGO region counters. |
| 64 |
/// |
| 65 |
/// PGOHash produces a stable hash of a given function's control flow. |
| 66 |
/// |
| 67 |
/// Changing the output of this hash will invalidate all previously generated |
| 68 |
/// profiles -- i.e., don't do it. |
| 69 |
/// |
| 70 |
/// \note When this hash does eventually change (years?), we still need to |
| 71 |
/// support old hashes. We'll need to pull in the version number from the |
| 72 |
/// profile data format and use the matching hash function. |
| 73 |
class PGOHash { |
| 74 |
uint64_t Working; |
| 75 |
unsigned Count; |
| 76 |
PGOHashVersion HashVersion; |
| 77 |
llvm::MD5 MD5; |
| 78 |
|
| 79 |
static const int NumBitsPerType = 6; |
| 80 |
static const unsigned NumTypesPerWord = sizeof(uint64_t) * 8 / NumBitsPerType; |
| 81 |
static const unsigned TooBig = 1u << NumBitsPerType; |
| 82 |
|
| 83 |
public: |
| 84 |
/// Hash values for AST nodes. |
| 85 |
/// |
| 86 |
/// Distinct values for AST nodes that have region counters attached. |
| 87 |
/// |
| 88 |
/// These values must be stable. All new members must be added at the end, |
| 89 |
/// and no members should be removed. Changing the enumeration value for an |
| 90 |
/// AST node will affect the hash of every function that contains that node. |
| 91 |
enum HashType : unsigned char { |
| 92 |
None = 0, |
| 93 |
LabelStmt = 1, |
| 94 |
WhileStmt, |
| 95 |
DoStmt, |
| 96 |
ForStmt, |
| 97 |
CXXForRangeStmt, |
| 98 |
ObjCForCollectionStmt, |
| 99 |
SwitchStmt, |
| 100 |
CaseStmt, |
| 101 |
DefaultStmt, |
| 102 |
IfStmt, |
| 103 |
CXXTryStmt, |
| 104 |
CXXCatchStmt, |
| 105 |
ConditionalOperator, |
| 106 |
BinaryOperatorLAnd, |
| 107 |
BinaryOperatorLOr, |
| 108 |
BinaryConditionalOperator, |
| 109 |
// The preceding values are available with PGO_HASH_V1. |
| 110 |
|
| 111 |
EndOfScope, |
| 112 |
IfThenBranch, |
| 113 |
IfElseBranch, |
| 114 |
GotoStmt, |
| 115 |
IndirectGotoStmt, |
| 116 |
BreakStmt, |
| 117 |
ContinueStmt, |
| 118 |
ReturnStmt, |
| 119 |
ThrowExpr, |
| 120 |
UnaryOperatorLNot, |
| 121 |
BinaryOperatorLT, |
| 122 |
BinaryOperatorGT, |
| 123 |
BinaryOperatorLE, |
| 124 |
BinaryOperatorGE, |
| 125 |
BinaryOperatorEQ, |
| 126 |
BinaryOperatorNE, |
| 127 |
// The preceding values are available since PGO_HASH_V2. |
| 128 |
|
| 129 |
// Keep this last. It's for the static assert that follows. |
| 130 |
LastHashType |
| 131 |
}; |
| 132 |
static_assert(LastHashType <= TooBig, "Too many types in HashType"); |
| 133 |
|
| 134 |
PGOHash(PGOHashVersion HashVersion) |
| 135 |
: Working(0), Count(0), HashVersion(HashVersion) {} |
| 136 |
void combine(HashType Type); |
| 137 |
uint64_t finalize(); |
| 138 |
PGOHashVersion getHashVersion() const { return HashVersion; } |
| 139 |
}; |
| 140 |
const int PGOHash::NumBitsPerType; |
| 141 |
const unsigned PGOHash::NumTypesPerWord; |
| 142 |
const unsigned PGOHash::TooBig; |
| 143 |
|
| 144 |
/// Get the PGO hash version used in the given indexed profile. |
| 145 |
static PGOHashVersion getPGOHashVersion(llvm::IndexedInstrProfReader *PGOReader, |
| 146 |
CodeGenModule &CGM) { |
| 147 |
if (PGOReader->getVersion() <= 4) |
| 148 |
return PGO_HASH_V1; |
| 149 |
if (PGOReader->getVersion() <= 5) |
| 150 |
return PGO_HASH_V2; |
| 151 |
return PGO_HASH_V3; |
| 152 |
} |
| 153 |
|
| 154 |
/// A RecursiveASTVisitor that fills a map of statements to PGO counters. |
| 155 |
struct MapRegionCounters : public RecursiveASTVisitor<MapRegionCounters> { |
| 156 |
using Base = RecursiveASTVisitor<MapRegionCounters>; |
| 157 |
|
| 158 |
/// The next counter value to assign. |
| 159 |
unsigned NextCounter; |
| 160 |
/// The function hash. |
| 161 |
PGOHash Hash; |
| 162 |
/// The map of statements to counters. |
| 163 |
llvm::DenseMap<const Stmt *, unsigned> &CounterMap; |
| 164 |
/// The profile version. |
| 165 |
uint64_t ProfileVersion; |
| 166 |
|
| 167 |
MapRegionCounters(PGOHashVersion HashVersion, uint64_t ProfileVersion, |
| 168 |
llvm::DenseMap<const Stmt *, unsigned> &CounterMap) |
| 169 |
: NextCounter(0), Hash(HashVersion), CounterMap(CounterMap), |
| 170 |
ProfileVersion(ProfileVersion) {} |
| 171 |
|
| 172 |
// Blocks and lambdas are handled as separate functions, so we need not |
| 173 |
// traverse them in the parent context. |
| 174 |
bool TraverseBlockExpr(BlockExpr *BE) { return true; } |
| 175 |
bool TraverseLambdaExpr(LambdaExpr *LE) { |
| 176 |
// Traverse the captures, but not the body. |
| 177 |
for (auto C : zip(LE->captures(), LE->capture_inits())) |
| 178 |
TraverseLambdaCapture(LE, &std::get<0>(C), std::get<1>(C)); |
| 179 |
return true; |
| 180 |
} |
| 181 |
bool TraverseCapturedStmt(CapturedStmt *CS) { return true; } |
| 182 |
|
| 183 |
bool VisitDecl(const Decl *D) { |
| 184 |
switch (D->getKind()) { |
| 185 |
default: |
| 186 |
break; |
| 187 |
case Decl::Function: |
| 188 |
case Decl::CXXMethod: |
| 189 |
case Decl::CXXConstructor: |
| 190 |
case Decl::CXXDestructor: |
| 191 |
case Decl::CXXConversion: |
| 192 |
case Decl::ObjCMethod: |
| 193 |
case Decl::Block: |
| 194 |
case Decl::Captured: |
| 195 |
CounterMap[D->getBody()] = NextCounter++; |
| 196 |
break; |
| 197 |
} |
| 198 |
return true; |
| 199 |
} |
| 200 |
|
| 201 |
/// If \p S gets a fresh counter, update the counter mappings. Return the |
| 202 |
/// V1 hash of \p S. |
| 203 |
PGOHash::HashType updateCounterMappings(Stmt *S) { |
| 204 |
auto Type = getHashType(PGO_HASH_V1, S); |
| 205 |
if (Type != PGOHash::None) |
| 206 |
CounterMap[S] = NextCounter++; |
| 207 |
return Type; |
| 208 |
} |
| 209 |
|
| 210 |
/// The RHS of all logical operators gets a fresh counter in order to count |
| 211 |
/// how many times the RHS evaluates to true or false, depending on the |
| 212 |
/// semantics of the operator. This is only valid for ">= v7" of the profile |
| 213 |
/// version so that we facilitate backward compatibility. |
| 214 |
bool VisitBinaryOperator(BinaryOperator *S) { |
| 215 |
if (ProfileVersion >= llvm::IndexedInstrProf::Version7) |
| 216 |
if (S->isLogicalOp() && |
| 217 |
CodeGenFunction::isInstrumentedCondition(S->getRHS())) |
| 218 |
CounterMap[S->getRHS()] = NextCounter++; |
| 219 |
return Base::VisitBinaryOperator(S); |
| 220 |
} |
| 221 |
|
| 222 |
/// Include \p S in the function hash. |
| 223 |
bool VisitStmt(Stmt *S) { |
| 224 |
auto Type = updateCounterMappings(S); |
| 225 |
if (Hash.getHashVersion() != PGO_HASH_V1) |
| 226 |
Type = getHashType(Hash.getHashVersion(), S); |
| 227 |
if (Type != PGOHash::None) |
| 228 |
Hash.combine(Type); |
| 229 |
return true; |
| 230 |
} |
| 231 |
|
| 232 |
bool TraverseIfStmt(IfStmt *If) { |
| 233 |
// If we used the V1 hash, use the default traversal. |
| 234 |
if (Hash.getHashVersion() == PGO_HASH_V1) |
| 235 |
return Base::TraverseIfStmt(If); |
| 236 |
|
| 237 |
// Otherwise, keep track of which branch we're in while traversing. |
| 238 |
VisitStmt(If); |
| 239 |
for (Stmt *CS : If->children()) { |
| 240 |
if (!CS) |
| 241 |
continue; |
| 242 |
if (CS == If->getThen()) |
| 243 |
Hash.combine(PGOHash::IfThenBranch); |
| 244 |
else if (CS == If->getElse()) |
| 245 |
Hash.combine(PGOHash::IfElseBranch); |
| 246 |
TraverseStmt(CS); |
| 247 |
} |
| 248 |
Hash.combine(PGOHash::EndOfScope); |
| 249 |
return true; |
| 250 |
} |
| 251 |
|
| 252 |
// If the statement type \p N is nestable, and its nesting impacts profile |
| 253 |
// stability, define a custom traversal which tracks the end of the statement |
| 254 |
// in the hash (provided we're not using the V1 hash). |
| 255 |
#define DEFINE_NESTABLE_TRAVERSAL(N) \ |
| 256 |
bool Traverse##N(N *S) { \ |
| 257 |
Base::Traverse##N(S); \ |
| 258 |
if (Hash.getHashVersion() != PGO_HASH_V1) \ |
| 259 |
Hash.combine(PGOHash::EndOfScope); \ |
| 260 |
return true; \ |
| 261 |
} |
| 262 |
|
| 263 |
DEFINE_NESTABLE_TRAVERSAL(WhileStmt) |
| 264 |
DEFINE_NESTABLE_TRAVERSAL(DoStmt) |
| 265 |
DEFINE_NESTABLE_TRAVERSAL(ForStmt) |
| 266 |
DEFINE_NESTABLE_TRAVERSAL(CXXForRangeStmt) |
| 267 |
DEFINE_NESTABLE_TRAVERSAL(ObjCForCollectionStmt) |
| 268 |
DEFINE_NESTABLE_TRAVERSAL(CXXTryStmt) |
| 269 |
DEFINE_NESTABLE_TRAVERSAL(CXXCatchStmt) |
| 270 |
|
| 271 |
/// Get version \p HashVersion of the PGO hash for \p S. |
| 272 |
PGOHash::HashType getHashType(PGOHashVersion HashVersion, const Stmt *S) { |
| 273 |
switch (S->getStmtClass()) { |
| 274 |
default: |
| 275 |
break; |
| 276 |
case Stmt::LabelStmtClass: |
| 277 |
return PGOHash::LabelStmt; |
| 278 |
case Stmt::WhileStmtClass: |
| 279 |
return PGOHash::WhileStmt; |
| 280 |
case Stmt::DoStmtClass: |
| 281 |
return PGOHash::DoStmt; |
| 282 |
case Stmt::ForStmtClass: |
| 283 |
return PGOHash::ForStmt; |
| 284 |
case Stmt::CXXForRangeStmtClass: |
| 285 |
return PGOHash::CXXForRangeStmt; |
| 286 |
case Stmt::ObjCForCollectionStmtClass: |
| 287 |
return PGOHash::ObjCForCollectionStmt; |
| 288 |
case Stmt::SwitchStmtClass: |
| 289 |
return PGOHash::SwitchStmt; |
| 290 |
case Stmt::CaseStmtClass: |
| 291 |
return PGOHash::CaseStmt; |
| 292 |
case Stmt::DefaultStmtClass: |
| 293 |
return PGOHash::DefaultStmt; |
| 294 |
case Stmt::IfStmtClass: |
| 295 |
return PGOHash::IfStmt; |
| 296 |
case Stmt::CXXTryStmtClass: |
| 297 |
return PGOHash::CXXTryStmt; |
| 298 |
case Stmt::CXXCatchStmtClass: |
| 299 |
return PGOHash::CXXCatchStmt; |
| 300 |
case Stmt::ConditionalOperatorClass: |
| 301 |
return PGOHash::ConditionalOperator; |
| 302 |
case Stmt::BinaryConditionalOperatorClass: |
| 303 |
return PGOHash::BinaryConditionalOperator; |
| 304 |
case Stmt::BinaryOperatorClass: { |
| 305 |
const BinaryOperator *BO = cast<BinaryOperator>(S); |
| 306 |
if (BO->getOpcode() == BO_LAnd) |
| 307 |
return PGOHash::BinaryOperatorLAnd; |
| 308 |
if (BO->getOpcode() == BO_LOr) |
| 309 |
return PGOHash::BinaryOperatorLOr; |
| 310 |
if (HashVersion >= PGO_HASH_V2) { |
| 311 |
switch (BO->getOpcode()) { |
| 312 |
default: |
| 313 |
break; |
| 314 |
case BO_LT: |
| 315 |
return PGOHash::BinaryOperatorLT; |
| 316 |
case BO_GT: |
| 317 |
return PGOHash::BinaryOperatorGT; |
| 318 |
case BO_LE: |
| 319 |
return PGOHash::BinaryOperatorLE; |
| 320 |
case BO_GE: |
| 321 |
return PGOHash::BinaryOperatorGE; |
| 322 |
case BO_EQ: |
| 323 |
return PGOHash::BinaryOperatorEQ; |
| 324 |
case BO_NE: |
| 325 |
return PGOHash::BinaryOperatorNE; |
| 326 |
} |
| 327 |
} |
| 328 |
break; |
| 329 |
} |
| 330 |
} |
| 331 |
|
| 332 |
if (HashVersion >= PGO_HASH_V2) { |
| 333 |
switch (S->getStmtClass()) { |
| 334 |
default: |
| 335 |
break; |
| 336 |
case Stmt::GotoStmtClass: |
| 337 |
return PGOHash::GotoStmt; |
| 338 |
case Stmt::IndirectGotoStmtClass: |
| 339 |
return PGOHash::IndirectGotoStmt; |
| 340 |
case Stmt::BreakStmtClass: |
| 341 |
return PGOHash::BreakStmt; |
| 342 |
case Stmt::ContinueStmtClass: |
| 343 |
return PGOHash::ContinueStmt; |
| 344 |
case Stmt::ReturnStmtClass: |
| 345 |
return PGOHash::ReturnStmt; |
| 346 |
case Stmt::CXXThrowExprClass: |
| 347 |
return PGOHash::ThrowExpr; |
| 348 |
case Stmt::UnaryOperatorClass: { |
| 349 |
const UnaryOperator *UO = cast<UnaryOperator>(S); |
| 350 |
if (UO->getOpcode() == UO_LNot) |
| 351 |
return PGOHash::UnaryOperatorLNot; |
| 352 |
break; |
| 353 |
} |
| 354 |
} |
| 355 |
} |
| 356 |
|
| 357 |
return PGOHash::None; |
| 358 |
} |
| 359 |
}; |
| 360 |
|
| 361 |
/// A StmtVisitor that propagates the raw counts through the AST and |
| 362 |
/// records the count at statements where the value may change. |
| 363 |
struct ComputeRegionCounts : public ConstStmtVisitor<ComputeRegionCounts> { |
| 364 |
/// PGO state. |
| 365 |
CodeGenPGO &PGO; |
| 366 |
|
| 367 |
/// A flag that is set when the current count should be recorded on the |
| 368 |
/// next statement, such as at the exit of a loop. |
| 369 |
bool RecordNextStmtCount; |
| 370 |
|
| 371 |
/// The count at the current location in the traversal. |
| 372 |
uint64_t CurrentCount; |
| 373 |
|
| 374 |
/// The map of statements to count values. |
| 375 |
llvm::DenseMap<const Stmt *, uint64_t> &CountMap; |
| 376 |
|
| 377 |
/// BreakContinueStack - Keep counts of breaks and continues inside loops. |
| 378 |
struct BreakContinue { |
| 379 |
uint64_t BreakCount; |
| 380 |
uint64_t ContinueCount; |
| 381 |
BreakContinue() : BreakCount(0), ContinueCount(0) {} |
| 382 |
}; |
| 383 |
SmallVector<BreakContinue, 8> BreakContinueStack; |
| 384 |
|
| 385 |
ComputeRegionCounts(llvm::DenseMap<const Stmt *, uint64_t> &CountMap, |
| 386 |
CodeGenPGO &PGO) |
| 387 |
: PGO(PGO), RecordNextStmtCount(false), CountMap(CountMap) {} |
| 388 |
|
| 389 |
void RecordStmtCount(const Stmt *S) { |
| 390 |
if (RecordNextStmtCount) { |
| 391 |
CountMap[S] = CurrentCount; |
| 392 |
RecordNextStmtCount = false; |
| 393 |
} |
| 394 |
} |
| 395 |
|
| 396 |
/// Set and return the current count. |
| 397 |
uint64_t setCount(uint64_t Count) { |
| 398 |
CurrentCount = Count; |
| 399 |
return Count; |
| 400 |
} |
| 401 |
|
| 402 |
void VisitStmt(const Stmt *S) { |
| 403 |
RecordStmtCount(S); |
| 404 |
for (const Stmt *Child : S->children()) |
| 405 |
if (Child) |
| 406 |
this->Visit(Child); |
| 407 |
} |
| 408 |
|
| 409 |
void VisitFunctionDecl(const FunctionDecl *D) { |
| 410 |
// Counter tracks entry to the function body. |
| 411 |
uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); |
| 412 |
CountMap[D->getBody()] = BodyCount; |
| 413 |
Visit(D->getBody()); |
| 414 |
} |
| 415 |
|
| 416 |
// Skip lambda expressions. We visit these as FunctionDecls when we're |
| 417 |
// generating them and aren't interested in the body when generating a |
| 418 |
// parent context. |
| 419 |
void VisitLambdaExpr(const LambdaExpr *LE) {} |
| 420 |
|
| 421 |
void VisitCapturedDecl(const CapturedDecl *D) { |
| 422 |
// Counter tracks entry to the capture body. |
| 423 |
uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); |
| 424 |
CountMap[D->getBody()] = BodyCount; |
| 425 |
Visit(D->getBody()); |
| 426 |
} |
| 427 |
|
| 428 |
void VisitObjCMethodDecl(const ObjCMethodDecl *D) { |
| 429 |
// Counter tracks entry to the method body. |
| 430 |
uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); |
| 431 |
CountMap[D->getBody()] = BodyCount; |
| 432 |
Visit(D->getBody()); |
| 433 |
} |
| 434 |
|
| 435 |
void VisitBlockDecl(const BlockDecl *D) { |
| 436 |
// Counter tracks entry to the block body. |
| 437 |
uint64_t BodyCount = setCount(PGO.getRegionCount(D->getBody())); |
| 438 |
CountMap[D->getBody()] = BodyCount; |
| 439 |
Visit(D->getBody()); |
| 440 |
} |
| 441 |
|
| 442 |
void VisitReturnStmt(const ReturnStmt *S) { |
| 443 |
RecordStmtCount(S); |
| 444 |
if (S->getRetValue()) |
| 445 |
Visit(S->getRetValue()); |
| 446 |
CurrentCount = 0; |
| 447 |
RecordNextStmtCount = true; |
| 448 |
} |
| 449 |
|
| 450 |
void VisitCXXThrowExpr(const CXXThrowExpr *E) { |
| 451 |
RecordStmtCount(E); |
| 452 |
if (E->getSubExpr()) |
| 453 |
Visit(E->getSubExpr()); |
| 454 |
CurrentCount = 0; |
| 455 |
RecordNextStmtCount = true; |
| 456 |
} |
| 457 |
|
| 458 |
void VisitGotoStmt(const GotoStmt *S) { |
| 459 |
RecordStmtCount(S); |
| 460 |
CurrentCount = 0; |
| 461 |
RecordNextStmtCount = true; |
| 462 |
} |
| 463 |
|
| 464 |
void VisitLabelStmt(const LabelStmt *S) { |
| 465 |
RecordNextStmtCount = false; |
| 466 |
// Counter tracks the block following the label. |
| 467 |
uint64_t BlockCount = setCount(PGO.getRegionCount(S)); |
| 468 |
CountMap[S] = BlockCount; |
| 469 |
Visit(S->getSubStmt()); |
| 470 |
} |
| 471 |
|
| 472 |
void VisitBreakStmt(const BreakStmt *S) { |
| 473 |
RecordStmtCount(S); |
| 474 |
assert(!BreakContinueStack.empty() && "break not in a loop or switch!"); |
| 475 |
BreakContinueStack.back().BreakCount += CurrentCount; |
| 476 |
CurrentCount = 0; |
| 477 |
RecordNextStmtCount = true; |
| 478 |
} |
| 479 |
|
| 480 |
void VisitContinueStmt(const ContinueStmt *S) { |
| 481 |
RecordStmtCount(S); |
| 482 |
assert(!BreakContinueStack.empty() && "continue stmt not in a loop!"); |
| 483 |
BreakContinueStack.back().ContinueCount += CurrentCount; |
| 484 |
CurrentCount = 0; |
| 485 |
RecordNextStmtCount = true; |
| 486 |
} |
| 487 |
|
| 488 |
void VisitWhileStmt(const WhileStmt *S) { |
| 489 |
RecordStmtCount(S); |
| 490 |
uint64_t ParentCount = CurrentCount; |
| 491 |
|
| 492 |
BreakContinueStack.push_back(BreakContinue()); |
| 493 |
// Visit the body region first so the break/continue adjustments can be |
| 494 |
// included when visiting the condition. |
| 495 |
uint64_t BodyCount = setCount(PGO.getRegionCount(S)); |
| 496 |
CountMap[S->getBody()] = CurrentCount; |
| 497 |
Visit(S->getBody()); |
| 498 |
uint64_t BackedgeCount = CurrentCount; |
| 499 |
|
| 500 |
// ...then go back and propagate counts through the condition. The count |
| 501 |
// at the start of the condition is the sum of the incoming edges, |
| 502 |
// the backedge from the end of the loop body, and the edges from |
| 503 |
// continue statements. |
| 504 |
BreakContinue BC = BreakContinueStack.pop_back_val(); |
| 505 |
uint64_t CondCount = |
| 506 |
setCount(ParentCount + BackedgeCount + BC.ContinueCount); |
| 507 |
CountMap[S->getCond()] = CondCount; |
| 508 |
Visit(S->getCond()); |
| 509 |
setCount(BC.BreakCount + CondCount - BodyCount); |
| 510 |
RecordNextStmtCount = true; |
| 511 |
} |
| 512 |
|
| 513 |
void VisitDoStmt(const DoStmt *S) { |
| 514 |
RecordStmtCount(S); |
| 515 |
uint64_t LoopCount = PGO.getRegionCount(S); |
| 516 |
|
| 517 |
BreakContinueStack.push_back(BreakContinue()); |
| 518 |
// The count doesn't include the fallthrough from the parent scope. Add it. |
| 519 |
uint64_t BodyCount = setCount(LoopCount + CurrentCount); |
| 520 |
CountMap[S->getBody()] = BodyCount; |
| 521 |
Visit(S->getBody()); |
| 522 |
uint64_t BackedgeCount = CurrentCount; |
| 523 |
|
| 524 |
BreakContinue BC = BreakContinueStack.pop_back_val(); |
| 525 |
// The count at the start of the condition is equal to the count at the |
| 526 |
// end of the body, plus any continues. |
| 527 |
uint64_t CondCount = setCount(BackedgeCount + BC.ContinueCount); |
| 528 |
CountMap[S->getCond()] = CondCount; |
| 529 |
Visit(S->getCond()); |
| 530 |
setCount(BC.BreakCount + CondCount - LoopCount); |
| 531 |
RecordNextStmtCount = true; |
| 532 |
} |
| 533 |
|
| 534 |
void VisitForStmt(const ForStmt *S) { |
| 535 |
RecordStmtCount(S); |
| 536 |
if (S->getInit()) |
| 537 |
Visit(S->getInit()); |
| 538 |
|
| 539 |
uint64_t ParentCount = CurrentCount; |
| 540 |
|
| 541 |
BreakContinueStack.push_back(BreakContinue()); |
| 542 |
// Visit the body region first. (This is basically the same as a while |
| 543 |
// loop; see further comments in VisitWhileStmt.) |
| 544 |
uint64_t BodyCount = setCount(PGO.getRegionCount(S)); |
| 545 |
CountMap[S->getBody()] = BodyCount; |
| 546 |
Visit(S->getBody()); |
| 547 |
uint64_t BackedgeCount = CurrentCount; |
| 548 |
BreakContinue BC = BreakContinueStack.pop_back_val(); |
| 549 |
|
| 550 |
// The increment is essentially part of the body but it needs to include |
| 551 |
// the count for all the continue statements. |
| 552 |
if (S->getInc()) { |
| 553 |
uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount); |
| 554 |
CountMap[S->getInc()] = IncCount; |
| 555 |
Visit(S->getInc()); |
| 556 |
} |
| 557 |
|
| 558 |
// ...then go back and propagate counts through the condition. |
| 559 |
uint64_t CondCount = |
| 560 |
setCount(ParentCount + BackedgeCount + BC.ContinueCount); |
| 561 |
if (S->getCond()) { |
| 562 |
CountMap[S->getCond()] = CondCount; |
| 563 |
Visit(S->getCond()); |
| 564 |
} |
| 565 |
setCount(BC.BreakCount + CondCount - BodyCount); |
| 566 |
RecordNextStmtCount = true; |
| 567 |
} |
| 568 |
|
| 569 |
void VisitCXXForRangeStmt(const CXXForRangeStmt *S) { |
| 570 |
RecordStmtCount(S); |
| 571 |
if (S->getInit()) |
| 572 |
Visit(S->getInit()); |
| 573 |
Visit(S->getLoopVarStmt()); |
| 574 |
Visit(S->getRangeStmt()); |
| 575 |
Visit(S->getBeginStmt()); |
| 576 |
Visit(S->getEndStmt()); |
| 577 |
|
| 578 |
uint64_t ParentCount = CurrentCount; |
| 579 |
BreakContinueStack.push_back(BreakContinue()); |
| 580 |
// Visit the body region first. (This is basically the same as a while |
| 581 |
// loop; see further comments in VisitWhileStmt.) |
| 582 |
uint64_t BodyCount = setCount(PGO.getRegionCount(S)); |
| 583 |
CountMap[S->getBody()] = BodyCount; |
| 584 |
Visit(S->getBody()); |
| 585 |
uint64_t BackedgeCount = CurrentCount; |
| 586 |
BreakContinue BC = BreakContinueStack.pop_back_val(); |
| 587 |
|
| 588 |
// The increment is essentially part of the body but it needs to include |
| 589 |
// the count for all the continue statements. |
| 590 |
uint64_t IncCount = setCount(BackedgeCount + BC.ContinueCount); |
| 591 |
CountMap[S->getInc()] = IncCount; |
| 592 |
Visit(S->getInc()); |
| 593 |
|
| 594 |
// ...then go back and propagate counts through the condition. |
| 595 |
uint64_t CondCount = |
| 596 |
setCount(ParentCount + BackedgeCount + BC.ContinueCount); |
| 597 |
CountMap[S->getCond()] = CondCount; |
| 598 |
Visit(S->getCond()); |
| 599 |
setCount(BC.BreakCount + CondCount - BodyCount); |
| 600 |
RecordNextStmtCount = true; |
| 601 |
} |
| 602 |
|
| 603 |
void VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S) { |
| 604 |
RecordStmtCount(S); |
| 605 |
Visit(S->getElement()); |
| 606 |
uint64_t ParentCount = CurrentCount; |
| 607 |
BreakContinueStack.push_back(BreakContinue()); |
| 608 |
// Counter tracks the body of the loop. |
| 609 |
uint64_t BodyCount = setCount(PGO.getRegionCount(S)); |
| 610 |
CountMap[S->getBody()] = BodyCount; |
| 611 |
Visit(S->getBody()); |
| 612 |
uint64_t BackedgeCount = CurrentCount; |
| 613 |
BreakContinue BC = BreakContinueStack.pop_back_val(); |
| 614 |
|
| 615 |
setCount(BC.BreakCount + ParentCount + BackedgeCount + BC.ContinueCount - |
| 616 |
BodyCount); |
| 617 |
RecordNextStmtCount = true; |
| 618 |
} |
| 619 |
|
| 620 |
void VisitSwitchStmt(const SwitchStmt *S) { |
| 621 |
RecordStmtCount(S); |
| 622 |
if (S->getInit()) |
| 623 |
Visit(S->getInit()); |
| 624 |
Visit(S->getCond()); |
| 625 |
CurrentCount = 0; |
| 626 |
BreakContinueStack.push_back(BreakContinue()); |
| 627 |
Visit(S->getBody()); |
| 628 |
// If the switch is inside a loop, add the continue counts. |
| 629 |
BreakContinue BC = BreakContinueStack.pop_back_val(); |
| 630 |
if (!BreakContinueStack.empty()) |
| 631 |
BreakContinueStack.back().ContinueCount += BC.ContinueCount; |
| 632 |
// Counter tracks the exit block of the switch. |
| 633 |
setCount(PGO.getRegionCount(S)); |
| 634 |
RecordNextStmtCount = true; |
| 635 |
} |
| 636 |
|
| 637 |
void VisitSwitchCase(const SwitchCase *S) { |
| 638 |
RecordNextStmtCount = false; |
| 639 |
// Counter for this particular case. This counts only jumps from the |
| 640 |
// switch header and does not include fallthrough from the case before |
| 641 |
// this one. |
| 642 |
uint64_t CaseCount = PGO.getRegionCount(S); |
| 643 |
setCount(CurrentCount + CaseCount); |
| 644 |
// We need the count without fallthrough in the mapping, so it's more useful |
| 645 |
// for branch probabilities. |
| 646 |
CountMap[S] = CaseCount; |
| 647 |
RecordNextStmtCount = true; |
| 648 |
Visit(S->getSubStmt()); |
| 649 |
} |
| 650 |
|
| 651 |
void VisitIfStmt(const IfStmt *S) { |
| 652 |
RecordStmtCount(S); |
| 653 |
|
| 654 |
if (S->isConsteval()) { |
| 655 |
const Stmt *Stm = S->isNegatedConsteval() ? S->getThen() : S->getElse(); |
| 656 |
if (Stm) |
| 657 |
Visit(Stm); |
| 658 |
return; |
| 659 |
} |
| 660 |
|
| 661 |
uint64_t ParentCount = CurrentCount; |
| 662 |
if (S->getInit()) |
| 663 |
Visit(S->getInit()); |
| 664 |
Visit(S->getCond()); |
| 665 |
|
| 666 |
// Counter tracks the "then" part of an if statement. The count for |
| 667 |
// the "else" part, if it exists, will be calculated from this counter. |
| 668 |
uint64_t ThenCount = setCount(PGO.getRegionCount(S)); |
| 669 |
CountMap[S->getThen()] = ThenCount; |
| 670 |
Visit(S->getThen()); |
| 671 |
uint64_t OutCount = CurrentCount; |
| 672 |
|
| 673 |
uint64_t ElseCount = ParentCount - ThenCount; |
| 674 |
if (S->getElse()) { |
| 675 |
setCount(ElseCount); |
| 676 |
CountMap[S->getElse()] = ElseCount; |
| 677 |
Visit(S->getElse()); |
| 678 |
OutCount += CurrentCount; |
| 679 |
} else |
| 680 |
OutCount += ElseCount; |
| 681 |
setCount(OutCount); |
| 682 |
RecordNextStmtCount = true; |
| 683 |
} |
| 684 |
|
| 685 |
void VisitCXXTryStmt(const CXXTryStmt *S) { |
| 686 |
RecordStmtCount(S); |
| 687 |
Visit(S->getTryBlock()); |
| 688 |
for (unsigned I = 0, E = S->getNumHandlers(); I < E; ++I) |
| 689 |
Visit(S->getHandler(I)); |
| 690 |
// Counter tracks the continuation block of the try statement. |
| 691 |
setCount(PGO.getRegionCount(S)); |
| 692 |
RecordNextStmtCount = true; |
| 693 |
} |
| 694 |
|
| 695 |
void VisitCXXCatchStmt(const CXXCatchStmt *S) { |
| 696 |
RecordNextStmtCount = false; |
| 697 |
// Counter tracks the catch statement's handler block. |
| 698 |
uint64_t CatchCount = setCount(PGO.getRegionCount(S)); |
| 699 |
CountMap[S] = CatchCount; |
| 700 |
Visit(S->getHandlerBlock()); |
| 701 |
} |
| 702 |
|
| 703 |
void VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { |
| 704 |
RecordStmtCount(E); |
| 705 |
uint64_t ParentCount = CurrentCount; |
| 706 |
Visit(E->getCond()); |
| 707 |
|
| 708 |
// Counter tracks the "true" part of a conditional operator. The |
| 709 |
// count in the "false" part will be calculated from this counter. |
| 710 |
uint64_t TrueCount = setCount(PGO.getRegionCount(E)); |
| 711 |
CountMap[E->getTrueExpr()] = TrueCount; |
| 712 |
Visit(E->getTrueExpr()); |
| 713 |
uint64_t OutCount = CurrentCount; |
| 714 |
|
| 715 |
uint64_t FalseCount = setCount(ParentCount - TrueCount); |
| 716 |
CountMap[E->getFalseExpr()] = FalseCount; |
| 717 |
Visit(E->getFalseExpr()); |
| 718 |
OutCount += CurrentCount; |
| 719 |
|
| 720 |
setCount(OutCount); |
| 721 |
RecordNextStmtCount = true; |
| 722 |
} |
| 723 |
|
| 724 |
void VisitBinLAnd(const BinaryOperator *E) { |
| 725 |
RecordStmtCount(E); |
| 726 |
uint64_t ParentCount = CurrentCount; |
| 727 |
Visit(E->getLHS()); |
| 728 |
// Counter tracks the right hand side of a logical and operator. |
| 729 |
uint64_t RHSCount = setCount(PGO.getRegionCount(E)); |
| 730 |
CountMap[E->getRHS()] = RHSCount; |
| 731 |
Visit(E->getRHS()); |
| 732 |
setCount(ParentCount + RHSCount - CurrentCount); |
| 733 |
RecordNextStmtCount = true; |
| 734 |
} |
| 735 |
|
| 736 |
void VisitBinLOr(const BinaryOperator *E) { |
| 737 |
RecordStmtCount(E); |
| 738 |
uint64_t ParentCount = CurrentCount; |
| 739 |
Visit(E->getLHS()); |
| 740 |
// Counter tracks the right hand side of a logical or operator. |
| 741 |
uint64_t RHSCount = setCount(PGO.getRegionCount(E)); |
| 742 |
CountMap[E->getRHS()] = RHSCount; |
| 743 |
Visit(E->getRHS()); |
| 744 |
setCount(ParentCount + RHSCount - CurrentCount); |
| 745 |
RecordNextStmtCount = true; |
| 746 |
} |
| 747 |
}; |
| 748 |
} // end anonymous namespace |
| 749 |
|
| 750 |
void PGOHash::combine(HashType Type) { |
| 751 |
// Check that we never combine 0 and only have six bits. |
| 752 |
assert(Type && "Hash is invalid: unexpected type 0"); |
| 753 |
assert(unsigned(Type) < TooBig && "Hash is invalid: too many types"); |
| 754 |
|
| 755 |
// Pass through MD5 if enough work has built up. |
| 756 |
if (Count && Count % NumTypesPerWord == 0) { |
| 757 |
using namespace llvm::support; |
| 758 |
uint64_t Swapped = endian::byte_swap<uint64_t, little>(Working); |
| 759 |
MD5.update(llvm::ArrayRef((uint8_t *)&Swapped, sizeof(Swapped))); |
| 760 |
Working = 0; |
| 761 |
} |
| 762 |
|
| 763 |
// Accumulate the current type. |
| 764 |
++Count; |
| 765 |
Working = Working << NumBitsPerType | Type; |
| 766 |
} |
| 767 |
|
| 768 |
uint64_t PGOHash::finalize() { |
| 769 |
// Use Working as the hash directly if we never used MD5. |
| 770 |
if (Count <= NumTypesPerWord) |
| 771 |
// No need to byte swap here, since none of the math was endian-dependent. |
| 772 |
// This number will be byte-swapped as required on endianness transitions, |
| 773 |
// so we will see the same value on the other side. |
| 774 |
return Working; |
| 775 |
|
| 776 |
// Check for remaining work in Working. |
| 777 |
if (Working) { |
| 778 |
// Keep the buggy behavior from v1 and v2 for backward-compatibility. This |
| 779 |
// is buggy because it converts a uint64_t into an array of uint8_t. |
| 780 |
if (HashVersion < PGO_HASH_V3) { |
| 781 |
MD5.update({(uint8_t)Working}); |
| 782 |
} else { |
| 783 |
using namespace llvm::support; |
| 784 |
uint64_t Swapped = endian::byte_swap<uint64_t, little>(Working); |
| 785 |
MD5.update(llvm::ArrayRef((uint8_t *)&Swapped, sizeof(Swapped))); |
| 786 |
} |
| 787 |
} |
| 788 |
|
| 789 |
// Finalize the MD5 and return the hash. |
| 790 |
llvm::MD5::MD5Result Result; |
| 791 |
MD5.final(Result); |
| 792 |
return Result.low(); |
| 793 |
} |
| 794 |
|
| 795 |
void CodeGenPGO::assignRegionCounters(GlobalDecl GD, llvm::Function *Fn) { |
| 796 |
const Decl *D = GD.getDecl(); |
| 797 |
if (!D->hasBody()) |
| 798 |
return; |
| 799 |
|
| 800 |
// Skip CUDA/HIP kernel launch stub functions. |
| 801 |
if (CGM.getLangOpts().CUDA && !CGM.getLangOpts().CUDAIsDevice && |
| 802 |
D->hasAttr<CUDAGlobalAttr>()) |
| 803 |
return; |
| 804 |
|
| 805 |
bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr(); |
| 806 |
llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader(); |
| 807 |
if (!InstrumentRegions && !PGOReader) |
| 808 |
return; |
| 809 |
if (D->isImplicit()) |
| 810 |
return; |
| 811 |
// Constructors and destructors may be represented by several functions in IR. |
| 812 |
// If so, instrument only base variant, others are implemented by delegation |
| 813 |
// to the base one, it would be counted twice otherwise. |
| 814 |
if (CGM.getTarget().getCXXABI().hasConstructorVariants()) { |
| 815 |
if (const auto *CCD = dyn_cast<CXXConstructorDecl>(D)) |
| 816 |
if (GD.getCtorType() != Ctor_Base && |
| 817 |
CodeGenFunction::IsConstructorDelegationValid(CCD)) |
| 818 |
return; |
| 819 |
} |
| 820 |
if (isa<CXXDestructorDecl>(D) && GD.getDtorType() != Dtor_Base) |
| 821 |
return; |
| 822 |
|
| 823 |
CGM.ClearUnusedCoverageMapping(D); |
| 824 |
if (Fn->hasFnAttribute(llvm::Attribute::NoProfile)) |
| 825 |
return; |
| 826 |
if (Fn->hasFnAttribute(llvm::Attribute::SkipProfile)) |
| 827 |
return; |
| 828 |
|
| 829 |
setFuncName(Fn); |
| 830 |
|
| 831 |
mapRegionCounters(D); |
| 832 |
if (CGM.getCodeGenOpts().CoverageMapping) |
| 833 |
emitCounterRegionMapping(D); |
| 834 |
if (PGOReader) { |
| 835 |
SourceManager &SM = CGM.getContext().getSourceManager(); |
| 836 |
loadRegionCounts(PGOReader, SM.isInMainFile(D->getLocation())); |
| 837 |
computeRegionCounts(D); |
| 838 |
applyFunctionAttributes(PGOReader, Fn); |
| 839 |
} |
| 840 |
} |
| 841 |
|
| 842 |
void CodeGenPGO::mapRegionCounters(const Decl *D) { |
| 843 |
// Use the latest hash version when inserting instrumentation, but use the |
| 844 |
// version in the indexed profile if we're reading PGO data. |
| 845 |
PGOHashVersion HashVersion = PGO_HASH_LATEST; |
| 846 |
uint64_t ProfileVersion = llvm::IndexedInstrProf::Version; |
| 847 |
if (auto *PGOReader = CGM.getPGOReader()) { |
| 848 |
HashVersion = getPGOHashVersion(PGOReader, CGM); |
| 849 |
ProfileVersion = PGOReader->getVersion(); |
| 850 |
} |
| 851 |
|
| 852 |
RegionCounterMap.reset(new llvm::DenseMap<const Stmt *, unsigned>); |
| 853 |
MapRegionCounters Walker(HashVersion, ProfileVersion, *RegionCounterMap); |
| 854 |
if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) |
| 855 |
Walker.TraverseDecl(const_cast<FunctionDecl *>(FD)); |
| 856 |
else if (const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D)) |
| 857 |
Walker.TraverseDecl(const_cast<ObjCMethodDecl *>(MD)); |
| 858 |
else if (const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D)) |
| 859 |
Walker.TraverseDecl(const_cast<BlockDecl *>(BD)); |
| 860 |
else if (const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D)) |
| 861 |
Walker.TraverseDecl(const_cast<CapturedDecl *>(CD)); |
| 862 |
assert(Walker.NextCounter > 0 && "no entry counter mapped for decl"); |
| 863 |
NumRegionCounters = Walker.NextCounter; |
| 864 |
FunctionHash = Walker.Hash.finalize(); |
| 865 |
} |
| 866 |
|
| 867 |
bool CodeGenPGO::skipRegionMappingForDecl(const Decl *D) { |
| 868 |
if (!D->getBody()) |
| 869 |
return true; |
| 870 |
|
| 871 |
// Skip host-only functions in the CUDA device compilation and device-only |
| 872 |
// functions in the host compilation. Just roughly filter them out based on |
| 873 |
// the function attributes. If there are effectively host-only or device-only |
| 874 |
// ones, their coverage mapping may still be generated. |
| 875 |
if (CGM.getLangOpts().CUDA && |
| 876 |
((CGM.getLangOpts().CUDAIsDevice && !D->hasAttr<CUDADeviceAttr>() && |
| 877 |
!D->hasAttr<CUDAGlobalAttr>()) || |
| 878 |
(!CGM.getLangOpts().CUDAIsDevice && |
| 879 |
(D->hasAttr<CUDAGlobalAttr>() || |
| 880 |
(!D->hasAttr<CUDAHostAttr>() && D->hasAttr<CUDADeviceAttr>()))))) |
| 881 |
return true; |
| 882 |
|
| 883 |
// Don't map the functions in system headers. |
| 884 |
const auto &SM = CGM.getContext().getSourceManager(); |
| 885 |
auto Loc = D->getBody()->getBeginLoc(); |
| 886 |
return SM.isInSystemHeader(Loc); |
| 887 |
} |
| 888 |
|
| 889 |
void CodeGenPGO::emitCounterRegionMapping(const Decl *D) { |
| 890 |
if (skipRegionMappingForDecl(D)) |
| 891 |
return; |
| 892 |
|
| 893 |
std::string CoverageMapping; |
| 894 |
llvm::raw_string_ostream OS(CoverageMapping); |
| 895 |
CoverageMappingGen MappingGen(*CGM.getCoverageMapping(), |
| 896 |
CGM.getContext().getSourceManager(), |
| 897 |
CGM.getLangOpts(), RegionCounterMap.get()); |
| 898 |
MappingGen.emitCounterMapping(D, OS); |
| 899 |
OS.flush(); |
| 900 |
|
| 901 |
if (CoverageMapping.empty()) |
| 902 |
return; |
| 903 |
|
| 904 |
CGM.getCoverageMapping()->addFunctionMappingRecord( |
| 905 |
FuncNameVar, FuncName, FunctionHash, CoverageMapping); |
| 906 |
} |
| 907 |
|
| 908 |
void |
| 909 |
CodeGenPGO::emitEmptyCounterMapping(const Decl *D, StringRef Name, |
| 910 |
llvm::GlobalValue::LinkageTypes Linkage) { |
| 911 |
if (skipRegionMappingForDecl(D)) |
| 912 |
return; |
| 913 |
|
| 914 |
std::string CoverageMapping; |
| 915 |
llvm::raw_string_ostream OS(CoverageMapping); |
| 916 |
CoverageMappingGen MappingGen(*CGM.getCoverageMapping(), |
| 917 |
CGM.getContext().getSourceManager(), |
| 918 |
CGM.getLangOpts()); |
| 919 |
MappingGen.emitEmptyMapping(D, OS); |
| 920 |
OS.flush(); |
| 921 |
|
| 922 |
if (CoverageMapping.empty()) |
| 923 |
return; |
| 924 |
|
| 925 |
setFuncName(Name, Linkage); |
| 926 |
CGM.getCoverageMapping()->addFunctionMappingRecord( |
| 927 |
FuncNameVar, FuncName, FunctionHash, CoverageMapping, false); |
| 928 |
} |
| 929 |
|
| 930 |
void CodeGenPGO::computeRegionCounts(const Decl *D) { |
| 931 |
StmtCountMap.reset(new llvm::DenseMap<const Stmt *, uint64_t>); |
| 932 |
ComputeRegionCounts Walker(*StmtCountMap, *this); |
| 933 |
if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) |
| 934 |
Walker.VisitFunctionDecl(FD); |
| 935 |
else if (const ObjCMethodDecl *MD = dyn_cast_or_null<ObjCMethodDecl>(D)) |
| 936 |
Walker.VisitObjCMethodDecl(MD); |
| 937 |
else if (const BlockDecl *BD = dyn_cast_or_null<BlockDecl>(D)) |
| 938 |
Walker.VisitBlockDecl(BD); |
| 939 |
else if (const CapturedDecl *CD = dyn_cast_or_null<CapturedDecl>(D)) |
| 940 |
Walker.VisitCapturedDecl(const_cast<CapturedDecl *>(CD)); |
| 941 |
} |
| 942 |
|
| 943 |
void |
| 944 |
CodeGenPGO::applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, |
| 945 |
llvm::Function *Fn) { |
| 946 |
if (!haveRegionCounts()) |
| 947 |
return; |
| 948 |
|
| 949 |
uint64_t FunctionCount = getRegionCount(nullptr); |
| 950 |
Fn->setEntryCount(FunctionCount); |
| 951 |
} |
| 952 |
|
| 953 |
void CodeGenPGO::emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S, |
| 954 |
llvm::Value *StepV) { |
| 955 |
if (!CGM.getCodeGenOpts().hasProfileClangInstr() || !RegionCounterMap) |
| 956 |
return; |
| 957 |
if (!Builder.GetInsertBlock()) |
| 958 |
return; |
| 959 |
|
| 960 |
unsigned Counter = (*RegionCounterMap)[S]; |
| 961 |
auto *I8PtrTy = llvm::Type::getInt8PtrTy(CGM.getLLVMContext()); |
| 962 |
|
| 963 |
llvm::Value *Args[] = {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy), |
| 964 |
Builder.getInt64(FunctionHash), |
| 965 |
Builder.getInt32(NumRegionCounters), |
| 966 |
Builder.getInt32(Counter), StepV}; |
| 967 |
if (!StepV) |
| 968 |
Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment), |
| 969 |
ArrayRef(Args, 4)); |
| 970 |
else |
| 971 |
Builder.CreateCall( |
| 972 |
CGM.getIntrinsic(llvm::Intrinsic::instrprof_increment_step), |
| 973 |
ArrayRef(Args)); |
| 974 |
} |
| 975 |
|
| 976 |
void CodeGenPGO::setValueProfilingFlag(llvm::Module &M) { |
| 977 |
if (CGM.getCodeGenOpts().hasProfileClangInstr()) |
| 978 |
M.addModuleFlag(llvm::Module::Warning, "EnableValueProfiling", |
| 979 |
uint32_t(EnableValueProfiling)); |
| 980 |
} |
| 981 |
|
| 982 |
// This method either inserts a call to the profile run-time during |
| 983 |
// instrumentation or puts profile data into metadata for PGO use. |
| 984 |
void CodeGenPGO::valueProfile(CGBuilderTy &Builder, uint32_t ValueKind, |
| 985 |
llvm::Instruction *ValueSite, llvm::Value *ValuePtr) { |
| 986 |
|
| 987 |
if (!EnableValueProfiling) |
| 988 |
return; |
| 989 |
|
| 990 |
if (!ValuePtr || !ValueSite || !Builder.GetInsertBlock()) |
| 991 |
return; |
| 992 |
|
| 993 |
if (isa<llvm::Constant>(ValuePtr)) |
| 994 |
return; |
| 995 |
|
| 996 |
bool InstrumentValueSites = CGM.getCodeGenOpts().hasProfileClangInstr(); |
| 997 |
if (InstrumentValueSites && RegionCounterMap) { |
| 998 |
auto BuilderInsertPoint = Builder.saveIP(); |
| 999 |
Builder.SetInsertPoint(ValueSite); |
| 1000 |
llvm::Value *Args[5] = { |
| 1001 |
llvm::ConstantExpr::getBitCast(FuncNameVar, Builder.getInt8PtrTy()), |
| 1002 |
Builder.getInt64(FunctionHash), |
| 1003 |
Builder.CreatePtrToInt(ValuePtr, Builder.getInt64Ty()), |
| 1004 |
Builder.getInt32(ValueKind), |
| 1005 |
Builder.getInt32(NumValueSites[ValueKind]++) |
| 1006 |
}; |
| 1007 |
Builder.CreateCall( |
| 1008 |
CGM.getIntrinsic(llvm::Intrinsic::instrprof_value_profile), Args); |
| 1009 |
Builder.restoreIP(BuilderInsertPoint); |
| 1010 |
return; |
| 1011 |
} |
| 1012 |
|
| 1013 |
llvm::IndexedInstrProfReader *PGOReader = CGM.getPGOReader(); |
| 1014 |
if (PGOReader && haveRegionCounts()) { |
| 1015 |
// We record the top most called three functions at each call site. |
| 1016 |
// Profile metadata contains "VP" string identifying this metadata |
| 1017 |
// as value profiling data, then a uint32_t value for the value profiling |
| 1018 |
// kind, a uint64_t value for the total number of times the call is |
| 1019 |
// executed, followed by the function hash and execution count (uint64_t) |
| 1020 |
// pairs for each function. |
| 1021 |
if (NumValueSites[ValueKind] >= ProfRecord->getNumValueSites(ValueKind)) |
| 1022 |
return; |
| 1023 |
|
| 1024 |
llvm::annotateValueSite(CGM.getModule(), *ValueSite, *ProfRecord, |
| 1025 |
(llvm::InstrProfValueKind)ValueKind, |
| 1026 |
NumValueSites[ValueKind]); |
| 1027 |
|
| 1028 |
NumValueSites[ValueKind]++; |
| 1029 |
} |
| 1030 |
} |
| 1031 |
|
| 1032 |
void CodeGenPGO::loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, |
| 1033 |
bool IsInMainFile) { |
| 1034 |
CGM.getPGOStats().addVisited(IsInMainFile); |
| 1035 |
RegionCounts.clear(); |
| 1036 |
llvm::Expected<llvm::InstrProfRecord> RecordExpected = |
| 1037 |
PGOReader->getInstrProfRecord(FuncName, FunctionHash); |
| 1038 |
if (auto E = RecordExpected.takeError()) { |
| 1039 |
auto IPE = llvm::InstrProfError::take(std::move(E)); |
| 1040 |
if (IPE == llvm::instrprof_error::unknown_function) |
| 1041 |
CGM.getPGOStats().addMissing(IsInMainFile); |
| 1042 |
else if (IPE == llvm::instrprof_error::hash_mismatch) |
| 1043 |
CGM.getPGOStats().addMismatched(IsInMainFile); |
| 1044 |
else if (IPE == llvm::instrprof_error::malformed) |
| 1045 |
// TODO: Consider a more specific warning for this case. |
| 1046 |
CGM.getPGOStats().addMismatched(IsInMainFile); |
| 1047 |
return; |
| 1048 |
} |
| 1049 |
ProfRecord = |
| 1050 |
std::make_unique<llvm::InstrProfRecord>(std::move(RecordExpected.get())); |
| 1051 |
RegionCounts = ProfRecord->Counts; |
| 1052 |
} |
| 1053 |
|
| 1054 |
/// Calculate what to divide by to scale weights. |
| 1055 |
/// |
| 1056 |
/// Given the maximum weight, calculate a divisor that will scale all the |
| 1057 |
/// weights to strictly less than UINT32_MAX. |
| 1058 |
static uint64_t calculateWeightScale(uint64_t MaxWeight) { |
| 1059 |
return MaxWeight < UINT32_MAX ? 1 : MaxWeight / UINT32_MAX + 1; |
| 1060 |
} |
| 1061 |
|
| 1062 |
/// Scale an individual branch weight (and add 1). |
| 1063 |
/// |
| 1064 |
/// Scale a 64-bit weight down to 32-bits using \c Scale. |
| 1065 |
/// |
| 1066 |
/// According to Laplace's Rule of Succession, it is better to compute the |
| 1067 |
/// weight based on the count plus 1, so universally add 1 to the value. |
| 1068 |
/// |
| 1069 |
/// \pre \c Scale was calculated by \a calculateWeightScale() with a weight no |
| 1070 |
/// greater than \c Weight. |
| 1071 |
static uint32_t scaleBranchWeight(uint64_t Weight, uint64_t Scale) { |
| 1072 |
assert(Scale && "scale by 0?"); |
| 1073 |
uint64_t Scaled = Weight / Scale + 1; |
| 1074 |
assert(Scaled <= UINT32_MAX && "overflow 32-bits"); |
| 1075 |
return Scaled; |
| 1076 |
} |
| 1077 |
|
| 1078 |
llvm::MDNode *CodeGenFunction::createProfileWeights(uint64_t TrueCount, |
| 1079 |
uint64_t FalseCount) const { |
| 1080 |
// Check for empty weights. |
| 1081 |
if (!TrueCount && !FalseCount) |
| 1082 |
return nullptr; |
| 1083 |
|
| 1084 |
// Calculate how to scale down to 32-bits. |
| 1085 |
uint64_t Scale = calculateWeightScale(std::max(TrueCount, FalseCount)); |
| 1086 |
|
| 1087 |
llvm::MDBuilder MDHelper(CGM.getLLVMContext()); |
| 1088 |
return MDHelper.createBranchWeights(scaleBranchWeight(TrueCount, Scale), |
| 1089 |
scaleBranchWeight(FalseCount, Scale)); |
| 1090 |
} |
| 1091 |
|
| 1092 |
llvm::MDNode * |
| 1093 |
CodeGenFunction::createProfileWeights(ArrayRef<uint64_t> Weights) const { |
| 1094 |
// We need at least two elements to create meaningful weights. |
| 1095 |
if (Weights.size() < 2) |
| 1096 |
return nullptr; |
| 1097 |
|
| 1098 |
// Check for empty weights. |
| 1099 |
uint64_t MaxWeight = *std::max_element(Weights.begin(), Weights.end()); |
| 1100 |
if (MaxWeight == 0) |
| 1101 |
return nullptr; |
| 1102 |
|
| 1103 |
// Calculate how to scale down to 32-bits. |
| 1104 |
uint64_t Scale = calculateWeightScale(MaxWeight); |
| 1105 |
|
| 1106 |
SmallVector<uint32_t, 16> ScaledWeights; |
| 1107 |
ScaledWeights.reserve(Weights.size()); |
| 1108 |
for (uint64_t W : Weights) |
| 1109 |
ScaledWeights.push_back(scaleBranchWeight(W, Scale)); |
| 1110 |
|
| 1111 |
llvm::MDBuilder MDHelper(CGM.getLLVMContext()); |
| 1112 |
return MDHelper.createBranchWeights(ScaledWeights); |
| 1113 |
} |
| 1114 |
|
| 1115 |
llvm::MDNode * |
| 1116 |
CodeGenFunction::createProfileWeightsForLoop(const Stmt *Cond, |
| 1117 |
uint64_t LoopCount) const { |
| 1118 |
if (!PGO.haveRegionCounts()) |
| 1119 |
return nullptr; |
| 1120 |
std::optional<uint64_t> CondCount = PGO.getStmtCount(Cond); |
| 1121 |
if (!CondCount || *CondCount == 0) |
| 1122 |
return nullptr; |
| 1123 |
return createProfileWeights(LoopCount, |
| 1124 |
std::max(*CondCount, LoopCount) - LoopCount); |
| 1125 |
} |