| 1 |
//===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===// |
| 2 |
// |
| 3 |
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 |
// See https://llvm.org/LICENSE.txt for license information. |
| 5 |
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 |
// |
| 7 |
//===----------------------------------------------------------------------===// |
| 8 |
// |
| 9 |
// Instrumentation-based profile-guided optimization |
| 10 |
// |
| 11 |
//===----------------------------------------------------------------------===// |
| 12 |
|
| 13 |
#ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H |
| 14 |
#define LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H |
| 15 |
|
| 16 |
#include "CGBuilder.h" |
| 17 |
#include "CodeGenModule.h" |
| 18 |
#include "CodeGenTypes.h" |
| 19 |
#include "llvm/ProfileData/InstrProfReader.h" |
| 20 |
#include <array> |
| 21 |
#include <memory> |
| 22 |
#include <optional> |
| 23 |
|
| 24 |
namespace clang { |
| 25 |
namespace CodeGen { |
| 26 |
|
| 27 |
/// Per-function PGO state. |
| 28 |
class CodeGenPGO { |
| 29 |
private: |
| 30 |
CodeGenModule &CGM; |
| 31 |
std::string FuncName; |
| 32 |
llvm::GlobalVariable *FuncNameVar; |
| 33 |
|
| 34 |
std::array <unsigned, llvm::IPVK_Last + 1> NumValueSites; |
| 35 |
unsigned NumRegionCounters; |
| 36 |
uint64_t FunctionHash; |
| 37 |
std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap; |
| 38 |
std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap; |
| 39 |
std::unique_ptr<llvm::InstrProfRecord> ProfRecord; |
| 40 |
std::vector<uint64_t> RegionCounts; |
| 41 |
uint64_t CurrentRegionCount; |
| 42 |
|
| 43 |
public: |
| 44 |
CodeGenPGO(CodeGenModule &CGModule) |
| 45 |
: CGM(CGModule), FuncNameVar(nullptr), NumValueSites({{0}}), |
| 46 |
NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0) {} |
| 47 |
|
| 48 |
/// Whether or not we have PGO region data for the current function. This is |
| 49 |
/// false both when we have no data at all and when our data has been |
| 50 |
/// discarded. |
| 51 |
bool haveRegionCounts() const { return !RegionCounts.empty(); } |
| 52 |
|
| 53 |
/// Return the counter value of the current region. |
| 54 |
uint64_t getCurrentRegionCount() const { return CurrentRegionCount; } |
| 55 |
|
| 56 |
/// Set the counter value for the current region. This is used to keep track |
| 57 |
/// of changes to the most recent counter from control flow and non-local |
| 58 |
/// exits. |
| 59 |
void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; } |
| 60 |
|
| 61 |
/// Check if an execution count is known for a given statement. If so, return |
| 62 |
/// true and put the value in Count; else return false. |
| 63 |
std::optional<uint64_t> getStmtCount(const Stmt *S) const { |
| 64 |
if (!StmtCountMap) |
| 65 |
return std::nullopt; |
| 66 |
auto I = StmtCountMap->find(S); |
| 67 |
if (I == StmtCountMap->end()) |
| 68 |
return std::nullopt; |
| 69 |
return I->second; |
| 70 |
} |
| 71 |
|
| 72 |
/// If the execution count for the current statement is known, record that |
| 73 |
/// as the current count. |
| 74 |
void setCurrentStmt(const Stmt *S) { |
| 75 |
if (auto Count = getStmtCount(S)) |
| 76 |
setCurrentRegionCount(*Count); |
| 77 |
} |
| 78 |
|
| 79 |
/// Assign counters to regions and configure them for PGO of a given |
| 80 |
/// function. Does nothing if instrumentation is not enabled and either |
| 81 |
/// generates global variables or associates PGO data with each of the |
| 82 |
/// counters depending on whether we are generating or using instrumentation. |
| 83 |
void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn); |
| 84 |
/// Emit a coverage mapping range with a counter zero |
| 85 |
/// for an unused declaration. |
| 86 |
void emitEmptyCounterMapping(const Decl *D, StringRef FuncName, |
| 87 |
llvm::GlobalValue::LinkageTypes Linkage); |
| 88 |
// Insert instrumentation or attach profile metadata at value sites |
| 89 |
void valueProfile(CGBuilderTy &Builder, uint32_t ValueKind, |
| 90 |
llvm::Instruction *ValueSite, llvm::Value *ValuePtr); |
| 91 |
|
| 92 |
// Set a module flag indicating if value profiling is enabled. |
| 93 |
void setValueProfilingFlag(llvm::Module &M); |
| 94 |
|
| 95 |
private: |
| 96 |
void setFuncName(llvm::Function *Fn); |
| 97 |
void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage); |
| 98 |
void mapRegionCounters(const Decl *D); |
| 99 |
void computeRegionCounts(const Decl *D); |
| 100 |
void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader, |
| 101 |
llvm::Function *Fn); |
| 102 |
void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader, |
| 103 |
bool IsInMainFile); |
| 104 |
bool skipRegionMappingForDecl(const Decl *D); |
| 105 |
void emitCounterRegionMapping(const Decl *D); |
| 106 |
|
| 107 |
public: |
| 108 |
void emitCounterIncrement(CGBuilderTy &Builder, const Stmt *S, |
| 109 |
llvm::Value *StepV); |
| 110 |
|
| 111 |
/// Return the region count for the counter at the given index. |
| 112 |
uint64_t getRegionCount(const Stmt *S) { |
| 113 |
if (!RegionCounterMap) |
| 114 |
return 0; |
| 115 |
if (!haveRegionCounts()) |
| 116 |
return 0; |
| 117 |
return RegionCounts[(*RegionCounterMap)[S]]; |
| 118 |
} |
| 119 |
}; |
| 120 |
|
| 121 |
} // end namespace CodeGen |
| 122 |
} // end namespace clang |
| 123 |
|
| 124 |
#endif |