| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* llvmjit_inline.cpp |
| 4 |
* Cross module inlining suitable for postgres' JIT |
| 5 |
* |
| 6 |
* The inliner iterates over external functions referenced from the passed |
| 7 |
* module and attempts to inline those. It does so by utilizing pre-built |
| 8 |
* indexes over both postgres core code and extension modules. When a match |
| 9 |
* for an external function is found - not guaranteed! - the index will then |
| 10 |
* be used to judge their instruction count / inline worthiness. After doing |
| 11 |
* so for all external functions, all the referenced functions (and |
| 12 |
* prerequisites) will be imported. |
| 13 |
* |
| 14 |
* Copyright (c) 2016-2025, PostgreSQL Global Development Group |
| 15 |
* |
| 16 |
* IDENTIFICATION |
| 17 |
* src/backend/lib/llvmjit/llvmjit_inline.cpp |
| 18 |
* |
| 19 |
*------------------------------------------------------------------------- |
| 20 |
*/ |
| 21 |
|
| 22 |
extern "C" |
| 23 |
{ |
| 24 |
#include "postgres.h" |
| 25 |
} |
| 26 |
|
| 27 |
#include "jit/llvmjit.h" |
| 28 |
|
| 29 |
extern "C" |
| 30 |
{ |
| 31 |
#include <fcntl.h> |
| 32 |
#include <sys/mman.h> |
| 33 |
#include <sys/stat.h> |
| 34 |
#include <sys/types.h> |
| 35 |
#include <unistd.h> |
| 36 |
|
| 37 |
#include "common/string.h" |
| 38 |
#include "miscadmin.h" |
| 39 |
#include "storage/fd.h" |
| 40 |
} |
| 41 |
|
| 42 |
#include <llvm-c/Core.h> |
| 43 |
#include <llvm-c/BitReader.h> |
| 44 |
|
| 45 |
/* Avoid macro clash with LLVM's C++ headers */ |
| 46 |
#undef Min |
| 47 |
|
| 48 |
#include <llvm/ADT/SetVector.h> |
| 49 |
#include <llvm/ADT/StringSet.h> |
| 50 |
#include <llvm/ADT/StringMap.h> |
| 51 |
#include <llvm/Analysis/ModuleSummaryAnalysis.h> |
| 52 |
#include <llvm/Bitcode/BitcodeReader.h> |
| 53 |
#include <llvm/IR/Attributes.h> |
| 54 |
#include <llvm/IR/DebugInfo.h> |
| 55 |
#include <llvm/IR/IntrinsicInst.h> |
| 56 |
#include <llvm/IR/IRBuilder.h> |
| 57 |
#include <llvm/IR/ModuleSummaryIndex.h> |
| 58 |
#include <llvm/Linker/IRMover.h> |
| 59 |
#include <llvm/Support/ManagedStatic.h> |
| 60 |
#include <llvm/Support/MemoryBuffer.h> |
| 61 |
|
| 62 |
|
| 63 |
/* |
| 64 |
* Type used to represent modules InlineWorkListItem's subject is searched for |
| 65 |
* in. |
| 66 |
*/ |
| 67 |
typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath; |
| 68 |
|
| 69 |
/* |
| 70 |
* Item in queue of to-be-checked symbols and corresponding queue. |
| 71 |
*/ |
| 72 |
typedef struct InlineWorkListItem |
| 73 |
{ |
| 74 |
llvm::StringRef symbolName; |
| 75 |
llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath; |
| 76 |
} InlineWorkListItem; |
| 77 |
typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList; |
| 78 |
|
| 79 |
/* |
| 80 |
* Information about symbols processed during inlining. Used to prevent |
| 81 |
* repeated searches and provide additional information. |
| 82 |
*/ |
| 83 |
typedef struct FunctionInlineState |
| 84 |
{ |
| 85 |
int costLimit; |
| 86 |
bool processed; |
| 87 |
bool inlined; |
| 88 |
bool allowReconsidering; |
| 89 |
} FunctionInlineState; |
| 90 |
typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates; |
| 91 |
|
| 92 |
/* |
| 93 |
* Map of modules that should be inlined, with a list of the to-be inlined |
| 94 |
* symbols. |
| 95 |
*/ |
| 96 |
typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy; |
| 97 |
|
| 98 |
|
| 99 |
const float inline_cost_decay_factor = 0.5; |
| 100 |
const int inline_initial_cost = 150; |
| 101 |
|
| 102 |
/* |
| 103 |
* These are managed statics so LLVM knows to deallocate them during an |
| 104 |
* LLVMShutdown(), rather than after (which'd cause crashes). |
| 105 |
*/ |
| 106 |
typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache; |
| 107 |
llvm::ManagedStatic<ModuleCache> module_cache; |
| 108 |
typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache; |
| 109 |
llvm::ManagedStatic<SummaryCache> summary_cache; |
| 110 |
|
| 111 |
|
| 112 |
static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod); |
| 113 |
static void llvm_execute_inline_plan(llvm::Module *mod, |
| 114 |
ImportMapTy *globalsToInline); |
| 115 |
|
| 116 |
static llvm::Module* load_module_cached(LLVMContextRef c, llvm::StringRef modPath); |
| 117 |
static std::unique_ptr<llvm::Module> load_module(LLVMContextRef c, llvm::StringRef Identifier); |
| 118 |
static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path); |
| 119 |
|
| 120 |
|
| 121 |
static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod, |
| 122 |
llvm::Function *F, |
| 123 |
llvm::StringRef Name); |
| 124 |
|
| 125 |
static bool function_inlinable(llvm::Function &F, |
| 126 |
int threshold, |
| 127 |
FunctionInlineStates &functionState, |
| 128 |
InlineWorkList &worklist, |
| 129 |
InlineSearchPath &searchpath, |
| 130 |
llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions, |
| 131 |
int &running_instcount, |
| 132 |
llvm::StringSet<> &importVars); |
| 133 |
static void function_references(llvm::Function &F, |
| 134 |
int &running_instcount, |
| 135 |
llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars, |
| 136 |
llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions); |
| 137 |
|
| 138 |
static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath); |
| 139 |
static llvm::SmallVector<llvm::GlobalValueSummary *, 1> |
| 140 |
summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid); |
| 141 |
|
| 142 |
/* verbose debugging for inliner development */ |
| 143 |
/* #define INLINE_DEBUG */ |
| 144 |
#ifdef INLINE_DEBUG |
| 145 |
#define ilog elog |
| 146 |
#else |
| 147 |
#define ilog(...) (void) 0 |
| 148 |
#endif |
| 149 |
|
| 150 |
/* |
| 151 |
* Reset inlining related state. This needs to be called before the currently |
| 152 |
* used LLVMContextRef is disposed (and a new one create), otherwise we would |
| 153 |
* have dangling references to deleted modules. |
| 154 |
*/ |
| 155 |
void |
| 156 |
llvm_inline_reset_caches(void) |
| 157 |
{ |
| 158 |
module_cache->clear(); |
| 159 |
summary_cache->clear(); |
| 160 |
} |
| 161 |
|
| 162 |
/* |
| 163 |
* Perform inlining of external function references in M based on a simple |
| 164 |
* cost based analysis. |
| 165 |
*/ |
| 166 |
void |
| 167 |
llvm_inline(LLVMModuleRef M) |
| 168 |
{ |
| 169 |
LLVMContextRef lc = LLVMGetModuleContext(M); |
| 170 |
llvm::Module *mod = llvm::unwrap(M); |
| 171 |
|
| 172 |
std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod); |
| 173 |
if (!globalsToInline) |
| 174 |
return; |
| 175 |
llvm_execute_inline_plan(mod, globalsToInline.get()); |
| 176 |
} |
| 177 |
|
| 178 |
/* |
| 179 |
* Build information necessary for inlining external function references in |
| 180 |
* mod. |
| 181 |
*/ |
| 182 |
static std::unique_ptr<ImportMapTy> |
| 183 |
llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod) |
| 184 |
{ |
| 185 |
std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy()); |
| 186 |
FunctionInlineStates functionStates; |
| 187 |
InlineWorkList worklist; |
| 188 |
|
| 189 |
InlineSearchPath defaultSearchPath; |
| 190 |
|
| 191 |
/* attempt to add module to search path */ |
| 192 |
add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres"); |
| 193 |
/* if postgres isn't available, no point continuing */ |
| 194 |
if (defaultSearchPath.empty()) |
| 195 |
return nullptr; |
| 196 |
|
| 197 |
/* |
| 198 |
* Start inlining with current references to external functions by putting |
| 199 |
* them on the inlining worklist. If, during inlining of those, new extern |
| 200 |
* functions need to be inlined, they'll also be put there, with a lower |
| 201 |
* priority. |
| 202 |
*/ |
| 203 |
for (const llvm::Function &funcDecl : mod->functions()) |
| 204 |
{ |
| 205 |
InlineWorkListItem item = {}; |
| 206 |
FunctionInlineState inlineState = {}; |
| 207 |
|
| 208 |
/* already has a definition */ |
| 209 |
if (!funcDecl.isDeclaration()) |
| 210 |
continue; |
| 211 |
|
| 212 |
/* llvm provides implementation */ |
| 213 |
if (funcDecl.isIntrinsic()) |
| 214 |
continue; |
| 215 |
|
| 216 |
item.symbolName = funcDecl.getName(); |
| 217 |
item.searchpath = defaultSearchPath; |
| 218 |
worklist.push_back(item); |
| 219 |
inlineState.costLimit = inline_initial_cost; |
| 220 |
inlineState.processed = false; |
| 221 |
inlineState.inlined = false; |
| 222 |
inlineState.allowReconsidering = false; |
| 223 |
functionStates[funcDecl.getName()] = inlineState; |
| 224 |
} |
| 225 |
|
| 226 |
/* |
| 227 |
* Iterate over pending worklist items, look them up in index, check |
| 228 |
* whether they should be inlined. |
| 229 |
*/ |
| 230 |
while (!worklist.empty()) |
| 231 |
{ |
| 232 |
InlineWorkListItem item = worklist.pop_back_val(); |
| 233 |
llvm::StringRef symbolName = item.symbolName; |
| 234 |
char *cmodname; |
| 235 |
char *cfuncname; |
| 236 |
FunctionInlineState &inlineState = functionStates[symbolName]; |
| 237 |
llvm::GlobalValue::GUID funcGUID; |
| 238 |
|
| 239 |
llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname); |
| 240 |
|
| 241 |
#if LLVM_VERSION_MAJOR >= 21 |
| 242 |
funcGUID = llvm::GlobalValue::getGUIDAssumingExternalLinkage(cfuncname); |
| 243 |
#else |
| 244 |
funcGUID = llvm::GlobalValue::getGUID(cfuncname); |
| 245 |
#endif |
| 246 |
|
| 247 |
/* already processed */ |
| 248 |
if (inlineState.processed) |
| 249 |
continue; |
| 250 |
|
| 251 |
|
| 252 |
if (cmodname) |
| 253 |
add_module_to_inline_search_path(item.searchpath, cmodname); |
| 254 |
|
| 255 |
/* |
| 256 |
* Iterate over all known definitions of function, via the index. Then |
| 257 |
* look up module(s), check if function actually is defined (there |
| 258 |
* could be hash conflicts). |
| 259 |
*/ |
| 260 |
for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID)) |
| 261 |
{ |
| 262 |
const llvm::FunctionSummary *fs; |
| 263 |
llvm::StringRef modPath = gvs->modulePath(); |
| 264 |
llvm::Module *defMod; |
| 265 |
llvm::Function *funcDef; |
| 266 |
|
| 267 |
fs = llvm::cast<llvm::FunctionSummary>(gvs); |
| 268 |
|
| 269 |
if (gvs->notEligibleToImport()) |
| 270 |
{ |
| 271 |
ilog(DEBUG1, "ineligibile to import %s due to summary", |
| 272 |
symbolName.data()); |
| 273 |
continue; |
| 274 |
} |
| 275 |
|
| 276 |
if ((int) fs->instCount() > inlineState.costLimit) |
| 277 |
{ |
| 278 |
ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u", |
| 279 |
symbolName.data(), fs->instCount(), inlineState.costLimit); |
| 280 |
inlineState.allowReconsidering = true; |
| 281 |
continue; |
| 282 |
} |
| 283 |
|
| 284 |
defMod = load_module_cached(lc, modPath); |
| 285 |
if (defMod->materializeMetadata()) |
| 286 |
elog(FATAL, "failed to materialize metadata"); |
| 287 |
|
| 288 |
funcDef = defMod->getFunction(cfuncname); |
| 289 |
|
| 290 |
/* |
| 291 |
* This can happen e.g. in case of a hash collision of the |
| 292 |
* function's name. |
| 293 |
*/ |
| 294 |
if (!funcDef) |
| 295 |
continue; |
| 296 |
|
| 297 |
if (funcDef->materialize()) |
| 298 |
elog(FATAL, "failed to materialize metadata"); |
| 299 |
|
| 300 |
Assert(!funcDef->isDeclaration()); |
| 301 |
Assert(funcDef->hasExternalLinkage()); |
| 302 |
|
| 303 |
llvm::StringSet<> importVars; |
| 304 |
llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions; |
| 305 |
int running_instcount = 0; |
| 306 |
|
| 307 |
/* |
| 308 |
* Check whether function, and objects it depends on, are |
| 309 |
* inlinable. |
| 310 |
*/ |
| 311 |
if (function_inlinable(*funcDef, |
| 312 |
inlineState.costLimit, |
| 313 |
functionStates, |
| 314 |
worklist, |
| 315 |
item.searchpath, |
| 316 |
visitedFunctions, |
| 317 |
running_instcount, |
| 318 |
importVars)) |
| 319 |
{ |
| 320 |
/* |
| 321 |
* Check whether function and all its dependencies are too |
| 322 |
* big. Dependencies already counted for other functions that |
| 323 |
* will get inlined are not counted again. While this make |
| 324 |
* things somewhat order dependent, I can't quite see a point |
| 325 |
* in a different behaviour. |
| 326 |
*/ |
| 327 |
if (running_instcount > inlineState.costLimit) |
| 328 |
{ |
| 329 |
ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d", |
| 330 |
symbolName.data(), running_instcount, inlineState.costLimit); |
| 331 |
inlineState.allowReconsidering = true; |
| 332 |
continue; |
| 333 |
} |
| 334 |
|
| 335 |
ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d", |
| 336 |
symbolName.data(), running_instcount, fs->instCount()); |
| 337 |
|
| 338 |
/* import referenced function itself */ |
| 339 |
importVars.insert(symbolName); |
| 340 |
|
| 341 |
{ |
| 342 |
llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath]; |
| 343 |
for (auto& importVar : importVars) |
| 344 |
modGlobalsToInline.insert(importVar.first()); |
| 345 |
Assert(modGlobalsToInline.size() > 0); |
| 346 |
} |
| 347 |
|
| 348 |
/* mark function as inlined */ |
| 349 |
inlineState.inlined = true; |
| 350 |
|
| 351 |
/* |
| 352 |
* Found definition to inline, don't look for further |
| 353 |
* potential definitions. |
| 354 |
*/ |
| 355 |
break; |
| 356 |
} |
| 357 |
else |
| 358 |
{ |
| 359 |
ilog(DEBUG1, "had to skip inlining %s", |
| 360 |
symbolName.data()); |
| 361 |
|
| 362 |
/* It's possible there's another definition that's inlinable. */ |
| 363 |
} |
| 364 |
} |
| 365 |
|
| 366 |
/* |
| 367 |
* Signal that we're done with symbol, whether successful (inlined = |
| 368 |
* true above) or not. |
| 369 |
*/ |
| 370 |
inlineState.processed = true; |
| 371 |
} |
| 372 |
|
| 373 |
return globalsToInline; |
| 374 |
} |
| 375 |
|
| 376 |
/* |
| 377 |
* Perform the actual inlining of external functions (and their dependencies) |
| 378 |
* into mod. |
| 379 |
*/ |
| 380 |
static void |
| 381 |
llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline) |
| 382 |
{ |
| 383 |
llvm::IRMover Mover(*mod); |
| 384 |
|
| 385 |
for (const auto& toInline : *globalsToInline) |
| 386 |
{ |
| 387 |
const llvm::StringRef& modPath = toInline.first(); |
| 388 |
const llvm::StringSet<>& modGlobalsToInline = toInline.second; |
| 389 |
llvm::SetVector<llvm::GlobalValue *> GlobalsToImport; |
| 390 |
|
| 391 |
Assert(module_cache->count(modPath)); |
| 392 |
std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath])); |
| 393 |
module_cache->erase(modPath); |
| 394 |
|
| 395 |
if (modGlobalsToInline.empty()) |
| 396 |
continue; |
| 397 |
|
| 398 |
for (auto &glob: modGlobalsToInline) |
| 399 |
{ |
| 400 |
llvm::StringRef SymbolName = glob.first(); |
| 401 |
char *modname; |
| 402 |
char *funcname; |
| 403 |
|
| 404 |
llvm_split_symbol_name(SymbolName.data(), &modname, &funcname); |
| 405 |
|
| 406 |
llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname); |
| 407 |
|
| 408 |
if (!valueToImport) |
| 409 |
elog(FATAL, "didn't refind value %s to import", SymbolName.data()); |
| 410 |
|
| 411 |
/* |
| 412 |
* For functions (global vars are only inlined if already static), |
| 413 |
* mark imported variables as being clones from other |
| 414 |
* functions. That a) avoids symbol conflicts b) allows the |
| 415 |
* optimizer to perform inlining. |
| 416 |
*/ |
| 417 |
if (llvm::isa<llvm::Function>(valueToImport)) |
| 418 |
{ |
| 419 |
llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport); |
| 420 |
typedef llvm::GlobalValue::LinkageTypes LinkageTypes; |
| 421 |
|
| 422 |
/* |
| 423 |
* Per-function info isn't necessarily stripped yet, as the |
| 424 |
* module is lazy-loaded when stripped above. |
| 425 |
*/ |
| 426 |
llvm::stripDebugInfo(*F); |
| 427 |
|
| 428 |
/* |
| 429 |
* If the to-be-imported function is one referenced including |
| 430 |
* its module name, create a tiny inline function that just |
| 431 |
* forwards the call. One might think a GlobalAlias would do |
| 432 |
* the trick, but a) IRMover doesn't override a declaration |
| 433 |
* with an alias pointing to a definition (instead renaming |
| 434 |
* it), b) Aliases can't be AvailableExternally. |
| 435 |
*/ |
| 436 |
if (modname) |
| 437 |
{ |
| 438 |
llvm::Function *AF; |
| 439 |
|
| 440 |
AF = create_redirection_function(importMod, F, SymbolName); |
| 441 |
|
| 442 |
GlobalsToImport.insert(AF); |
| 443 |
llvm::stripDebugInfo(*AF); |
| 444 |
} |
| 445 |
|
| 446 |
if (valueToImport->hasExternalLinkage()) |
| 447 |
{ |
| 448 |
valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage); |
| 449 |
} |
| 450 |
} |
| 451 |
|
| 452 |
GlobalsToImport.insert(valueToImport); |
| 453 |
ilog(DEBUG1, "performing import of %s %s", |
| 454 |
modPath.data(), SymbolName.data()); |
| 455 |
|
| 456 |
} |
| 457 |
|
| 458 |
if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(), |
| 459 |
[](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}, |
| 460 |
/*IsPerformingImport=*/false)) |
| 461 |
elog(FATAL, "function import failed with linker error"); |
| 462 |
} |
| 463 |
} |
| 464 |
|
| 465 |
/* |
| 466 |
* Return a module identified by modPath, caching it in memory. |
| 467 |
* |
| 468 |
* Note that such a module may *not* be modified without copying, otherwise |
| 469 |
* the cache state would get corrupted. |
| 470 |
*/ |
| 471 |
static llvm::Module* |
| 472 |
load_module_cached(LLVMContextRef lc, llvm::StringRef modPath) |
| 473 |
{ |
| 474 |
auto it = module_cache->find(modPath); |
| 475 |
if (it == module_cache->end()) |
| 476 |
{ |
| 477 |
it = module_cache->insert( |
| 478 |
std::make_pair(modPath, load_module(lc, modPath))).first; |
| 479 |
} |
| 480 |
|
| 481 |
return it->second.get(); |
| 482 |
} |
| 483 |
|
| 484 |
static std::unique_ptr<llvm::Module> |
| 485 |
load_module(LLVMContextRef lc, llvm::StringRef Identifier) |
| 486 |
{ |
| 487 |
LLVMMemoryBufferRef buf; |
| 488 |
LLVMModuleRef mod; |
| 489 |
char path[MAXPGPATH]; |
| 490 |
char *msg; |
| 491 |
|
| 492 |
snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data()); |
| 493 |
|
| 494 |
if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg)) |
| 495 |
elog(FATAL, "failed to open bitcode file \"%s\": %s", |
| 496 |
path, msg); |
| 497 |
if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod)) |
| 498 |
elog(FATAL, "failed to parse bitcode in file \"%s\"", path); |
| 499 |
|
| 500 |
/* |
| 501 |
* Currently there's no use in more detailed debug info for JITed |
| 502 |
* code. Until that changes, not much point in wasting memory and cycles |
| 503 |
* on processing debuginfo. |
| 504 |
*/ |
| 505 |
llvm::StripDebugInfo(*llvm::unwrap(mod)); |
| 506 |
|
| 507 |
return std::unique_ptr<llvm::Module>(llvm::unwrap(mod)); |
| 508 |
} |
| 509 |
|
| 510 |
/* |
| 511 |
* Compute list of referenced variables, functions and the instruction count |
| 512 |
* for a function. |
| 513 |
*/ |
| 514 |
static void |
| 515 |
function_references(llvm::Function &F, |
| 516 |
int &running_instcount, |
| 517 |
llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars, |
| 518 |
llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions) |
| 519 |
{ |
| 520 |
llvm::SmallPtrSet<const llvm::User *, 32> Visited; |
| 521 |
|
| 522 |
for (llvm::BasicBlock &BB : F) |
| 523 |
{ |
| 524 |
for (llvm::Instruction &I : BB) |
| 525 |
{ |
| 526 |
if (llvm::isa<llvm::DbgInfoIntrinsic>(I)) |
| 527 |
continue; |
| 528 |
|
| 529 |
llvm::SmallVector<llvm::User *, 8> Worklist; |
| 530 |
Worklist.push_back(&I); |
| 531 |
|
| 532 |
running_instcount++; |
| 533 |
|
| 534 |
while (!Worklist.empty()) { |
| 535 |
llvm::User *U = Worklist.pop_back_val(); |
| 536 |
|
| 537 |
/* visited before */ |
| 538 |
if (!Visited.insert(U).second) |
| 539 |
continue; |
| 540 |
|
| 541 |
for (auto &OI : U->operands()) { |
| 542 |
llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI); |
| 543 |
if (!Operand) |
| 544 |
continue; |
| 545 |
if (llvm::isa<llvm::BlockAddress>(Operand)) |
| 546 |
continue; |
| 547 |
if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) { |
| 548 |
referencedVars.insert(GV); |
| 549 |
if (GV->hasInitializer()) |
| 550 |
Worklist.push_back(GV->getInitializer()); |
| 551 |
continue; |
| 552 |
} |
| 553 |
if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) { |
| 554 |
referencedFunctions.insert(CF); |
| 555 |
continue; |
| 556 |
} |
| 557 |
Worklist.push_back(Operand); |
| 558 |
} |
| 559 |
} |
| 560 |
} |
| 561 |
} |
| 562 |
} |
| 563 |
|
| 564 |
/* |
| 565 |
* Check whether function F is inlinable and, if so, what globals need to be |
| 566 |
* imported. |
| 567 |
* |
| 568 |
* References to external functions from, potentially recursively, inlined |
| 569 |
* functions are added to the passed in worklist. |
| 570 |
*/ |
| 571 |
static bool |
| 572 |
function_inlinable(llvm::Function &F, |
| 573 |
int threshold, |
| 574 |
FunctionInlineStates &functionStates, |
| 575 |
InlineWorkList &worklist, |
| 576 |
InlineSearchPath &searchpath, |
| 577 |
llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions, |
| 578 |
int &running_instcount, |
| 579 |
llvm::StringSet<> &importVars) |
| 580 |
{ |
| 581 |
int subThreshold = threshold * inline_cost_decay_factor; |
| 582 |
llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars; |
| 583 |
llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions; |
| 584 |
|
| 585 |
/* can't rely on what may be inlined */ |
| 586 |
if (F.isInterposable()) |
| 587 |
return false; |
| 588 |
|
| 589 |
/* |
| 590 |
* Can't rely on function being present. Alternatively we could create a |
| 591 |
* static version of these functions? |
| 592 |
*/ |
| 593 |
if (F.hasAvailableExternallyLinkage()) |
| 594 |
return false; |
| 595 |
|
| 596 |
ilog(DEBUG1, "checking inlinability of %s", F.getName().data()); |
| 597 |
|
| 598 |
if (F.materialize()) |
| 599 |
elog(FATAL, "failed to materialize metadata"); |
| 600 |
|
| 601 |
if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline)) |
| 602 |
{ |
| 603 |
ilog(DEBUG1, "ineligibile to import %s due to noinline", |
| 604 |
F.getName().data()); |
| 605 |
return false; |
| 606 |
} |
| 607 |
|
| 608 |
function_references(F, running_instcount, referencedVars, referencedFunctions); |
| 609 |
|
| 610 |
for (llvm::GlobalVariable* rv: referencedVars) |
| 611 |
{ |
| 612 |
if (rv->materialize()) |
| 613 |
elog(FATAL, "failed to materialize metadata"); |
| 614 |
|
| 615 |
/* |
| 616 |
* Don't inline functions that access thread local variables. That |
| 617 |
* doesn't work on current LLVM releases (but might in future). |
| 618 |
*/ |
| 619 |
if (rv->isThreadLocal()) |
| 620 |
{ |
| 621 |
ilog(DEBUG1, "cannot inline %s due to thread-local variable %s", |
| 622 |
F.getName().data(), rv->getName().data()); |
| 623 |
return false; |
| 624 |
} |
| 625 |
|
| 626 |
/* |
| 627 |
* Never want to inline externally visible vars, cheap enough to |
| 628 |
* reference. |
| 629 |
*/ |
| 630 |
if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage()) |
| 631 |
continue; |
| 632 |
|
| 633 |
/* |
| 634 |
* If variable is file-local, we need to inline it, to be able to |
| 635 |
* inline the function itself. Can't do that if the variable can be |
| 636 |
* modified, because they'd obviously get out of sync. |
| 637 |
* |
| 638 |
* XXX: Currently not a problem, but there'd be problems with |
| 639 |
* nontrivial initializers if they were allowed for postgres. |
| 640 |
*/ |
| 641 |
if (!rv->isConstant()) |
| 642 |
{ |
| 643 |
ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s", |
| 644 |
F.getName().data(), rv->getName().data()); |
| 645 |
return false; |
| 646 |
} |
| 647 |
|
| 648 |
ilog(DEBUG1, "memorizing global var %s linkage %d for inlining", |
| 649 |
rv->getName().data(), (int)rv->getLinkage()); |
| 650 |
|
| 651 |
importVars.insert(rv->getName()); |
| 652 |
/* small cost attributed to each cloned global */ |
| 653 |
running_instcount += 5; |
| 654 |
} |
| 655 |
|
| 656 |
visitedFunctions.insert(&F); |
| 657 |
|
| 658 |
/* |
| 659 |
* Check referenced functions. Check whether used static ones are |
| 660 |
* inlinable, and remember external ones for inlining. |
| 661 |
*/ |
| 662 |
for (llvm::Function* referencedFunction: referencedFunctions) |
| 663 |
{ |
| 664 |
llvm::StringSet<> recImportVars; |
| 665 |
|
| 666 |
if (referencedFunction->materialize()) |
| 667 |
elog(FATAL, "failed to materialize metadata"); |
| 668 |
|
| 669 |
if (referencedFunction->isIntrinsic()) |
| 670 |
continue; |
| 671 |
|
| 672 |
/* if already visited skip, otherwise remember */ |
| 673 |
if (!visitedFunctions.insert(referencedFunction).second) |
| 674 |
continue; |
| 675 |
|
| 676 |
/* |
| 677 |
* We don't inline external functions directly here, instead we put |
| 678 |
* them on the worklist if appropriate and check them from |
| 679 |
* llvm_build_inline_plan(). |
| 680 |
*/ |
| 681 |
if (referencedFunction->hasExternalLinkage()) |
| 682 |
{ |
| 683 |
llvm::StringRef funcName = referencedFunction->getName(); |
| 684 |
|
| 685 |
/* |
| 686 |
* Don't bother checking for inlining if remaining cost budget is |
| 687 |
* very small. |
| 688 |
*/ |
| 689 |
if (subThreshold < 5) |
| 690 |
continue; |
| 691 |
|
| 692 |
auto it = functionStates.find(funcName); |
| 693 |
if (it == functionStates.end()) |
| 694 |
{ |
| 695 |
FunctionInlineState inlineState; |
| 696 |
|
| 697 |
inlineState.costLimit = subThreshold; |
| 698 |
inlineState.processed = false; |
| 699 |
inlineState.inlined = false; |
| 700 |
inlineState.allowReconsidering = false; |
| 701 |
|
| 702 |
functionStates[funcName] = inlineState; |
| 703 |
worklist.push_back({funcName, searchpath}); |
| 704 |
|
| 705 |
ilog(DEBUG1, |
| 706 |
"considering extern function %s at %d for inlining", |
| 707 |
funcName.data(), subThreshold); |
| 708 |
} |
| 709 |
else if (!it->second.inlined && |
| 710 |
(!it->second.processed || it->second.allowReconsidering) && |
| 711 |
it->second.costLimit < subThreshold) |
| 712 |
{ |
| 713 |
/* |
| 714 |
* Update inlining threshold if higher. Need to re-queue |
| 715 |
* to be processed if already processed with lower |
| 716 |
* threshold. |
| 717 |
*/ |
| 718 |
if (it->second.processed) |
| 719 |
{ |
| 720 |
ilog(DEBUG1, |
| 721 |
"reconsidering extern function %s at %d for inlining, increasing from %d", |
| 722 |
funcName.data(), subThreshold, it->second.costLimit); |
| 723 |
|
| 724 |
it->second.processed = false; |
| 725 |
it->second.allowReconsidering = false; |
| 726 |
worklist.push_back({funcName, searchpath}); |
| 727 |
} |
| 728 |
it->second.costLimit = subThreshold; |
| 729 |
} |
| 730 |
continue; |
| 731 |
} |
| 732 |
|
| 733 |
/* can't rely on what may be inlined */ |
| 734 |
if (referencedFunction->isInterposable()) |
| 735 |
return false; |
| 736 |
|
| 737 |
if (!function_inlinable(*referencedFunction, |
| 738 |
subThreshold, |
| 739 |
functionStates, |
| 740 |
worklist, |
| 741 |
searchpath, |
| 742 |
visitedFunctions, |
| 743 |
running_instcount, |
| 744 |
recImportVars)) |
| 745 |
{ |
| 746 |
ilog(DEBUG1, |
| 747 |
"cannot inline %s due to required function %s not being inlinable", |
| 748 |
F.getName().data(), referencedFunction->getName().data()); |
| 749 |
return false; |
| 750 |
} |
| 751 |
|
| 752 |
/* import referenced function itself */ |
| 753 |
importVars.insert(referencedFunction->getName()); |
| 754 |
|
| 755 |
/* import referenced function and its dependents */ |
| 756 |
for (auto& recImportVar : recImportVars) |
| 757 |
importVars.insert(recImportVar.first()); |
| 758 |
} |
| 759 |
|
| 760 |
return true; |
| 761 |
} |
| 762 |
|
| 763 |
/* |
| 764 |
* Attempt to load module summary located at path. Return empty pointer when |
| 765 |
* loading fails. |
| 766 |
*/ |
| 767 |
static std::unique_ptr<llvm::ModuleSummaryIndex> |
| 768 |
llvm_load_summary(llvm::StringRef path) |
| 769 |
{ |
| 770 |
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr = |
| 771 |
llvm::MemoryBuffer::getFile(path); |
| 772 |
|
| 773 |
if (std::error_code EC = MBOrErr.getError()) |
| 774 |
{ |
| 775 |
ilog(DEBUG1, "failed to open %s: %s", path.data(), |
| 776 |
EC.message().c_str()); |
| 777 |
} |
| 778 |
else |
| 779 |
{ |
| 780 |
llvm::MemoryBufferRef ref(*MBOrErr.get().get()); |
| 781 |
|
| 782 |
llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr = |
| 783 |
llvm::getModuleSummaryIndex(ref); |
| 784 |
if (IndexOrErr) |
| 785 |
return std::move(IndexOrErr.get()); |
| 786 |
elog(FATAL, "failed to load summary \"%s\": %s", |
| 787 |
path.data(), |
| 788 |
toString(IndexOrErr.takeError()).c_str()); |
| 789 |
} |
| 790 |
return nullptr; |
| 791 |
} |
| 792 |
|
| 793 |
/* |
| 794 |
* Attempt to add modpath to the search path. |
| 795 |
*/ |
| 796 |
static void |
| 797 |
add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath) |
| 798 |
{ |
| 799 |
/* only extension in libdir are candidates for inlining for now */ |
| 800 |
#if LLVM_VERSION_MAJOR < 16 |
| 801 |
#define starts_with startswith |
| 802 |
#endif |
| 803 |
if (!modpath.starts_with("$libdir/")) |
| 804 |
return; |
| 805 |
|
| 806 |
/* if there's no match, attempt to load */ |
| 807 |
auto it = summary_cache->find(modpath); |
| 808 |
if (it == summary_cache->end()) |
| 809 |
{ |
| 810 |
std::string path(modpath); |
| 811 |
path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode"); |
| 812 |
path += ".index.bc"; |
| 813 |
(*summary_cache)[modpath] = llvm_load_summary(path); |
| 814 |
it = summary_cache->find(modpath); |
| 815 |
} |
| 816 |
|
| 817 |
Assert(it != summary_cache->end()); |
| 818 |
|
| 819 |
/* if the entry isn't NULL, it's validly loaded */ |
| 820 |
if (it->second) |
| 821 |
searchpath.push_back(it->second.get()); |
| 822 |
} |
| 823 |
|
| 824 |
/* |
| 825 |
* Search for all references for functions hashing to guid in the search path, |
| 826 |
* and return them in search path order. |
| 827 |
*/ |
| 828 |
static llvm::SmallVector<llvm::GlobalValueSummary *, 1> |
| 829 |
summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid) |
| 830 |
{ |
| 831 |
llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches; |
| 832 |
|
| 833 |
for (auto index : path) |
| 834 |
{ |
| 835 |
llvm::ValueInfo funcVI = index->getValueInfo(guid); |
| 836 |
|
| 837 |
/* if index doesn't know function, we don't have a body, continue */ |
| 838 |
if (funcVI) |
| 839 |
for (auto &gv : funcVI.getSummaryList()) |
| 840 |
matches.push_back(gv.get()); |
| 841 |
} |
| 842 |
|
| 843 |
return matches; |
| 844 |
} |
| 845 |
|
| 846 |
/* |
| 847 |
* Create inline wrapper with the name Name, redirecting the call to F. |
| 848 |
*/ |
| 849 |
static llvm::Function* |
| 850 |
create_redirection_function(std::unique_ptr<llvm::Module> &importMod, |
| 851 |
llvm::Function *F, |
| 852 |
llvm::StringRef Name) |
| 853 |
{ |
| 854 |
typedef llvm::GlobalValue::LinkageTypes LinkageTypes; |
| 855 |
|
| 856 |
llvm::LLVMContext &Context = F->getContext(); |
| 857 |
llvm::IRBuilder<> Builder(Context); |
| 858 |
llvm::Function *AF; |
| 859 |
llvm::BasicBlock *BB; |
| 860 |
llvm::CallInst *fwdcall; |
| 861 |
|
| 862 |
AF = llvm::Function::Create(F->getFunctionType(), |
| 863 |
LinkageTypes::AvailableExternallyLinkage, |
| 864 |
Name, importMod.get()); |
| 865 |
BB = llvm::BasicBlock::Create(Context, "entry", AF); |
| 866 |
|
| 867 |
Builder.SetInsertPoint(BB); |
| 868 |
fwdcall = Builder.CreateCall(F, &*AF->arg_begin()); |
| 869 |
fwdcall->addFnAttr(llvm::Attribute::AlwaysInline); |
| 870 |
Builder.CreateRet(fwdcall); |
| 871 |
|
| 872 |
return AF; |
| 873 |
} |