| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* llvmjit.c |
| 4 |
* Core part of the LLVM JIT provider. |
| 5 |
* |
| 6 |
* Copyright (c) 2016-2025, PostgreSQL Global Development Group |
| 7 |
* |
| 8 |
* IDENTIFICATION |
| 9 |
* src/backend/jit/llvm/llvmjit.c |
| 10 |
* |
| 11 |
*------------------------------------------------------------------------- |
| 12 |
*/ |
| 13 |
|
| 14 |
#include "postgres.h" |
| 15 |
|
| 16 |
#include <llvm-c/Analysis.h> |
| 17 |
#include <llvm-c/BitReader.h> |
| 18 |
#include <llvm-c/BitWriter.h> |
| 19 |
#include <llvm-c/Core.h> |
| 20 |
#include <llvm-c/ExecutionEngine.h> |
| 21 |
#if LLVM_VERSION_MAJOR > 16 |
| 22 |
#include <llvm-c/Transforms/PassBuilder.h> |
| 23 |
#endif |
| 24 |
#include <llvm-c/Orc.h> |
| 25 |
#include <llvm-c/OrcEE.h> |
| 26 |
#include <llvm-c/LLJIT.h> |
| 27 |
#include <llvm-c/Support.h> |
| 28 |
#include <llvm-c/Target.h> |
| 29 |
#if LLVM_VERSION_MAJOR < 17 |
| 30 |
#include <llvm-c/Transforms/IPO.h> |
| 31 |
#include <llvm-c/Transforms/PassManagerBuilder.h> |
| 32 |
#include <llvm-c/Transforms/Scalar.h> |
| 33 |
#include <llvm-c/Transforms/Utils.h> |
| 34 |
#endif |
| 35 |
|
| 36 |
#include "jit/llvmjit.h" |
| 37 |
#include "jit/llvmjit_backport.h" |
| 38 |
#include "jit/llvmjit_emit.h" |
| 39 |
#include "miscadmin.h" |
| 40 |
#include "portability/instr_time.h" |
| 41 |
#include "storage/ipc.h" |
| 42 |
#include "utils/memutils.h" |
| 43 |
#include "utils/resowner.h" |
| 44 |
|
| 45 |
#define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100 |
| 46 |
|
| 47 |
/* Handle of a module emitted via ORC JIT */ |
| 48 |
typedef struct LLVMJitHandle |
| 49 |
{ |
| 50 |
LLVMOrcLLJITRef lljit; |
| 51 |
LLVMOrcResourceTrackerRef resource_tracker; |
| 52 |
} LLVMJitHandle; |
| 53 |
|
| 54 |
|
| 55 |
/* types & functions commonly needed for JITing */ |
| 56 |
LLVMTypeRef TypeSizeT; |
| 57 |
LLVMTypeRef TypeParamBool; |
| 58 |
LLVMTypeRef TypeStorageBool; |
| 59 |
LLVMTypeRef TypePGFunction; |
| 60 |
LLVMTypeRef StructNullableDatum; |
| 61 |
LLVMTypeRef StructHeapTupleData; |
| 62 |
LLVMTypeRef StructMinimalTupleData; |
| 63 |
LLVMTypeRef StructTupleDescData; |
| 64 |
LLVMTypeRef StructTupleTableSlot; |
| 65 |
LLVMTypeRef StructHeapTupleHeaderData; |
| 66 |
LLVMTypeRef StructHeapTupleTableSlot; |
| 67 |
LLVMTypeRef StructMinimalTupleTableSlot; |
| 68 |
LLVMTypeRef StructMemoryContextData; |
| 69 |
LLVMTypeRef StructFunctionCallInfoData; |
| 70 |
LLVMTypeRef StructExprContext; |
| 71 |
LLVMTypeRef StructExprEvalStep; |
| 72 |
LLVMTypeRef StructExprState; |
| 73 |
LLVMTypeRef StructAggState; |
| 74 |
LLVMTypeRef StructAggStatePerGroupData; |
| 75 |
LLVMTypeRef StructAggStatePerTransData; |
| 76 |
LLVMTypeRef StructPlanState; |
| 77 |
|
| 78 |
LLVMValueRef AttributeTemplate; |
| 79 |
LLVMValueRef ExecEvalSubroutineTemplate; |
| 80 |
LLVMValueRef ExecEvalBoolSubroutineTemplate; |
| 81 |
|
| 82 |
static LLVMModuleRef llvm_types_module = NULL; |
| 83 |
|
| 84 |
static bool llvm_session_initialized = false; |
| 85 |
static size_t llvm_generation = 0; |
| 86 |
|
| 87 |
/* number of LLVMJitContexts that currently are in use */ |
| 88 |
static size_t llvm_jit_context_in_use_count = 0; |
| 89 |
|
| 90 |
/* how many times has the current LLVMContextRef been used */ |
| 91 |
static size_t llvm_llvm_context_reuse_count = 0; |
| 92 |
static const char *llvm_triple = NULL; |
| 93 |
static const char *llvm_layout = NULL; |
| 94 |
static LLVMContextRef llvm_context; |
| 95 |
|
| 96 |
|
| 97 |
static LLVMTargetRef llvm_targetref; |
| 98 |
static LLVMOrcThreadSafeContextRef llvm_ts_context; |
| 99 |
static LLVMOrcLLJITRef llvm_opt0_orc; |
| 100 |
static LLVMOrcLLJITRef llvm_opt3_orc; |
| 101 |
|
| 102 |
|
| 103 |
static void llvm_release_context(JitContext *context); |
| 104 |
static void llvm_session_initialize(void); |
| 105 |
static void llvm_shutdown(int code, Datum arg); |
| 106 |
static void llvm_compile_module(LLVMJitContext *context); |
| 107 |
static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module); |
| 108 |
|
| 109 |
static void llvm_create_types(void); |
| 110 |
static void llvm_set_target(void); |
| 111 |
static void llvm_recreate_llvm_context(void); |
| 112 |
static uint64_t llvm_resolve_symbol(const char *name, void *ctx); |
| 113 |
|
| 114 |
static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm); |
| 115 |
static char *llvm_error_message(LLVMErrorRef error); |
| 116 |
|
| 117 |
/* ResourceOwner callbacks to hold JitContexts */ |
| 118 |
static void ResOwnerReleaseJitContext(Datum res); |
| 119 |
|
| 120 |
static const ResourceOwnerDesc jit_resowner_desc = |
| 121 |
{ |
| 122 |
.name = "LLVM JIT context", |
| 123 |
.release_phase = RESOURCE_RELEASE_BEFORE_LOCKS, |
| 124 |
.release_priority = RELEASE_PRIO_JIT_CONTEXTS, |
| 125 |
.ReleaseResource = ResOwnerReleaseJitContext, |
| 126 |
.DebugPrint = NULL /* the default message is fine */ |
| 127 |
}; |
| 128 |
|
| 129 |
/* Convenience wrappers over ResourceOwnerRemember/Forget */ |
| 130 |
static inline void |
| 131 |
ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle) |
| 132 |
{ |
| 133 |
ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc); |
| 134 |
} |
| 135 |
static inline void |
| 136 |
ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle) |
| 137 |
{ |
| 138 |
ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc); |
| 139 |
} |
| 140 |
|
| 141 |
PG_MODULE_MAGIC_EXT( |
| 142 |
.name = "llvmjit", |
| 143 |
.version = PG_VERSION |
| 144 |
); |
| 145 |
|
| 146 |
|
| 147 |
/* |
| 148 |
* Initialize LLVM JIT provider. |
| 149 |
*/ |
| 150 |
void |
| 151 |
_PG_jit_provider_init(JitProviderCallbacks *cb) |
| 152 |
{ |
| 153 |
cb->reset_after_error = llvm_reset_after_error; |
| 154 |
cb->release_context = llvm_release_context; |
| 155 |
cb->compile_expr = llvm_compile_expr; |
| 156 |
} |
| 157 |
|
| 158 |
|
| 159 |
/* |
| 160 |
* Every now and then create a new LLVMContextRef. Unfortunately, during every |
| 161 |
* round of inlining, types may "leak" (they can still be found/used via the |
| 162 |
* context, but new types will be created the next time in inlining is |
| 163 |
* performed). To prevent that from slowly accumulating problematic amounts of |
| 164 |
* memory, recreate the LLVMContextRef we use. We don't want to do so too |
| 165 |
* often, as that implies some overhead (particularly re-loading the module |
| 166 |
* summaries / modules is fairly expensive). A future TODO would be to make |
| 167 |
* this more finegrained and only drop/recreate the LLVMContextRef when we know |
| 168 |
* there has been inlining. If we can get the size of the context from LLVM |
| 169 |
* then that might be a better way to determine when to drop/recreate rather |
| 170 |
* then the usagecount heuristic currently employed. |
| 171 |
*/ |
| 172 |
static void |
| 173 |
llvm_recreate_llvm_context(void) |
| 174 |
{ |
| 175 |
if (!llvm_context) |
| 176 |
elog(ERROR, "Trying to recreate a non-existing context"); |
| 177 |
|
| 178 |
/* |
| 179 |
* We can only safely recreate the LLVM context if no other code is being |
| 180 |
* JITed, otherwise we'd release the types in use for that. |
| 181 |
*/ |
| 182 |
if (llvm_jit_context_in_use_count > 0) |
| 183 |
{ |
| 184 |
llvm_llvm_context_reuse_count++; |
| 185 |
return; |
| 186 |
} |
| 187 |
|
| 188 |
if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX) |
| 189 |
{ |
| 190 |
llvm_llvm_context_reuse_count++; |
| 191 |
return; |
| 192 |
} |
| 193 |
|
| 194 |
/* |
| 195 |
* Need to reset the modules that the inlining code caches before |
| 196 |
* disposing of the context. LLVM modules exist within a specific LLVM |
| 197 |
* context, therefore disposing of the context before resetting the cache |
| 198 |
* would lead to dangling pointers to modules. |
| 199 |
*/ |
| 200 |
llvm_inline_reset_caches(); |
| 201 |
|
| 202 |
LLVMContextDispose(llvm_context); |
| 203 |
llvm_context = LLVMContextCreate(); |
| 204 |
llvm_llvm_context_reuse_count = 0; |
| 205 |
|
| 206 |
/* |
| 207 |
* Re-build cached type information, so code generation code can rely on |
| 208 |
* that information to be present (also prevents the variables to be |
| 209 |
* dangling references). |
| 210 |
*/ |
| 211 |
llvm_create_types(); |
| 212 |
} |
| 213 |
|
| 214 |
|
| 215 |
/* |
| 216 |
* Create a context for JITing work. |
| 217 |
* |
| 218 |
* The context, including subsidiary resources, will be cleaned up either when |
| 219 |
* the context is explicitly released, or when the lifetime of |
| 220 |
* CurrentResourceOwner ends (usually the end of the current [sub]xact). |
| 221 |
*/ |
| 222 |
LLVMJitContext * |
| 223 |
llvm_create_context(int jitFlags) |
| 224 |
{ |
| 225 |
LLVMJitContext *context; |
| 226 |
|
| 227 |
llvm_assert_in_fatal_section(); |
| 228 |
|
| 229 |
llvm_session_initialize(); |
| 230 |
|
| 231 |
llvm_recreate_llvm_context(); |
| 232 |
|
| 233 |
ResourceOwnerEnlarge(CurrentResourceOwner); |
| 234 |
|
| 235 |
context = MemoryContextAllocZero(TopMemoryContext, |
| 236 |
sizeof(LLVMJitContext)); |
| 237 |
context->base.flags = jitFlags; |
| 238 |
|
| 239 |
/* ensure cleanup */ |
| 240 |
context->resowner = CurrentResourceOwner; |
| 241 |
ResourceOwnerRememberJIT(CurrentResourceOwner, context); |
| 242 |
|
| 243 |
llvm_jit_context_in_use_count++; |
| 244 |
|
| 245 |
return context; |
| 246 |
} |
| 247 |
|
| 248 |
/* |
| 249 |
* Release resources required by one llvm context. |
| 250 |
*/ |
| 251 |
static void |
| 252 |
llvm_release_context(JitContext *context) |
| 253 |
{ |
| 254 |
LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context; |
| 255 |
ListCell *lc; |
| 256 |
|
| 257 |
/* |
| 258 |
* Consider as cleaned up even if we skip doing so below, that way we can |
| 259 |
* verify the tracking is correct (see llvm_shutdown()). |
| 260 |
*/ |
| 261 |
llvm_jit_context_in_use_count--; |
| 262 |
|
| 263 |
/* |
| 264 |
* When this backend is exiting, don't clean up LLVM. As an error might |
| 265 |
* have occurred from within LLVM, we do not want to risk reentering. All |
| 266 |
* resource cleanup is going to happen through process exit. |
| 267 |
*/ |
| 268 |
if (proc_exit_inprogress) |
| 269 |
return; |
| 270 |
|
| 271 |
llvm_enter_fatal_on_oom(); |
| 272 |
|
| 273 |
if (llvm_jit_context->module) |
| 274 |
{ |
| 275 |
LLVMDisposeModule(llvm_jit_context->module); |
| 276 |
llvm_jit_context->module = NULL; |
| 277 |
} |
| 278 |
|
| 279 |
foreach(lc, llvm_jit_context->handles) |
| 280 |
{ |
| 281 |
LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc); |
| 282 |
|
| 283 |
{ |
| 284 |
LLVMOrcExecutionSessionRef ee; |
| 285 |
LLVMOrcSymbolStringPoolRef sp; |
| 286 |
|
| 287 |
LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker); |
| 288 |
LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker); |
| 289 |
|
| 290 |
/* |
| 291 |
* Without triggering cleanup of the string pool, we'd leak |
| 292 |
* memory. It'd be sufficient to do this far less often, but in |
| 293 |
* experiments the required time was small enough to just always |
| 294 |
* do it. |
| 295 |
*/ |
| 296 |
ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit); |
| 297 |
sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee); |
| 298 |
LLVMOrcSymbolStringPoolClearDeadEntries(sp); |
| 299 |
} |
| 300 |
|
| 301 |
pfree(jit_handle); |
| 302 |
} |
| 303 |
list_free(llvm_jit_context->handles); |
| 304 |
llvm_jit_context->handles = NIL; |
| 305 |
|
| 306 |
llvm_leave_fatal_on_oom(); |
| 307 |
|
| 308 |
if (llvm_jit_context->resowner) |
| 309 |
ResourceOwnerForgetJIT(llvm_jit_context->resowner, llvm_jit_context); |
| 310 |
} |
| 311 |
|
| 312 |
/* |
| 313 |
* Return module which may be modified, e.g. by creating new functions. |
| 314 |
*/ |
| 315 |
LLVMModuleRef |
| 316 |
llvm_mutable_module(LLVMJitContext *context) |
| 317 |
{ |
| 318 |
llvm_assert_in_fatal_section(); |
| 319 |
|
| 320 |
/* |
| 321 |
* If there's no in-progress module, create a new one. |
| 322 |
*/ |
| 323 |
if (!context->module) |
| 324 |
{ |
| 325 |
context->compiled = false; |
| 326 |
context->module_generation = llvm_generation++; |
| 327 |
context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context); |
| 328 |
LLVMSetTarget(context->module, llvm_triple); |
| 329 |
LLVMSetDataLayout(context->module, llvm_layout); |
| 330 |
} |
| 331 |
|
| 332 |
return context->module; |
| 333 |
} |
| 334 |
|
| 335 |
/* |
| 336 |
* Expand function name to be non-conflicting. This should be used by code |
| 337 |
* generating code, when adding new externally visible function definitions to |
| 338 |
* a Module. |
| 339 |
*/ |
| 340 |
char * |
| 341 |
llvm_expand_funcname(struct LLVMJitContext *context, const char *basename) |
| 342 |
{ |
| 343 |
Assert(context->module != NULL); |
| 344 |
|
| 345 |
context->base.instr.created_functions++; |
| 346 |
|
| 347 |
/* |
| 348 |
* Previously we used dots to separate, but turns out some tools, e.g. |
| 349 |
* GDB, don't like that and truncate name. |
| 350 |
*/ |
| 351 |
return psprintf("%s_%zu_%d", |
| 352 |
basename, |
| 353 |
context->module_generation, |
| 354 |
context->counter++); |
| 355 |
} |
| 356 |
|
| 357 |
/* |
| 358 |
* Return pointer to function funcname, which has to exist. If there's pending |
| 359 |
* code to be optimized and emitted, do so first. |
| 360 |
*/ |
| 361 |
void * |
| 362 |
llvm_get_function(LLVMJitContext *context, const char *funcname) |
| 363 |
{ |
| 364 |
ListCell *lc; |
| 365 |
|
| 366 |
llvm_assert_in_fatal_section(); |
| 367 |
|
| 368 |
/* |
| 369 |
* If there is a pending / not emitted module, compile and emit now. |
| 370 |
* Otherwise we might not find the [correct] function. |
| 371 |
*/ |
| 372 |
if (!context->compiled) |
| 373 |
{ |
| 374 |
llvm_compile_module(context); |
| 375 |
} |
| 376 |
|
| 377 |
/* |
| 378 |
* ORC's symbol table is of *unmangled* symbols. Therefore we don't need |
| 379 |
* to mangle here. |
| 380 |
*/ |
| 381 |
|
| 382 |
foreach(lc, context->handles) |
| 383 |
{ |
| 384 |
LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc); |
| 385 |
instr_time starttime; |
| 386 |
instr_time endtime; |
| 387 |
LLVMErrorRef error; |
| 388 |
LLVMOrcJITTargetAddress addr; |
| 389 |
|
| 390 |
INSTR_TIME_SET_CURRENT(starttime); |
| 391 |
|
| 392 |
addr = 0; |
| 393 |
error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname); |
| 394 |
if (error) |
| 395 |
elog(ERROR, "failed to look up symbol \"%s\": %s", |
| 396 |
funcname, llvm_error_message(error)); |
| 397 |
|
| 398 |
/* |
| 399 |
* LLJIT only actually emits code the first time a symbol is |
| 400 |
* referenced. Thus add lookup time to emission time. That's counting |
| 401 |
* a bit more than with older LLVM versions, but unlikely to ever |
| 402 |
* matter. |
| 403 |
*/ |
| 404 |
INSTR_TIME_SET_CURRENT(endtime); |
| 405 |
INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter, |
| 406 |
endtime, starttime); |
| 407 |
|
| 408 |
if (addr) |
| 409 |
return (void *) (uintptr_t) addr; |
| 410 |
} |
| 411 |
|
| 412 |
elog(ERROR, "failed to JIT: %s", funcname); |
| 413 |
|
| 414 |
return NULL; |
| 415 |
} |
| 416 |
|
| 417 |
/* |
| 418 |
* Return type of a variable in llvmjit_types.c. This is useful to keep types |
| 419 |
* in sync between plain C and JIT related code. |
| 420 |
*/ |
| 421 |
LLVMTypeRef |
| 422 |
llvm_pg_var_type(const char *varname) |
| 423 |
{ |
| 424 |
LLVMValueRef v_srcvar; |
| 425 |
LLVMTypeRef typ; |
| 426 |
|
| 427 |
/* this'll return a *pointer* to the global */ |
| 428 |
v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname); |
| 429 |
if (!v_srcvar) |
| 430 |
elog(ERROR, "variable %s not in llvmjit_types.c", varname); |
| 431 |
|
| 432 |
typ = LLVMGlobalGetValueType(v_srcvar); |
| 433 |
|
| 434 |
return typ; |
| 435 |
} |
| 436 |
|
| 437 |
/* |
| 438 |
* Return function type of a variable in llvmjit_types.c. This is useful to |
| 439 |
* keep function types in sync between C and JITed code. |
| 440 |
*/ |
| 441 |
LLVMTypeRef |
| 442 |
llvm_pg_var_func_type(const char *varname) |
| 443 |
{ |
| 444 |
LLVMValueRef v_srcvar; |
| 445 |
LLVMTypeRef typ; |
| 446 |
|
| 447 |
v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname); |
| 448 |
if (!v_srcvar) |
| 449 |
elog(ERROR, "function %s not in llvmjit_types.c", varname); |
| 450 |
|
| 451 |
typ = LLVMGetFunctionType(v_srcvar); |
| 452 |
|
| 453 |
return typ; |
| 454 |
} |
| 455 |
|
| 456 |
/* |
| 457 |
* Return declaration for a function referenced in llvmjit_types.c, adding it |
| 458 |
* to the module if necessary. |
| 459 |
* |
| 460 |
* This is used to make functions discovered via llvm_create_types() known to |
| 461 |
* the module that's currently being worked on. |
| 462 |
*/ |
| 463 |
LLVMValueRef |
| 464 |
llvm_pg_func(LLVMModuleRef mod, const char *funcname) |
| 465 |
{ |
| 466 |
LLVMValueRef v_srcfn; |
| 467 |
LLVMValueRef v_fn; |
| 468 |
|
| 469 |
/* don't repeatedly add function */ |
| 470 |
v_fn = LLVMGetNamedFunction(mod, funcname); |
| 471 |
if (v_fn) |
| 472 |
return v_fn; |
| 473 |
|
| 474 |
v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname); |
| 475 |
|
| 476 |
if (!v_srcfn) |
| 477 |
elog(ERROR, "function %s not in llvmjit_types.c", funcname); |
| 478 |
|
| 479 |
v_fn = LLVMAddFunction(mod, |
| 480 |
funcname, |
| 481 |
LLVMGetFunctionType(v_srcfn)); |
| 482 |
llvm_copy_attributes(v_srcfn, v_fn); |
| 483 |
|
| 484 |
return v_fn; |
| 485 |
} |
| 486 |
|
| 487 |
/* |
| 488 |
* Copy attributes from one function to another, for a specific index (an |
| 489 |
* index can reference return value, function and parameter attributes). |
| 490 |
*/ |
| 491 |
static void |
| 492 |
llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index) |
| 493 |
{ |
| 494 |
int num_attributes; |
| 495 |
LLVMAttributeRef *attrs; |
| 496 |
|
| 497 |
num_attributes = LLVMGetAttributeCountAtIndex(v_from, index); |
| 498 |
|
| 499 |
if (num_attributes == 0) |
| 500 |
return; |
| 501 |
|
| 502 |
attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes); |
| 503 |
LLVMGetAttributesAtIndex(v_from, index, attrs); |
| 504 |
|
| 505 |
for (int attno = 0; attno < num_attributes; attno++) |
| 506 |
LLVMAddAttributeAtIndex(v_to, index, attrs[attno]); |
| 507 |
|
| 508 |
pfree(attrs); |
| 509 |
} |
| 510 |
|
| 511 |
/* |
| 512 |
* Copy all attributes from one function to another. I.e. function, return and |
| 513 |
* parameters will be copied. |
| 514 |
*/ |
| 515 |
void |
| 516 |
llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to) |
| 517 |
{ |
| 518 |
uint32 param_count; |
| 519 |
|
| 520 |
/* copy function attributes */ |
| 521 |
llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex); |
| 522 |
|
| 523 |
if (LLVMGetTypeKind(LLVMGetFunctionReturnType(v_to)) != LLVMVoidTypeKind) |
| 524 |
{ |
| 525 |
/* and the return value attributes */ |
| 526 |
llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex); |
| 527 |
} |
| 528 |
|
| 529 |
/* and each function parameter's attribute */ |
| 530 |
param_count = LLVMCountParams(v_from); |
| 531 |
|
| 532 |
for (int paramidx = 1; paramidx <= param_count; paramidx++) |
| 533 |
llvm_copy_attributes_at_index(v_from, v_to, paramidx); |
| 534 |
} |
| 535 |
|
| 536 |
/* |
| 537 |
* Return a callable LLVMValueRef for fcinfo. |
| 538 |
*/ |
| 539 |
LLVMValueRef |
| 540 |
llvm_function_reference(LLVMJitContext *context, |
| 541 |
LLVMBuilderRef builder, |
| 542 |
LLVMModuleRef mod, |
| 543 |
FunctionCallInfo fcinfo) |
| 544 |
{ |
| 545 |
char *modname; |
| 546 |
char *basename; |
| 547 |
char *funcname; |
| 548 |
|
| 549 |
LLVMValueRef v_fn; |
| 550 |
|
| 551 |
fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename); |
| 552 |
|
| 553 |
if (modname != NULL && basename != NULL) |
| 554 |
{ |
| 555 |
/* external function in loadable library */ |
| 556 |
funcname = psprintf("pgextern.%s.%s", modname, basename); |
| 557 |
} |
| 558 |
else if (basename != NULL) |
| 559 |
{ |
| 560 |
/* internal function */ |
| 561 |
funcname = pstrdup(basename); |
| 562 |
} |
| 563 |
else |
| 564 |
{ |
| 565 |
/* |
| 566 |
* Function we don't know to handle, return pointer. We do so by |
| 567 |
* creating a global constant containing a pointer to the function. |
| 568 |
* Makes IR more readable. |
| 569 |
*/ |
| 570 |
LLVMValueRef v_fn_addr; |
| 571 |
|
| 572 |
funcname = psprintf("pgoidextern.%u", |
| 573 |
fcinfo->flinfo->fn_oid); |
| 574 |
v_fn = LLVMGetNamedGlobal(mod, funcname); |
| 575 |
if (v_fn != 0) |
| 576 |
return l_load(builder, TypePGFunction, v_fn, ""); |
| 577 |
|
| 578 |
v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction); |
| 579 |
|
| 580 |
v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname); |
| 581 |
LLVMSetInitializer(v_fn, v_fn_addr); |
| 582 |
LLVMSetGlobalConstant(v_fn, true); |
| 583 |
LLVMSetLinkage(v_fn, LLVMPrivateLinkage); |
| 584 |
LLVMSetUnnamedAddr(v_fn, true); |
| 585 |
|
| 586 |
return l_load(builder, TypePGFunction, v_fn, ""); |
| 587 |
} |
| 588 |
|
| 589 |
/* check if function already has been added */ |
| 590 |
v_fn = LLVMGetNamedFunction(mod, funcname); |
| 591 |
if (v_fn != 0) |
| 592 |
return v_fn; |
| 593 |
|
| 594 |
v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate)); |
| 595 |
|
| 596 |
return v_fn; |
| 597 |
} |
| 598 |
|
| 599 |
/* |
| 600 |
* Optimize code in module using the flags set in context. |
| 601 |
*/ |
| 602 |
static void |
| 603 |
llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module) |
| 604 |
{ |
| 605 |
#if LLVM_VERSION_MAJOR < 17 |
| 606 |
LLVMPassManagerBuilderRef llvm_pmb; |
| 607 |
LLVMPassManagerRef llvm_mpm; |
| 608 |
LLVMPassManagerRef llvm_fpm; |
| 609 |
LLVMValueRef func; |
| 610 |
int compile_optlevel; |
| 611 |
|
| 612 |
if (context->base.flags & PGJIT_OPT3) |
| 613 |
compile_optlevel = 3; |
| 614 |
else |
| 615 |
compile_optlevel = 0; |
| 616 |
|
| 617 |
/* |
| 618 |
* Have to create a new pass manager builder every pass through, as the |
| 619 |
* inliner has some per-builder state. Otherwise one ends up only inlining |
| 620 |
* a function the first time though. |
| 621 |
*/ |
| 622 |
llvm_pmb = LLVMPassManagerBuilderCreate(); |
| 623 |
LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel); |
| 624 |
llvm_fpm = LLVMCreateFunctionPassManagerForModule(module); |
| 625 |
|
| 626 |
if (context->base.flags & PGJIT_OPT3) |
| 627 |
{ |
| 628 |
/* TODO: Unscientifically determined threshold */ |
| 629 |
LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512); |
| 630 |
} |
| 631 |
else |
| 632 |
{ |
| 633 |
/* we rely on mem2reg heavily, so emit even in the O0 case */ |
| 634 |
LLVMAddPromoteMemoryToRegisterPass(llvm_fpm); |
| 635 |
} |
| 636 |
|
| 637 |
LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm); |
| 638 |
|
| 639 |
/* |
| 640 |
* Do function level optimization. This could be moved to the point where |
| 641 |
* functions are emitted, to reduce memory usage a bit. |
| 642 |
*/ |
| 643 |
LLVMInitializeFunctionPassManager(llvm_fpm); |
| 644 |
for (func = LLVMGetFirstFunction(context->module); |
| 645 |
func != NULL; |
| 646 |
func = LLVMGetNextFunction(func)) |
| 647 |
LLVMRunFunctionPassManager(llvm_fpm, func); |
| 648 |
LLVMFinalizeFunctionPassManager(llvm_fpm); |
| 649 |
LLVMDisposePassManager(llvm_fpm); |
| 650 |
|
| 651 |
/* |
| 652 |
* Perform module level optimization. We do so even in the non-optimized |
| 653 |
* case, so always-inline functions etc get inlined. It's cheap enough. |
| 654 |
*/ |
| 655 |
llvm_mpm = LLVMCreatePassManager(); |
| 656 |
LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb, |
| 657 |
llvm_mpm); |
| 658 |
/* always use always-inliner pass */ |
| 659 |
if (!(context->base.flags & PGJIT_OPT3)) |
| 660 |
LLVMAddAlwaysInlinerPass(llvm_mpm); |
| 661 |
/* if doing inlining, but no expensive optimization, add inlining pass */ |
| 662 |
if (context->base.flags & PGJIT_INLINE |
| 663 |
&& !(context->base.flags & PGJIT_OPT3)) |
| 664 |
LLVMAddFunctionInliningPass(llvm_mpm); |
| 665 |
LLVMRunPassManager(llvm_mpm, context->module); |
| 666 |
LLVMDisposePassManager(llvm_mpm); |
| 667 |
|
| 668 |
LLVMPassManagerBuilderDispose(llvm_pmb); |
| 669 |
#else |
| 670 |
LLVMPassBuilderOptionsRef options; |
| 671 |
LLVMErrorRef err; |
| 672 |
const char *passes; |
| 673 |
|
| 674 |
if (context->base.flags & PGJIT_OPT3) |
| 675 |
passes = "default<O3>"; |
| 676 |
else if (context->base.flags & PGJIT_INLINE) |
| 677 |
/* if doing inlining, but no expensive optimization, add inline pass */ |
| 678 |
passes = "default<O0>,mem2reg,inline"; |
| 679 |
else |
| 680 |
/* default<O0> includes always-inline pass */ |
| 681 |
passes = "default<O0>,mem2reg"; |
| 682 |
|
| 683 |
options = LLVMCreatePassBuilderOptions(); |
| 684 |
|
| 685 |
#ifdef LLVM_PASS_DEBUG |
| 686 |
LLVMPassBuilderOptionsSetDebugLogging(options, 1); |
| 687 |
#endif |
| 688 |
|
| 689 |
/* In assertion builds, run the LLVM verify pass. */ |
| 690 |
#ifdef USE_ASSERT_CHECKING |
| 691 |
LLVMPassBuilderOptionsSetVerifyEach(options, true); |
| 692 |
#endif |
| 693 |
|
| 694 |
LLVMPassBuilderOptionsSetInlinerThreshold(options, 512); |
| 695 |
|
| 696 |
err = LLVMRunPasses(module, passes, NULL, options); |
| 697 |
|
| 698 |
if (err) |
| 699 |
elog(ERROR, "failed to JIT module: %s", llvm_error_message(err)); |
| 700 |
|
| 701 |
LLVMDisposePassBuilderOptions(options); |
| 702 |
#endif |
| 703 |
} |
| 704 |
|
| 705 |
/* |
| 706 |
* Emit code for the currently pending module. |
| 707 |
*/ |
| 708 |
static void |
| 709 |
llvm_compile_module(LLVMJitContext *context) |
| 710 |
{ |
| 711 |
LLVMJitHandle *handle; |
| 712 |
MemoryContext oldcontext; |
| 713 |
instr_time starttime; |
| 714 |
instr_time endtime; |
| 715 |
LLVMOrcLLJITRef compile_orc; |
| 716 |
|
| 717 |
if (context->base.flags & PGJIT_OPT3) |
| 718 |
compile_orc = llvm_opt3_orc; |
| 719 |
else |
| 720 |
compile_orc = llvm_opt0_orc; |
| 721 |
|
| 722 |
/* perform inlining */ |
| 723 |
if (context->base.flags & PGJIT_INLINE) |
| 724 |
{ |
| 725 |
INSTR_TIME_SET_CURRENT(starttime); |
| 726 |
llvm_inline(context->module); |
| 727 |
INSTR_TIME_SET_CURRENT(endtime); |
| 728 |
INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter, |
| 729 |
endtime, starttime); |
| 730 |
} |
| 731 |
|
| 732 |
if (jit_dump_bitcode) |
| 733 |
{ |
| 734 |
char *filename; |
| 735 |
|
| 736 |
filename = psprintf("%d.%zu.bc", |
| 737 |
MyProcPid, |
| 738 |
context->module_generation); |
| 739 |
LLVMWriteBitcodeToFile(context->module, filename); |
| 740 |
pfree(filename); |
| 741 |
} |
| 742 |
|
| 743 |
|
| 744 |
/* optimize according to the chosen optimization settings */ |
| 745 |
INSTR_TIME_SET_CURRENT(starttime); |
| 746 |
llvm_optimize_module(context, context->module); |
| 747 |
INSTR_TIME_SET_CURRENT(endtime); |
| 748 |
INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter, |
| 749 |
endtime, starttime); |
| 750 |
|
| 751 |
if (jit_dump_bitcode) |
| 752 |
{ |
| 753 |
char *filename; |
| 754 |
|
| 755 |
filename = psprintf("%d.%zu.optimized.bc", |
| 756 |
MyProcPid, |
| 757 |
context->module_generation); |
| 758 |
LLVMWriteBitcodeToFile(context->module, filename); |
| 759 |
pfree(filename); |
| 760 |
} |
| 761 |
|
| 762 |
handle = (LLVMJitHandle *) |
| 763 |
MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle)); |
| 764 |
|
| 765 |
/* |
| 766 |
* Emit the code. Note that this can, depending on the optimization |
| 767 |
* settings, take noticeable resources as code emission executes low-level |
| 768 |
* instruction combining/selection passes etc. Without optimization a |
| 769 |
* faster instruction selection mechanism is used. |
| 770 |
*/ |
| 771 |
INSTR_TIME_SET_CURRENT(starttime); |
| 772 |
{ |
| 773 |
LLVMOrcThreadSafeModuleRef ts_module; |
| 774 |
LLVMErrorRef error; |
| 775 |
LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc); |
| 776 |
|
| 777 |
ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context); |
| 778 |
|
| 779 |
handle->lljit = compile_orc; |
| 780 |
handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd); |
| 781 |
|
| 782 |
/* |
| 783 |
* NB: This doesn't actually emit code. That happens lazily the first |
| 784 |
* time a symbol defined in the module is requested. Due to that |
| 785 |
* llvm_get_function() also accounts for emission time. |
| 786 |
*/ |
| 787 |
|
| 788 |
context->module = NULL; /* will be owned by LLJIT */ |
| 789 |
error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc, |
| 790 |
handle->resource_tracker, |
| 791 |
ts_module); |
| 792 |
|
| 793 |
if (error) |
| 794 |
elog(ERROR, "failed to JIT module: %s", |
| 795 |
llvm_error_message(error)); |
| 796 |
|
| 797 |
/* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */ |
| 798 |
} |
| 799 |
|
| 800 |
INSTR_TIME_SET_CURRENT(endtime); |
| 801 |
INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter, |
| 802 |
endtime, starttime); |
| 803 |
|
| 804 |
context->module = NULL; |
| 805 |
context->compiled = true; |
| 806 |
|
| 807 |
/* remember emitted code for cleanup and lookups */ |
| 808 |
oldcontext = MemoryContextSwitchTo(TopMemoryContext); |
| 809 |
context->handles = lappend(context->handles, handle); |
| 810 |
MemoryContextSwitchTo(oldcontext); |
| 811 |
|
| 812 |
ereport(DEBUG1, |
| 813 |
(errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs", |
| 814 |
INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter), |
| 815 |
INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter), |
| 816 |
INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)), |
| 817 |
errhidestmt(true), |
| 818 |
errhidecontext(true))); |
| 819 |
} |
| 820 |
|
| 821 |
/* |
| 822 |
* Per session initialization. |
| 823 |
*/ |
| 824 |
static void |
| 825 |
llvm_session_initialize(void) |
| 826 |
{ |
| 827 |
MemoryContext oldcontext; |
| 828 |
char *error = NULL; |
| 829 |
char *cpu = NULL; |
| 830 |
char *features = NULL; |
| 831 |
LLVMTargetMachineRef opt0_tm; |
| 832 |
LLVMTargetMachineRef opt3_tm; |
| 833 |
|
| 834 |
if (llvm_session_initialized) |
| 835 |
return; |
| 836 |
|
| 837 |
oldcontext = MemoryContextSwitchTo(TopMemoryContext); |
| 838 |
|
| 839 |
LLVMInitializeNativeTarget(); |
| 840 |
LLVMInitializeNativeAsmPrinter(); |
| 841 |
LLVMInitializeNativeAsmParser(); |
| 842 |
|
| 843 |
if (llvm_context == NULL) |
| 844 |
{ |
| 845 |
llvm_context = LLVMContextCreate(); |
| 846 |
|
| 847 |
llvm_jit_context_in_use_count = 0; |
| 848 |
llvm_llvm_context_reuse_count = 0; |
| 849 |
} |
| 850 |
|
| 851 |
/* |
| 852 |
* Synchronize types early, as that also includes inferring the target |
| 853 |
* triple. |
| 854 |
*/ |
| 855 |
llvm_create_types(); |
| 856 |
|
| 857 |
/* |
| 858 |
* Extract target information from loaded module. |
| 859 |
*/ |
| 860 |
llvm_set_target(); |
| 861 |
|
| 862 |
if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0) |
| 863 |
{ |
| 864 |
elog(FATAL, "failed to query triple %s", error); |
| 865 |
} |
| 866 |
|
| 867 |
/* |
| 868 |
* We want the generated code to use all available features. Therefore |
| 869 |
* grab the host CPU string and detect features of the current CPU. The |
| 870 |
* latter is needed because some CPU architectures default to enabling |
| 871 |
* features not all CPUs have (weird, huh). |
| 872 |
*/ |
| 873 |
cpu = LLVMGetHostCPUName(); |
| 874 |
features = LLVMGetHostCPUFeatures(); |
| 875 |
elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"", |
| 876 |
cpu, features); |
| 877 |
|
| 878 |
opt0_tm = |
| 879 |
LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features, |
| 880 |
LLVMCodeGenLevelNone, |
| 881 |
LLVMRelocDefault, |
| 882 |
LLVMCodeModelJITDefault); |
| 883 |
opt3_tm = |
| 884 |
LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features, |
| 885 |
LLVMCodeGenLevelAggressive, |
| 886 |
LLVMRelocDefault, |
| 887 |
LLVMCodeModelJITDefault); |
| 888 |
|
| 889 |
LLVMDisposeMessage(cpu); |
| 890 |
cpu = NULL; |
| 891 |
LLVMDisposeMessage(features); |
| 892 |
features = NULL; |
| 893 |
|
| 894 |
/* force symbols in main binary to be loaded */ |
| 895 |
LLVMLoadLibraryPermanently(NULL); |
| 896 |
|
| 897 |
{ |
| 898 |
llvm_ts_context = LLVMOrcCreateNewThreadSafeContext(); |
| 899 |
|
| 900 |
llvm_opt0_orc = llvm_create_jit_instance(opt0_tm); |
| 901 |
opt0_tm = 0; |
| 902 |
|
| 903 |
llvm_opt3_orc = llvm_create_jit_instance(opt3_tm); |
| 904 |
opt3_tm = 0; |
| 905 |
} |
| 906 |
|
| 907 |
on_proc_exit(llvm_shutdown, 0); |
| 908 |
|
| 909 |
llvm_session_initialized = true; |
| 910 |
|
| 911 |
MemoryContextSwitchTo(oldcontext); |
| 912 |
} |
| 913 |
|
| 914 |
static void |
| 915 |
llvm_shutdown(int code, Datum arg) |
| 916 |
{ |
| 917 |
/* |
| 918 |
* If llvm_shutdown() is reached while in a fatal-on-oom section an error |
| 919 |
* has occurred in the middle of LLVM code. It is not safe to call back |
| 920 |
* into LLVM (which is why a FATAL error was thrown). |
| 921 |
* |
| 922 |
* We do need to shutdown LLVM in other shutdown cases, otherwise e.g. |
| 923 |
* profiling data won't be written out. |
| 924 |
*/ |
| 925 |
if (llvm_in_fatal_on_oom()) |
| 926 |
{ |
| 927 |
Assert(proc_exit_inprogress); |
| 928 |
return; |
| 929 |
} |
| 930 |
|
| 931 |
if (llvm_jit_context_in_use_count != 0) |
| 932 |
elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)", |
| 933 |
llvm_jit_context_in_use_count); |
| 934 |
|
| 935 |
{ |
| 936 |
if (llvm_opt3_orc) |
| 937 |
{ |
| 938 |
LLVMOrcDisposeLLJIT(llvm_opt3_orc); |
| 939 |
llvm_opt3_orc = NULL; |
| 940 |
} |
| 941 |
if (llvm_opt0_orc) |
| 942 |
{ |
| 943 |
LLVMOrcDisposeLLJIT(llvm_opt0_orc); |
| 944 |
llvm_opt0_orc = NULL; |
| 945 |
} |
| 946 |
if (llvm_ts_context) |
| 947 |
{ |
| 948 |
LLVMOrcDisposeThreadSafeContext(llvm_ts_context); |
| 949 |
llvm_ts_context = NULL; |
| 950 |
} |
| 951 |
} |
| 952 |
} |
| 953 |
|
| 954 |
/* helper for llvm_create_types, returning a function's return type */ |
| 955 |
static LLVMTypeRef |
| 956 |
load_return_type(LLVMModuleRef mod, const char *name) |
| 957 |
{ |
| 958 |
LLVMValueRef value; |
| 959 |
LLVMTypeRef typ; |
| 960 |
|
| 961 |
/* this'll return a *pointer* to the function */ |
| 962 |
value = LLVMGetNamedFunction(mod, name); |
| 963 |
if (!value) |
| 964 |
elog(ERROR, "function %s is unknown", name); |
| 965 |
|
| 966 |
typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */ |
| 967 |
|
| 968 |
return typ; |
| 969 |
} |
| 970 |
|
| 971 |
/* |
| 972 |
* Load triple & layout from clang emitted file so we're guaranteed to be |
| 973 |
* compatible. |
| 974 |
*/ |
| 975 |
static void |
| 976 |
llvm_set_target(void) |
| 977 |
{ |
| 978 |
if (!llvm_types_module) |
| 979 |
elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded"); |
| 980 |
|
| 981 |
if (llvm_triple == NULL) |
| 982 |
llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module)); |
| 983 |
|
| 984 |
if (llvm_layout == NULL) |
| 985 |
llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module)); |
| 986 |
} |
| 987 |
|
| 988 |
/* |
| 989 |
* Load required information, types, function signatures from llvmjit_types.c |
| 990 |
* and make them available in global variables. |
| 991 |
* |
| 992 |
* Those global variables are then used while emitting code. |
| 993 |
*/ |
| 994 |
static void |
| 995 |
llvm_create_types(void) |
| 996 |
{ |
| 997 |
char path[MAXPGPATH]; |
| 998 |
LLVMMemoryBufferRef buf; |
| 999 |
char *msg; |
| 1000 |
|
| 1001 |
snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc"); |
| 1002 |
|
| 1003 |
/* open file */ |
| 1004 |
if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg)) |
| 1005 |
{ |
| 1006 |
elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s", |
| 1007 |
path, msg); |
| 1008 |
} |
| 1009 |
|
| 1010 |
/* eagerly load contents, going to need it all */ |
| 1011 |
if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module)) |
| 1012 |
{ |
| 1013 |
elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path); |
| 1014 |
} |
| 1015 |
LLVMDisposeMemoryBuffer(buf); |
| 1016 |
|
| 1017 |
TypeSizeT = llvm_pg_var_type("TypeSizeT"); |
| 1018 |
TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool"); |
| 1019 |
TypeStorageBool = llvm_pg_var_type("TypeStorageBool"); |
| 1020 |
TypePGFunction = llvm_pg_var_type("TypePGFunction"); |
| 1021 |
StructNullableDatum = llvm_pg_var_type("StructNullableDatum"); |
| 1022 |
StructExprContext = llvm_pg_var_type("StructExprContext"); |
| 1023 |
StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep"); |
| 1024 |
StructExprState = llvm_pg_var_type("StructExprState"); |
| 1025 |
StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData"); |
| 1026 |
StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData"); |
| 1027 |
StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot"); |
| 1028 |
StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot"); |
| 1029 |
StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot"); |
| 1030 |
StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData"); |
| 1031 |
StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData"); |
| 1032 |
StructTupleDescData = llvm_pg_var_type("StructTupleDescData"); |
| 1033 |
StructAggState = llvm_pg_var_type("StructAggState"); |
| 1034 |
StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData"); |
| 1035 |
StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData"); |
| 1036 |
StructPlanState = llvm_pg_var_type("StructPlanState"); |
| 1037 |
StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData"); |
| 1038 |
|
| 1039 |
AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate"); |
| 1040 |
ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate"); |
| 1041 |
ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate"); |
| 1042 |
} |
| 1043 |
|
| 1044 |
/* |
| 1045 |
* Split a symbol into module / function parts. If the function is in the |
| 1046 |
* main binary (or an external library) *modname will be NULL. |
| 1047 |
*/ |
| 1048 |
void |
| 1049 |
llvm_split_symbol_name(const char *name, char **modname, char **funcname) |
| 1050 |
{ |
| 1051 |
/* |
| 1052 |
* Module function names are pgextern.$module.$funcname |
| 1053 |
*/ |
| 1054 |
if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0) |
| 1055 |
{ |
| 1056 |
/* |
| 1057 |
* Symbol names cannot contain a ., therefore we can split based on |
| 1058 |
* first and last occurrence of one. |
| 1059 |
*/ |
| 1060 |
const char *lastdot; |
| 1061 |
|
| 1062 |
name += strlen("pgextern."); |
| 1063 |
lastdot = strrchr(name, '.'); |
| 1064 |
if (lastdot) |
| 1065 |
{ |
| 1066 |
*modname = pnstrdup(name, lastdot - name); |
| 1067 |
*funcname = pstrdup(lastdot + 1); |
| 1068 |
} |
| 1069 |
else |
| 1070 |
{ |
| 1071 |
/* hmm, no second dot? */ |
| 1072 |
*modname = NULL; |
| 1073 |
*funcname = pstrdup(name); |
| 1074 |
} |
| 1075 |
} |
| 1076 |
else |
| 1077 |
{ |
| 1078 |
*modname = NULL; |
| 1079 |
*funcname = pstrdup(name); |
| 1080 |
} |
| 1081 |
} |
| 1082 |
|
| 1083 |
/* |
| 1084 |
* Attempt to resolve symbol, so LLVM can emit a reference to it. |
| 1085 |
*/ |
| 1086 |
static uint64_t |
| 1087 |
llvm_resolve_symbol(const char *symname, void *ctx) |
| 1088 |
{ |
| 1089 |
uintptr_t addr; |
| 1090 |
char *funcname; |
| 1091 |
char *modname; |
| 1092 |
|
| 1093 |
/* |
| 1094 |
* macOS prefixes all object level symbols with an underscore. But neither |
| 1095 |
* dlsym() nor PG's inliner expect that. So undo. |
| 1096 |
*/ |
| 1097 |
#if defined(__darwin__) |
| 1098 |
if (symname[0] != '_') |
| 1099 |
elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname); |
| 1100 |
symname++; |
| 1101 |
#endif |
| 1102 |
|
| 1103 |
llvm_split_symbol_name(symname, &modname, &funcname); |
| 1104 |
|
| 1105 |
/* functions that aren't resolved to names shouldn't ever get here */ |
| 1106 |
Assert(funcname); |
| 1107 |
|
| 1108 |
if (modname) |
| 1109 |
addr = (uintptr_t) load_external_function(modname, funcname, |
| 1110 |
true, NULL); |
| 1111 |
else |
| 1112 |
addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname); |
| 1113 |
|
| 1114 |
pfree(funcname); |
| 1115 |
if (modname) |
| 1116 |
pfree(modname); |
| 1117 |
|
| 1118 |
/* let LLVM will error out - should never happen */ |
| 1119 |
if (!addr) |
| 1120 |
elog(WARNING, "failed to resolve name %s", symname); |
| 1121 |
|
| 1122 |
return (uint64_t) addr; |
| 1123 |
} |
| 1124 |
|
| 1125 |
static LLVMErrorRef |
| 1126 |
llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx, |
| 1127 |
LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind, |
| 1128 |
LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags, |
| 1129 |
LLVMOrcCLookupSet LookupSet, size_t LookupSetSize) |
| 1130 |
{ |
| 1131 |
#if LLVM_VERSION_MAJOR > 14 |
| 1132 |
LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize); |
| 1133 |
#else |
| 1134 |
LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize); |
| 1135 |
#endif |
| 1136 |
LLVMErrorRef error; |
| 1137 |
LLVMOrcMaterializationUnitRef mu; |
| 1138 |
|
| 1139 |
for (int i = 0; i < LookupSetSize; i++) |
| 1140 |
{ |
| 1141 |
const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name); |
| 1142 |
|
| 1143 |
LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name); |
| 1144 |
symbols[i].Name = LookupSet[i].Name; |
| 1145 |
symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL); |
| 1146 |
symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported; |
| 1147 |
} |
| 1148 |
|
| 1149 |
mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize); |
| 1150 |
error = LLVMOrcJITDylibDefine(JD, mu); |
| 1151 |
if (error != LLVMErrorSuccess) |
| 1152 |
LLVMOrcDisposeMaterializationUnit(mu); |
| 1153 |
|
| 1154 |
pfree(symbols); |
| 1155 |
|
| 1156 |
return error; |
| 1157 |
} |
| 1158 |
|
| 1159 |
/* |
| 1160 |
* We cannot throw errors through LLVM (without causing a FATAL at least), so |
| 1161 |
* just use WARNING here. That's OK anyway, as the error is also reported at |
| 1162 |
* the top level action (with less detail) and there might be multiple |
| 1163 |
* invocations of errors with details. |
| 1164 |
* |
| 1165 |
* This doesn't really happen during normal operation, but in cases like |
| 1166 |
* symbol resolution breakage. So just using elog(WARNING) is fine. |
| 1167 |
*/ |
| 1168 |
static void |
| 1169 |
llvm_log_jit_error(void *ctx, LLVMErrorRef error) |
| 1170 |
{ |
| 1171 |
elog(WARNING, "error during JITing: %s", |
| 1172 |
llvm_error_message(error)); |
| 1173 |
} |
| 1174 |
|
| 1175 |
/* |
| 1176 |
* Create our own object layer, so we can add event listeners. |
| 1177 |
*/ |
| 1178 |
static LLVMOrcObjectLayerRef |
| 1179 |
llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple) |
| 1180 |
{ |
| 1181 |
#if LLVM_VERSION_MAJOR >= 22 |
| 1182 |
LLVMOrcObjectLayerRef objlayer = |
| 1183 |
LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManagerReserveAlloc(ES, true); |
| 1184 |
#elif defined(USE_LLVM_BACKPORT_SECTION_MEMORY_MANAGER) |
| 1185 |
LLVMOrcObjectLayerRef objlayer = |
| 1186 |
LLVMOrcCreateRTDyldObjectLinkingLayerWithSafeSectionMemoryManager(ES); |
| 1187 |
#else |
| 1188 |
LLVMOrcObjectLayerRef objlayer = |
| 1189 |
LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES); |
| 1190 |
#endif |
| 1191 |
|
| 1192 |
|
| 1193 |
#if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER |
| 1194 |
if (jit_debugging_support) |
| 1195 |
{ |
| 1196 |
LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener(); |
| 1197 |
|
| 1198 |
LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l); |
| 1199 |
} |
| 1200 |
#endif |
| 1201 |
|
| 1202 |
#if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER |
| 1203 |
if (jit_profiling_support) |
| 1204 |
{ |
| 1205 |
LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener(); |
| 1206 |
|
| 1207 |
if (l) |
| 1208 |
LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l); |
| 1209 |
} |
| 1210 |
#endif |
| 1211 |
|
| 1212 |
return objlayer; |
| 1213 |
} |
| 1214 |
|
| 1215 |
/* |
| 1216 |
* Create LLJIT instance, using the passed in target machine. Note that the |
| 1217 |
* target machine afterwards is owned by the LLJIT instance. |
| 1218 |
*/ |
| 1219 |
static LLVMOrcLLJITRef |
| 1220 |
llvm_create_jit_instance(LLVMTargetMachineRef tm) |
| 1221 |
{ |
| 1222 |
LLVMOrcLLJITRef lljit; |
| 1223 |
LLVMOrcJITTargetMachineBuilderRef tm_builder; |
| 1224 |
LLVMOrcLLJITBuilderRef lljit_builder; |
| 1225 |
LLVMErrorRef error; |
| 1226 |
LLVMOrcDefinitionGeneratorRef main_gen; |
| 1227 |
LLVMOrcDefinitionGeneratorRef ref_gen; |
| 1228 |
|
| 1229 |
lljit_builder = LLVMOrcCreateLLJITBuilder(); |
| 1230 |
tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm); |
| 1231 |
LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder); |
| 1232 |
|
| 1233 |
LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder, |
| 1234 |
llvm_create_object_layer, |
| 1235 |
NULL); |
| 1236 |
|
| 1237 |
error = LLVMOrcCreateLLJIT(&lljit, lljit_builder); |
| 1238 |
if (error) |
| 1239 |
elog(ERROR, "failed to create lljit instance: %s", |
| 1240 |
llvm_error_message(error)); |
| 1241 |
|
| 1242 |
LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit), |
| 1243 |
llvm_log_jit_error, NULL); |
| 1244 |
|
| 1245 |
/* |
| 1246 |
* Symbol resolution support for symbols in the postgres binary / |
| 1247 |
* libraries already loaded. |
| 1248 |
*/ |
| 1249 |
error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen, |
| 1250 |
LLVMOrcLLJITGetGlobalPrefix(lljit), |
| 1251 |
0, NULL); |
| 1252 |
if (error) |
| 1253 |
elog(ERROR, "failed to create generator: %s", |
| 1254 |
llvm_error_message(error)); |
| 1255 |
LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen); |
| 1256 |
|
| 1257 |
/* |
| 1258 |
* Symbol resolution support for "special" functions, e.g. a call into an |
| 1259 |
* SQL callable function. |
| 1260 |
*/ |
| 1261 |
#if LLVM_VERSION_MAJOR > 14 |
| 1262 |
ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL); |
| 1263 |
#else |
| 1264 |
ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL); |
| 1265 |
#endif |
| 1266 |
LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen); |
| 1267 |
|
| 1268 |
return lljit; |
| 1269 |
} |
| 1270 |
|
| 1271 |
static char * |
| 1272 |
llvm_error_message(LLVMErrorRef error) |
| 1273 |
{ |
| 1274 |
char *orig = LLVMGetErrorMessage(error); |
| 1275 |
char *msg = pstrdup(orig); |
| 1276 |
|
| 1277 |
LLVMDisposeErrorMessage(orig); |
| 1278 |
|
| 1279 |
return msg; |
| 1280 |
} |
| 1281 |
|
| 1282 |
/* |
| 1283 |
* ResourceOwner callbacks |
| 1284 |
*/ |
| 1285 |
static void |
| 1286 |
ResOwnerReleaseJitContext(Datum res) |
| 1287 |
{ |
| 1288 |
LLVMJitContext *context = (LLVMJitContext *) DatumGetPointer(res); |
| 1289 |
|
| 1290 |
context->resowner = NULL; |
| 1291 |
jit_release_context(&context->base); |
| 1292 |
} |