Debian Codegraph @

src/backend/jit/llvm/llvmjit.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * llvmjit.c
4 * Core part of the LLVM JIT provider.
5 *
6 * Copyright (c) 2016-2025, PostgreSQL Global Development Group
7 *
8 * IDENTIFICATION
9 * src/backend/jit/llvm/llvmjit.c
10 *
11 *-------------------------------------------------------------------------
12 */
13
14 #include "postgres.h"
15
16 #include <llvm-c/Analysis.h>
17 #include <llvm-c/BitReader.h>
18 #include <llvm-c/BitWriter.h>
19 #include <llvm-c/Core.h>
20 #include <llvm-c/ExecutionEngine.h>
21 #if LLVM_VERSION_MAJOR > 16
22 #include <llvm-c/Transforms/PassBuilder.h>
23 #endif
24 #include <llvm-c/Orc.h>
25 #include <llvm-c/OrcEE.h>
26 #include <llvm-c/LLJIT.h>
27 #include <llvm-c/Support.h>
28 #include <llvm-c/Target.h>
29 #if LLVM_VERSION_MAJOR < 17
30 #include <llvm-c/Transforms/IPO.h>
31 #include <llvm-c/Transforms/PassManagerBuilder.h>
32 #include <llvm-c/Transforms/Scalar.h>
33 #include <llvm-c/Transforms/Utils.h>
34 #endif
35
36 #include "jit/llvmjit.h"
37 #include "jit/llvmjit_backport.h"
38 #include "jit/llvmjit_emit.h"
39 #include "miscadmin.h"
40 #include "portability/instr_time.h"
41 #include "storage/ipc.h"
42 #include "utils/memutils.h"
43 #include "utils/resowner.h"
44
45 #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
46
47 /* Handle of a module emitted via ORC JIT */
48 typedef struct LLVMJitHandle
49 {
50 LLVMOrcLLJITRef lljit;
51 LLVMOrcResourceTrackerRef resource_tracker;
52 } LLVMJitHandle;
53
54
55 /* types & functions commonly needed for JITing */
56 LLVMTypeRef TypeSizeT;
57 LLVMTypeRef TypeParamBool;
58 LLVMTypeRef TypeStorageBool;
59 LLVMTypeRef TypePGFunction;
60 LLVMTypeRef StructNullableDatum;
61 LLVMTypeRef StructHeapTupleData;
62 LLVMTypeRef StructMinimalTupleData;
63 LLVMTypeRef StructTupleDescData;
64 LLVMTypeRef StructTupleTableSlot;
65 LLVMTypeRef StructHeapTupleHeaderData;
66 LLVMTypeRef StructHeapTupleTableSlot;
67 LLVMTypeRef StructMinimalTupleTableSlot;
68 LLVMTypeRef StructMemoryContextData;
69 LLVMTypeRef StructFunctionCallInfoData;
70 LLVMTypeRef StructExprContext;
71 LLVMTypeRef StructExprEvalStep;
72 LLVMTypeRef StructExprState;
73 LLVMTypeRef StructAggState;
74 LLVMTypeRef StructAggStatePerGroupData;
75 LLVMTypeRef StructAggStatePerTransData;
76 LLVMTypeRef StructPlanState;
77
78 LLVMValueRef AttributeTemplate;
79 LLVMValueRef ExecEvalSubroutineTemplate;
80 LLVMValueRef ExecEvalBoolSubroutineTemplate;
81
82 static LLVMModuleRef llvm_types_module = NULL;
83
84 static bool llvm_session_initialized = false;
85 static size_t llvm_generation = 0;
86
87 /* number of LLVMJitContexts that currently are in use */
88 static size_t llvm_jit_context_in_use_count = 0;
89
90 /* how many times has the current LLVMContextRef been used */
91 static size_t llvm_llvm_context_reuse_count = 0;
92 static const char *llvm_triple = NULL;
93 static const char *llvm_layout = NULL;
94 static LLVMContextRef llvm_context;
95
96
97 static LLVMTargetRef llvm_targetref;
98 static LLVMOrcThreadSafeContextRef llvm_ts_context;
99 static LLVMOrcLLJITRef llvm_opt0_orc;
100 static LLVMOrcLLJITRef llvm_opt3_orc;
101
102
103 static void llvm_release_context(JitContext *context);
104 static void llvm_session_initialize(void);
105 static void llvm_shutdown(int code, Datum arg);
106 static void llvm_compile_module(LLVMJitContext *context);
107 static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
108
109 static void llvm_create_types(void);
110 static void llvm_set_target(void);
111 static void llvm_recreate_llvm_context(void);
112 static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
113
114 static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
115 static char *llvm_error_message(LLVMErrorRef error);
116
117 /* ResourceOwner callbacks to hold JitContexts */
118 static void ResOwnerReleaseJitContext(Datum res);
119
120 static const ResourceOwnerDesc jit_resowner_desc =
121 {
122 .name = "LLVM JIT context",
123 .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
124 .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
125 .ReleaseResource = ResOwnerReleaseJitContext,
126 .DebugPrint = NULL /* the default message is fine */
127 };
128
129 /* Convenience wrappers over ResourceOwnerRemember/Forget */
130 static inline void
131 ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
132 {
133 ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
134 }
135 static inline void
136 ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
137 {
138 ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
139 }
140
141 PG_MODULE_MAGIC_EXT(
142 .name = "llvmjit",
143 .version = PG_VERSION
144 );
145
146
147 /*
148 * Initialize LLVM JIT provider.
149 */
150 void
151 _PG_jit_provider_init(JitProviderCallbacks *cb)
152 {
153 cb->reset_after_error = llvm_reset_after_error;
154 cb->release_context = llvm_release_context;
155 cb->compile_expr = llvm_compile_expr;
156 }
157
158
159 /*
160 * Every now and then create a new LLVMContextRef. Unfortunately, during every
161 * round of inlining, types may "leak" (they can still be found/used via the
162 * context, but new types will be created the next time in inlining is
163 * performed). To prevent that from slowly accumulating problematic amounts of
164 * memory, recreate the LLVMContextRef we use. We don't want to do so too
165 * often, as that implies some overhead (particularly re-loading the module
166 * summaries / modules is fairly expensive). A future TODO would be to make
167 * this more finegrained and only drop/recreate the LLVMContextRef when we know
168 * there has been inlining. If we can get the size of the context from LLVM
169 * then that might be a better way to determine when to drop/recreate rather
170 * then the usagecount heuristic currently employed.
171 */
172 static void
173 llvm_recreate_llvm_context(void)
174 {
175 if (!llvm_context)
176 elog(ERROR, "Trying to recreate a non-existing context");
177
178 /*
179 * We can only safely recreate the LLVM context if no other code is being
180 * JITed, otherwise we'd release the types in use for that.
181 */
182 if (llvm_jit_context_in_use_count > 0)
183 {
184 llvm_llvm_context_reuse_count++;
185 return;
186 }
187
188 if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
189 {
190 llvm_llvm_context_reuse_count++;
191 return;
192 }
193
194 /*
195 * Need to reset the modules that the inlining code caches before
196 * disposing of the context. LLVM modules exist within a specific LLVM
197 * context, therefore disposing of the context before resetting the cache
198 * would lead to dangling pointers to modules.
199 */
200 llvm_inline_reset_caches();
201
202 LLVMContextDispose(llvm_context);
203 llvm_context = LLVMContextCreate();
204 llvm_llvm_context_reuse_count = 0;
205
206 /*
207 * Re-build cached type information, so code generation code can rely on
208 * that information to be present (also prevents the variables to be
209 * dangling references).
210 */
211 llvm_create_types();
212 }
213
214
215 /*
216 * Create a context for JITing work.
217 *
218 * The context, including subsidiary resources, will be cleaned up either when
219 * the context is explicitly released, or when the lifetime of
220 * CurrentResourceOwner ends (usually the end of the current [sub]xact).
221 */
222 LLVMJitContext *
223 llvm_create_context(int jitFlags)
224 {
225 LLVMJitContext *context;
226
227 llvm_assert_in_fatal_section();
228
229 llvm_session_initialize();
230
231 llvm_recreate_llvm_context();
232
233 ResourceOwnerEnlarge(CurrentResourceOwner);
234
235 context = MemoryContextAllocZero(TopMemoryContext,
236 sizeof(LLVMJitContext));
237 context->base.flags = jitFlags;
238
239 /* ensure cleanup */
240 context->resowner = CurrentResourceOwner;
241 ResourceOwnerRememberJIT(CurrentResourceOwner, context);
242
243 llvm_jit_context_in_use_count++;
244
245 return context;
246 }
247
248 /*
249 * Release resources required by one llvm context.
250 */
251 static void
252 llvm_release_context(JitContext *context)
253 {
254 LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
255 ListCell *lc;
256
257 /*
258 * Consider as cleaned up even if we skip doing so below, that way we can
259 * verify the tracking is correct (see llvm_shutdown()).
260 */
261 llvm_jit_context_in_use_count--;
262
263 /*
264 * When this backend is exiting, don't clean up LLVM. As an error might
265 * have occurred from within LLVM, we do not want to risk reentering. All
266 * resource cleanup is going to happen through process exit.
267 */
268 if (proc_exit_inprogress)
269 return;
270
271 llvm_enter_fatal_on_oom();
272
273 if (llvm_jit_context->module)
274 {
275 LLVMDisposeModule(llvm_jit_context->module);
276 llvm_jit_context->module = NULL;
277 }
278
279 foreach(lc, llvm_jit_context->handles)
280 {
281 LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
282
283 {
284 LLVMOrcExecutionSessionRef ee;
285 LLVMOrcSymbolStringPoolRef sp;
286
287 LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
288 LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
289
290 /*
291 * Without triggering cleanup of the string pool, we'd leak
292 * memory. It'd be sufficient to do this far less often, but in
293 * experiments the required time was small enough to just always
294 * do it.
295 */
296 ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
297 sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
298 LLVMOrcSymbolStringPoolClearDeadEntries(sp);
299 }
300
301 pfree(jit_handle);
302 }
303 list_free(llvm_jit_context->handles);
304 llvm_jit_context->handles = NIL;
305
306 llvm_leave_fatal_on_oom();
307
308 if (llvm_jit_context->resowner)
309 ResourceOwnerForgetJIT(llvm_jit_context->resowner, llvm_jit_context);
310 }
311
312 /*
313 * Return module which may be modified, e.g. by creating new functions.
314 */
315 LLVMModuleRef
316 llvm_mutable_module(LLVMJitContext *context)
317 {
318 llvm_assert_in_fatal_section();
319
320 /*
321 * If there's no in-progress module, create a new one.
322 */
323 if (!context->module)
324 {
325 context->compiled = false;
326 context->module_generation = llvm_generation++;
327 context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
328 LLVMSetTarget(context->module, llvm_triple);
329 LLVMSetDataLayout(context->module, llvm_layout);
330 }
331
332 return context->module;
333 }
334
335 /*
336 * Expand function name to be non-conflicting. This should be used by code
337 * generating code, when adding new externally visible function definitions to
338 * a Module.
339 */
340 char *
341 llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
342 {
343 Assert(context->module != NULL);
344
345 context->base.instr.created_functions++;
346
347 /*
348 * Previously we used dots to separate, but turns out some tools, e.g.
349 * GDB, don't like that and truncate name.
350 */
351 return psprintf("%s_%zu_%d",
352 basename,
353 context->module_generation,
354 context->counter++);
355 }
356
357 /*
358 * Return pointer to function funcname, which has to exist. If there's pending
359 * code to be optimized and emitted, do so first.
360 */
361 void *
362 llvm_get_function(LLVMJitContext *context, const char *funcname)
363 {
364 ListCell *lc;
365
366 llvm_assert_in_fatal_section();
367
368 /*
369 * If there is a pending / not emitted module, compile and emit now.
370 * Otherwise we might not find the [correct] function.
371 */
372 if (!context->compiled)
373 {
374 llvm_compile_module(context);
375 }
376
377 /*
378 * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
379 * to mangle here.
380 */
381
382 foreach(lc, context->handles)
383 {
384 LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
385 instr_time starttime;
386 instr_time endtime;
387 LLVMErrorRef error;
388 LLVMOrcJITTargetAddress addr;
389
390 INSTR_TIME_SET_CURRENT(starttime);
391
392 addr = 0;
393 error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
394 if (error)
395 elog(ERROR, "failed to look up symbol \"%s\": %s",
396 funcname, llvm_error_message(error));
397
398 /*
399 * LLJIT only actually emits code the first time a symbol is
400 * referenced. Thus add lookup time to emission time. That's counting
401 * a bit more than with older LLVM versions, but unlikely to ever
402 * matter.
403 */
404 INSTR_TIME_SET_CURRENT(endtime);
405 INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
406 endtime, starttime);
407
408 if (addr)
409 return (void *) (uintptr_t) addr;
410 }
411
412 elog(ERROR, "failed to JIT: %s", funcname);
413
414 return NULL;
415 }
416
417 /*
418 * Return type of a variable in llvmjit_types.c. This is useful to keep types
419 * in sync between plain C and JIT related code.
420 */
421 LLVMTypeRef
422 llvm_pg_var_type(const char *varname)
423 {
424 LLVMValueRef v_srcvar;
425 LLVMTypeRef typ;
426
427 /* this'll return a *pointer* to the global */
428 v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
429 if (!v_srcvar)
430 elog(ERROR, "variable %s not in llvmjit_types.c", varname);
431
432 typ = LLVMGlobalGetValueType(v_srcvar);
433
434 return typ;
435 }
436
437 /*
438 * Return function type of a variable in llvmjit_types.c. This is useful to
439 * keep function types in sync between C and JITed code.
440 */
441 LLVMTypeRef
442 llvm_pg_var_func_type(const char *varname)
443 {
444 LLVMValueRef v_srcvar;
445 LLVMTypeRef typ;
446
447 v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
448 if (!v_srcvar)
449 elog(ERROR, "function %s not in llvmjit_types.c", varname);
450
451 typ = LLVMGetFunctionType(v_srcvar);
452
453 return typ;
454 }
455
456 /*
457 * Return declaration for a function referenced in llvmjit_types.c, adding it
458 * to the module if necessary.
459 *
460 * This is used to make functions discovered via llvm_create_types() known to
461 * the module that's currently being worked on.
462 */
463 LLVMValueRef
464 llvm_pg_func(LLVMModuleRef mod, const char *funcname)
465 {
466 LLVMValueRef v_srcfn;
467 LLVMValueRef v_fn;
468
469 /* don't repeatedly add function */
470 v_fn = LLVMGetNamedFunction(mod, funcname);
471 if (v_fn)
472 return v_fn;
473
474 v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
475
476 if (!v_srcfn)
477 elog(ERROR, "function %s not in llvmjit_types.c", funcname);
478
479 v_fn = LLVMAddFunction(mod,
480 funcname,
481 LLVMGetFunctionType(v_srcfn));
482 llvm_copy_attributes(v_srcfn, v_fn);
483
484 return v_fn;
485 }
486
487 /*
488 * Copy attributes from one function to another, for a specific index (an
489 * index can reference return value, function and parameter attributes).
490 */
491 static void
492 llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
493 {
494 int num_attributes;
495 LLVMAttributeRef *attrs;
496
497 num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
498
499 if (num_attributes == 0)
500 return;
501
502 attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
503 LLVMGetAttributesAtIndex(v_from, index, attrs);
504
505 for (int attno = 0; attno < num_attributes; attno++)
506 LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
507
508 pfree(attrs);
509 }
510
511 /*
512 * Copy all attributes from one function to another. I.e. function, return and
513 * parameters will be copied.
514 */
515 void
516 llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
517 {
518 uint32 param_count;
519
520 /* copy function attributes */
521 llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
522
523 if (LLVMGetTypeKind(LLVMGetFunctionReturnType(v_to)) != LLVMVoidTypeKind)
524 {
525 /* and the return value attributes */
526 llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
527 }
528
529 /* and each function parameter's attribute */
530 param_count = LLVMCountParams(v_from);
531
532 for (int paramidx = 1; paramidx <= param_count; paramidx++)
533 llvm_copy_attributes_at_index(v_from, v_to, paramidx);
534 }
535
536 /*
537 * Return a callable LLVMValueRef for fcinfo.
538 */
539 LLVMValueRef
540 llvm_function_reference(LLVMJitContext *context,
541 LLVMBuilderRef builder,
542 LLVMModuleRef mod,
543 FunctionCallInfo fcinfo)
544 {
545 char *modname;
546 char *basename;
547 char *funcname;
548
549 LLVMValueRef v_fn;
550
551 fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
552
553 if (modname != NULL && basename != NULL)
554 {
555 /* external function in loadable library */
556 funcname = psprintf("pgextern.%s.%s", modname, basename);
557 }
558 else if (basename != NULL)
559 {
560 /* internal function */
561 funcname = pstrdup(basename);
562 }
563 else
564 {
565 /*
566 * Function we don't know to handle, return pointer. We do so by
567 * creating a global constant containing a pointer to the function.
568 * Makes IR more readable.
569 */
570 LLVMValueRef v_fn_addr;
571
572 funcname = psprintf("pgoidextern.%u",
573 fcinfo->flinfo->fn_oid);
574 v_fn = LLVMGetNamedGlobal(mod, funcname);
575 if (v_fn != 0)
576 return l_load(builder, TypePGFunction, v_fn, "");
577
578 v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
579
580 v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
581 LLVMSetInitializer(v_fn, v_fn_addr);
582 LLVMSetGlobalConstant(v_fn, true);
583 LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
584 LLVMSetUnnamedAddr(v_fn, true);
585
586 return l_load(builder, TypePGFunction, v_fn, "");
587 }
588
589 /* check if function already has been added */
590 v_fn = LLVMGetNamedFunction(mod, funcname);
591 if (v_fn != 0)
592 return v_fn;
593
594 v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
595
596 return v_fn;
597 }
598
599 /*
600 * Optimize code in module using the flags set in context.
601 */
602 static void
603 llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
604 {
605 #if LLVM_VERSION_MAJOR < 17
606 LLVMPassManagerBuilderRef llvm_pmb;
607 LLVMPassManagerRef llvm_mpm;
608 LLVMPassManagerRef llvm_fpm;
609 LLVMValueRef func;
610 int compile_optlevel;
611
612 if (context->base.flags & PGJIT_OPT3)
613 compile_optlevel = 3;
614 else
615 compile_optlevel = 0;
616
617 /*
618 * Have to create a new pass manager builder every pass through, as the
619 * inliner has some per-builder state. Otherwise one ends up only inlining
620 * a function the first time though.
621 */
622 llvm_pmb = LLVMPassManagerBuilderCreate();
623 LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
624 llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
625
626 if (context->base.flags & PGJIT_OPT3)
627 {
628 /* TODO: Unscientifically determined threshold */
629 LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
630 }
631 else
632 {
633 /* we rely on mem2reg heavily, so emit even in the O0 case */
634 LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
635 }
636
637 LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
638
639 /*
640 * Do function level optimization. This could be moved to the point where
641 * functions are emitted, to reduce memory usage a bit.
642 */
643 LLVMInitializeFunctionPassManager(llvm_fpm);
644 for (func = LLVMGetFirstFunction(context->module);
645 func != NULL;
646 func = LLVMGetNextFunction(func))
647 LLVMRunFunctionPassManager(llvm_fpm, func);
648 LLVMFinalizeFunctionPassManager(llvm_fpm);
649 LLVMDisposePassManager(llvm_fpm);
650
651 /*
652 * Perform module level optimization. We do so even in the non-optimized
653 * case, so always-inline functions etc get inlined. It's cheap enough.
654 */
655 llvm_mpm = LLVMCreatePassManager();
656 LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
657 llvm_mpm);
658 /* always use always-inliner pass */
659 if (!(context->base.flags & PGJIT_OPT3))
660 LLVMAddAlwaysInlinerPass(llvm_mpm);
661 /* if doing inlining, but no expensive optimization, add inlining pass */
662 if (context->base.flags & PGJIT_INLINE
663 && !(context->base.flags & PGJIT_OPT3))
664 LLVMAddFunctionInliningPass(llvm_mpm);
665 LLVMRunPassManager(llvm_mpm, context->module);
666 LLVMDisposePassManager(llvm_mpm);
667
668 LLVMPassManagerBuilderDispose(llvm_pmb);
669 #else
670 LLVMPassBuilderOptionsRef options;
671 LLVMErrorRef err;
672 const char *passes;
673
674 if (context->base.flags & PGJIT_OPT3)
675 passes = "default<O3>";
676 else if (context->base.flags & PGJIT_INLINE)
677 /* if doing inlining, but no expensive optimization, add inline pass */
678 passes = "default<O0>,mem2reg,inline";
679 else
680 /* default<O0> includes always-inline pass */
681 passes = "default<O0>,mem2reg";
682
683 options = LLVMCreatePassBuilderOptions();
684
685 #ifdef LLVM_PASS_DEBUG
686 LLVMPassBuilderOptionsSetDebugLogging(options, 1);
687 #endif
688
689 /* In assertion builds, run the LLVM verify pass. */
690 #ifdef USE_ASSERT_CHECKING
691 LLVMPassBuilderOptionsSetVerifyEach(options, true);
692 #endif
693
694 LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
695
696 err = LLVMRunPasses(module, passes, NULL, options);
697
698 if (err)
699 elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
700
701 LLVMDisposePassBuilderOptions(options);
702 #endif
703 }
704
705 /*
706 * Emit code for the currently pending module.
707 */
708 static void
709 llvm_compile_module(LLVMJitContext *context)
710 {
711 LLVMJitHandle *handle;
712 MemoryContext oldcontext;
713 instr_time starttime;
714 instr_time endtime;
715 LLVMOrcLLJITRef compile_orc;
716
717 if (context->base.flags & PGJIT_OPT3)
718 compile_orc = llvm_opt3_orc;
719 else
720 compile_orc = llvm_opt0_orc;
721
722 /* perform inlining */
723 if (context->base.flags & PGJIT_INLINE)
724 {
725 INSTR_TIME_SET_CURRENT(starttime);
726 llvm_inline(context->module);
727 INSTR_TIME_SET_CURRENT(endtime);
728 INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
729 endtime, starttime);
730 }
731
732 if (jit_dump_bitcode)
733 {
734 char *filename;
735
736 filename = psprintf("%d.%zu.bc",
737 MyProcPid,
738 context->module_generation);
739 LLVMWriteBitcodeToFile(context->module, filename);
740 pfree(filename);
741 }
742
743
744 /* optimize according to the chosen optimization settings */
745 INSTR_TIME_SET_CURRENT(starttime);
746 llvm_optimize_module(context, context->module);
747 INSTR_TIME_SET_CURRENT(endtime);
748 INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
749 endtime, starttime);
750
751 if (jit_dump_bitcode)
752 {
753 char *filename;
754
755 filename = psprintf("%d.%zu.optimized.bc",
756 MyProcPid,
757 context->module_generation);
758 LLVMWriteBitcodeToFile(context->module, filename);
759 pfree(filename);
760 }
761
762 handle = (LLVMJitHandle *)
763 MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
764
765 /*
766 * Emit the code. Note that this can, depending on the optimization
767 * settings, take noticeable resources as code emission executes low-level
768 * instruction combining/selection passes etc. Without optimization a
769 * faster instruction selection mechanism is used.
770 */
771 INSTR_TIME_SET_CURRENT(starttime);
772 {
773 LLVMOrcThreadSafeModuleRef ts_module;
774 LLVMErrorRef error;
775 LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
776
777 ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
778
779 handle->lljit = compile_orc;
780 handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
781
782 /*
783 * NB: This doesn't actually emit code. That happens lazily the first
784 * time a symbol defined in the module is requested. Due to that
785 * llvm_get_function() also accounts for emission time.
786 */
787
788 context->module = NULL; /* will be owned by LLJIT */
789 error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
790 handle->resource_tracker,
791 ts_module);
792
793 if (error)
794 elog(ERROR, "failed to JIT module: %s",
795 llvm_error_message(error));
796
797 /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
798 }
799
800 INSTR_TIME_SET_CURRENT(endtime);
801 INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
802 endtime, starttime);
803
804 context->module = NULL;
805 context->compiled = true;
806
807 /* remember emitted code for cleanup and lookups */
808 oldcontext = MemoryContextSwitchTo(TopMemoryContext);
809 context->handles = lappend(context->handles, handle);
810 MemoryContextSwitchTo(oldcontext);
811
812 ereport(DEBUG1,
813 (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
814 INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
815 INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
816 INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
817 errhidestmt(true),
818 errhidecontext(true)));
819 }
820
821 /*
822 * Per session initialization.
823 */
824 static void
825 llvm_session_initialize(void)
826 {
827 MemoryContext oldcontext;
828 char *error = NULL;
829 char *cpu = NULL;
830 char *features = NULL;
831 LLVMTargetMachineRef opt0_tm;
832 LLVMTargetMachineRef opt3_tm;
833
834 if (llvm_session_initialized)
835 return;
836
837 oldcontext = MemoryContextSwitchTo(TopMemoryContext);
838
839 LLVMInitializeNativeTarget();
840 LLVMInitializeNativeAsmPrinter();
841 LLVMInitializeNativeAsmParser();
842
843 if (llvm_context == NULL)
844 {
845 llvm_context = LLVMContextCreate();
846
847 llvm_jit_context_in_use_count = 0;
848 llvm_llvm_context_reuse_count = 0;
849 }
850
851 /*
852 * Synchronize types early, as that also includes inferring the target
853 * triple.
854 */
855 llvm_create_types();
856
857 /*
858 * Extract target information from loaded module.
859 */
860 llvm_set_target();
861
862 if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
863 {
864 elog(FATAL, "failed to query triple %s", error);
865 }
866
867 /*
868 * We want the generated code to use all available features. Therefore
869 * grab the host CPU string and detect features of the current CPU. The
870 * latter is needed because some CPU architectures default to enabling
871 * features not all CPUs have (weird, huh).
872 */
873 cpu = LLVMGetHostCPUName();
874 features = LLVMGetHostCPUFeatures();
875 elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
876 cpu, features);
877
878 opt0_tm =
879 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
880 LLVMCodeGenLevelNone,
881 LLVMRelocDefault,
882 LLVMCodeModelJITDefault);
883 opt3_tm =
884 LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
885 LLVMCodeGenLevelAggressive,
886 LLVMRelocDefault,
887 LLVMCodeModelJITDefault);
888
889 LLVMDisposeMessage(cpu);
890 cpu = NULL;
891 LLVMDisposeMessage(features);
892 features = NULL;
893
894 /* force symbols in main binary to be loaded */
895 LLVMLoadLibraryPermanently(NULL);
896
897 {
898 llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
899
900 llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
901 opt0_tm = 0;
902
903 llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
904 opt3_tm = 0;
905 }
906
907 on_proc_exit(llvm_shutdown, 0);
908
909 llvm_session_initialized = true;
910
911 MemoryContextSwitchTo(oldcontext);
912 }
913
914 static void
915 llvm_shutdown(int code, Datum arg)
916 {
917 /*
918 * If llvm_shutdown() is reached while in a fatal-on-oom section an error
919 * has occurred in the middle of LLVM code. It is not safe to call back
920 * into LLVM (which is why a FATAL error was thrown).
921 *
922 * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
923 * profiling data won't be written out.
924 */
925 if (llvm_in_fatal_on_oom())
926 {
927 Assert(proc_exit_inprogress);
928 return;
929 }
930
931 if (llvm_jit_context_in_use_count != 0)
932 elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
933 llvm_jit_context_in_use_count);
934
935 {
936 if (llvm_opt3_orc)
937 {
938 LLVMOrcDisposeLLJIT(llvm_opt3_orc);
939 llvm_opt3_orc = NULL;
940 }
941 if (llvm_opt0_orc)
942 {
943 LLVMOrcDisposeLLJIT(llvm_opt0_orc);
944 llvm_opt0_orc = NULL;
945 }
946 if (llvm_ts_context)
947 {
948 LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
949 llvm_ts_context = NULL;
950 }
951 }
952 }
953
954 /* helper for llvm_create_types, returning a function's return type */
955 static LLVMTypeRef
956 load_return_type(LLVMModuleRef mod, const char *name)
957 {
958 LLVMValueRef value;
959 LLVMTypeRef typ;
960
961 /* this'll return a *pointer* to the function */
962 value = LLVMGetNamedFunction(mod, name);
963 if (!value)
964 elog(ERROR, "function %s is unknown", name);
965
966 typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
967
968 return typ;
969 }
970
971 /*
972 * Load triple & layout from clang emitted file so we're guaranteed to be
973 * compatible.
974 */
975 static void
976 llvm_set_target(void)
977 {
978 if (!llvm_types_module)
979 elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
980
981 if (llvm_triple == NULL)
982 llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
983
984 if (llvm_layout == NULL)
985 llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
986 }
987
988 /*
989 * Load required information, types, function signatures from llvmjit_types.c
990 * and make them available in global variables.
991 *
992 * Those global variables are then used while emitting code.
993 */
994 static void
995 llvm_create_types(void)
996 {
997 char path[MAXPGPATH];
998 LLVMMemoryBufferRef buf;
999 char *msg;
1000
1001 snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
1002
1003 /* open file */
1004 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
1005 {
1006 elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1007 path, msg);
1008 }
1009
1010 /* eagerly load contents, going to need it all */
1011 if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
1012 {
1013 elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
1014 }
1015 LLVMDisposeMemoryBuffer(buf);
1016
1017 TypeSizeT = llvm_pg_var_type("TypeSizeT");
1018 TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1019 TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1020 TypePGFunction = llvm_pg_var_type("TypePGFunction");
1021 StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1022 StructExprContext = llvm_pg_var_type("StructExprContext");
1023 StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1024 StructExprState = llvm_pg_var_type("StructExprState");
1025 StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1026 StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1027 StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1028 StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1029 StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1030 StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1031 StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
1032 StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1033 StructAggState = llvm_pg_var_type("StructAggState");
1034 StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1035 StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1036 StructPlanState = llvm_pg_var_type("StructPlanState");
1037 StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
1038
1039 AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1040 ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
1041 ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
1042 }
1043
1044 /*
1045 * Split a symbol into module / function parts. If the function is in the
1046 * main binary (or an external library) *modname will be NULL.
1047 */
1048 void
1049 llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1050 {
1051 /*
1052 * Module function names are pgextern.$module.$funcname
1053 */
1054 if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1055 {
1056 /*
1057 * Symbol names cannot contain a ., therefore we can split based on
1058 * first and last occurrence of one.
1059 */
1060 const char *lastdot;
1061
1062 name += strlen("pgextern.");
1063 lastdot = strrchr(name, '.');
1064 if (lastdot)
1065 {
1066 *modname = pnstrdup(name, lastdot - name);
1067 *funcname = pstrdup(lastdot + 1);
1068 }
1069 else
1070 {
1071 /* hmm, no second dot? */
1072 *modname = NULL;
1073 *funcname = pstrdup(name);
1074 }
1075 }
1076 else
1077 {
1078 *modname = NULL;
1079 *funcname = pstrdup(name);
1080 }
1081 }
1082
1083 /*
1084 * Attempt to resolve symbol, so LLVM can emit a reference to it.
1085 */
1086 static uint64_t
1087 llvm_resolve_symbol(const char *symname, void *ctx)
1088 {
1089 uintptr_t addr;
1090 char *funcname;
1091 char *modname;
1092
1093 /*
1094 * macOS prefixes all object level symbols with an underscore. But neither
1095 * dlsym() nor PG's inliner expect that. So undo.
1096 */
1097 #if defined(__darwin__)
1098 if (symname[0] != '_')
1099 elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1100 symname++;
1101 #endif
1102
1103 llvm_split_symbol_name(symname, &modname, &funcname);
1104
1105 /* functions that aren't resolved to names shouldn't ever get here */
1106 Assert(funcname);
1107
1108 if (modname)
1109 addr = (uintptr_t) load_external_function(modname, funcname,
1110 true, NULL);
1111 else
1112 addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1113
1114 pfree(funcname);
1115 if (modname)
1116 pfree(modname);
1117
1118 /* let LLVM will error out - should never happen */
1119 if (!addr)
1120 elog(WARNING, "failed to resolve name %s", symname);
1121
1122 return (uint64_t) addr;
1123 }
1124
1125 static LLVMErrorRef
1126 llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1127 LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1128 LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1129 LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1130 {
1131 #if LLVM_VERSION_MAJOR > 14
1132 LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1133 #else
1134 LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1135 #endif
1136 LLVMErrorRef error;
1137 LLVMOrcMaterializationUnitRef mu;
1138
1139 for (int i = 0; i < LookupSetSize; i++)
1140 {
1141 const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1142
1143 LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1144 symbols[i].Name = LookupSet[i].Name;
1145 symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1146 symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1147 }
1148
1149 mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1150 error = LLVMOrcJITDylibDefine(JD, mu);
1151 if (error != LLVMErrorSuccess)
1152 LLVMOrcDisposeMaterializationUnit(mu);
1153
1154 pfree(symbols);
1155
1156 return error;
1157 }
1158
1159 /*
1160 * We cannot throw errors through LLVM (without causing a FATAL at least), so
1161 * just use WARNING here. That's OK anyway, as the error is also reported at
1162 * the top level action (with less detail) and there might be multiple
1163 * invocations of errors with details.
1164 *
1165 * This doesn't really happen during normal operation, but in cases like
1166 * symbol resolution breakage. So just using elog(WARNING) is fine.
1167 */
1168 static void
1169 llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1170 {
1171 elog(WARNING, "error during JITing: %s",
1172 llvm_error_message(error));
1173 }
1174
1175 /*
1176 * Create our own object layer, so we can add event listeners.
1177 */
1178 static LLVMOrcObjectLayerRef
1179 llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1180 {
1181 #if LLVM_VERSION_MAJOR >= 22
1182 LLVMOrcObjectLayerRef objlayer =
1183 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManagerReserveAlloc(ES, true);
1184 #elif defined(USE_LLVM_BACKPORT_SECTION_MEMORY_MANAGER)
1185 LLVMOrcObjectLayerRef objlayer =
1186 LLVMOrcCreateRTDyldObjectLinkingLayerWithSafeSectionMemoryManager(ES);
1187 #else
1188 LLVMOrcObjectLayerRef objlayer =
1189 LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1190 #endif
1191
1192
1193 #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1194 if (jit_debugging_support)
1195 {
1196 LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1197
1198 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1199 }
1200 #endif
1201
1202 #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1203 if (jit_profiling_support)
1204 {
1205 LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1206
1207 if (l)
1208 LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1209 }
1210 #endif
1211
1212 return objlayer;
1213 }
1214
1215 /*
1216 * Create LLJIT instance, using the passed in target machine. Note that the
1217 * target machine afterwards is owned by the LLJIT instance.
1218 */
1219 static LLVMOrcLLJITRef
1220 llvm_create_jit_instance(LLVMTargetMachineRef tm)
1221 {
1222 LLVMOrcLLJITRef lljit;
1223 LLVMOrcJITTargetMachineBuilderRef tm_builder;
1224 LLVMOrcLLJITBuilderRef lljit_builder;
1225 LLVMErrorRef error;
1226 LLVMOrcDefinitionGeneratorRef main_gen;
1227 LLVMOrcDefinitionGeneratorRef ref_gen;
1228
1229 lljit_builder = LLVMOrcCreateLLJITBuilder();
1230 tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1231 LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1232
1233 LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1234 llvm_create_object_layer,
1235 NULL);
1236
1237 error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1238 if (error)
1239 elog(ERROR, "failed to create lljit instance: %s",
1240 llvm_error_message(error));
1241
1242 LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1243 llvm_log_jit_error, NULL);
1244
1245 /*
1246 * Symbol resolution support for symbols in the postgres binary /
1247 * libraries already loaded.
1248 */
1249 error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1250 LLVMOrcLLJITGetGlobalPrefix(lljit),
1251 0, NULL);
1252 if (error)
1253 elog(ERROR, "failed to create generator: %s",
1254 llvm_error_message(error));
1255 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1256
1257 /*
1258 * Symbol resolution support for "special" functions, e.g. a call into an
1259 * SQL callable function.
1260 */
1261 #if LLVM_VERSION_MAJOR > 14
1262 ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1263 #else
1264 ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1265 #endif
1266 LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1267
1268 return lljit;
1269 }
1270
1271 static char *
1272 llvm_error_message(LLVMErrorRef error)
1273 {
1274 char *orig = LLVMGetErrorMessage(error);
1275 char *msg = pstrdup(orig);
1276
1277 LLVMDisposeErrorMessage(orig);
1278
1279 return msg;
1280 }
1281
1282 /*
1283 * ResourceOwner callbacks
1284 */
1285 static void
1286 ResOwnerReleaseJitContext(Datum res)
1287 {
1288 LLVMJitContext *context = (LLVMJitContext *) DatumGetPointer(res);
1289
1290 context->resowner = NULL;
1291 jit_release_context(&context->base);
1292 }