| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* llvmjit_deform.c |
| 4 |
* Generate code for deforming a heap tuple. |
| 5 |
* |
| 6 |
* This gains performance benefits over unJITed deforming from compile-time |
| 7 |
* knowledge of the tuple descriptor. Fixed column widths, NOT NULLness, etc |
| 8 |
* can be taken advantage of. |
| 9 |
* |
| 10 |
* Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group |
| 11 |
* Portions Copyright (c) 1994, Regents of the University of California |
| 12 |
* |
| 13 |
* IDENTIFICATION |
| 14 |
* src/backend/jit/llvm/llvmjit_deform.c |
| 15 |
* |
| 16 |
*------------------------------------------------------------------------- |
| 17 |
*/ |
| 18 |
|
| 19 |
#include "postgres.h" |
| 20 |
|
| 21 |
#include <llvm-c/Core.h> |
| 22 |
|
| 23 |
#include "access/htup_details.h" |
| 24 |
#include "access/tupdesc_details.h" |
| 25 |
#include "executor/tuptable.h" |
| 26 |
#include "jit/llvmjit.h" |
| 27 |
#include "jit/llvmjit_emit.h" |
| 28 |
|
| 29 |
|
| 30 |
/* |
| 31 |
* Create a function that deforms a tuple of type desc up to natts columns. |
| 32 |
*/ |
| 33 |
LLVMValueRef |
| 34 |
slot_compile_deform(LLVMJitContext *context, TupleDesc desc, |
| 35 |
const TupleTableSlotOps *ops, int natts) |
| 36 |
{ |
| 37 |
char *funcname; |
| 38 |
|
| 39 |
LLVMModuleRef mod; |
| 40 |
LLVMContextRef lc; |
| 41 |
LLVMBuilderRef b; |
| 42 |
|
| 43 |
LLVMTypeRef deform_sig; |
| 44 |
LLVMValueRef v_deform_fn; |
| 45 |
|
| 46 |
LLVMBasicBlockRef b_entry; |
| 47 |
LLVMBasicBlockRef b_adjust_unavail_cols; |
| 48 |
LLVMBasicBlockRef b_find_start; |
| 49 |
|
| 50 |
LLVMBasicBlockRef b_out; |
| 51 |
LLVMBasicBlockRef b_dead; |
| 52 |
LLVMBasicBlockRef *attcheckattnoblocks; |
| 53 |
LLVMBasicBlockRef *attstartblocks; |
| 54 |
LLVMBasicBlockRef *attisnullblocks; |
| 55 |
LLVMBasicBlockRef *attcheckalignblocks; |
| 56 |
LLVMBasicBlockRef *attalignblocks; |
| 57 |
LLVMBasicBlockRef *attstoreblocks; |
| 58 |
|
| 59 |
LLVMValueRef v_offp; |
| 60 |
|
| 61 |
LLVMValueRef v_tupdata_base; |
| 62 |
LLVMValueRef v_tts_values; |
| 63 |
LLVMValueRef v_tts_nulls; |
| 64 |
LLVMValueRef v_slotoffp; |
| 65 |
LLVMValueRef v_flagsp; |
| 66 |
LLVMValueRef v_nvalidp; |
| 67 |
LLVMValueRef v_nvalid; |
| 68 |
LLVMValueRef v_maxatt; |
| 69 |
|
| 70 |
LLVMValueRef v_slot; |
| 71 |
|
| 72 |
LLVMValueRef v_tupleheaderp; |
| 73 |
LLVMValueRef v_tuplep; |
| 74 |
LLVMValueRef v_infomask1; |
| 75 |
LLVMValueRef v_infomask2; |
| 76 |
LLVMValueRef v_bits; |
| 77 |
|
| 78 |
LLVMValueRef v_hoff; |
| 79 |
|
| 80 |
LLVMValueRef v_hasnulls; |
| 81 |
|
| 82 |
/* last column (0 indexed) guaranteed to exist */ |
| 83 |
int guaranteed_column_number = -1; |
| 84 |
|
| 85 |
/* current known alignment */ |
| 86 |
int known_alignment = 0; |
| 87 |
|
| 88 |
/* if true, known_alignment describes definite offset of column */ |
| 89 |
bool attguaranteedalign = true; |
| 90 |
|
| 91 |
int attnum; |
| 92 |
|
| 93 |
/* virtual tuples never need deforming, so don't generate code */ |
| 94 |
if (ops == &TTSOpsVirtual) |
| 95 |
return NULL; |
| 96 |
|
| 97 |
/* decline to JIT for slot types we don't know to handle */ |
| 98 |
if (ops != &TTSOpsHeapTuple && ops != &TTSOpsBufferHeapTuple && |
| 99 |
ops != &TTSOpsMinimalTuple) |
| 100 |
return NULL; |
| 101 |
|
| 102 |
mod = llvm_mutable_module(context); |
| 103 |
lc = LLVMGetModuleContext(mod); |
| 104 |
|
| 105 |
funcname = llvm_expand_funcname(context, "deform"); |
| 106 |
|
| 107 |
/* |
| 108 |
* Check which columns have to exist in all tuples, so we don't have to |
| 109 |
* check the row's natts unnecessarily. |
| 110 |
*/ |
| 111 |
for (attnum = 0; attnum < desc->natts; attnum++) |
| 112 |
{ |
| 113 |
CompactAttribute *catt = TupleDescCompactAttr(desc, attnum); |
| 114 |
Form_pg_attribute attr = TupleDescAttr(desc, attnum); |
| 115 |
|
| 116 |
/* |
| 117 |
* If the column is declared NOT NULL then it must be present in every |
| 118 |
* tuple, unless there's a "missing" entry that could provide a |
| 119 |
* non-NULL value for it. That in turn guarantees that the NULL bitmap |
| 120 |
* - if there are any NULLable columns - is at least long enough to |
| 121 |
* cover columns up to attnum. We treat virtual generated columns |
| 122 |
* similar to atthasmissing columns, as these columns could either not |
| 123 |
* be represented in the tuple or could have the column represented as |
| 124 |
* a NULL in the null bitmap. |
| 125 |
* |
| 126 |
* Be paranoid and also check !attisdropped, even though the |
| 127 |
* combination of attisdropped && attnotnull combination shouldn't |
| 128 |
* exist. |
| 129 |
*/ |
| 130 |
if (catt->attnullability == ATTNULLABLE_VALID && |
| 131 |
!catt->atthasmissing && |
| 132 |
!catt->attisdropped && |
| 133 |
attr->attgenerated != ATTRIBUTE_GENERATED_VIRTUAL) |
| 134 |
guaranteed_column_number = attnum; |
| 135 |
} |
| 136 |
|
| 137 |
/* Create the signature and function */ |
| 138 |
{ |
| 139 |
LLVMTypeRef param_types[1]; |
| 140 |
|
| 141 |
param_types[0] = l_ptr(StructTupleTableSlot); |
| 142 |
|
| 143 |
deform_sig = LLVMFunctionType(LLVMVoidTypeInContext(lc), |
| 144 |
param_types, lengthof(param_types), 0); |
| 145 |
} |
| 146 |
v_deform_fn = LLVMAddFunction(mod, funcname, deform_sig); |
| 147 |
LLVMSetLinkage(v_deform_fn, LLVMInternalLinkage); |
| 148 |
LLVMSetParamAlignment(LLVMGetParam(v_deform_fn, 0), MAXIMUM_ALIGNOF); |
| 149 |
llvm_copy_attributes(AttributeTemplate, v_deform_fn); |
| 150 |
|
| 151 |
b_entry = |
| 152 |
LLVMAppendBasicBlockInContext(lc, v_deform_fn, "entry"); |
| 153 |
b_adjust_unavail_cols = |
| 154 |
LLVMAppendBasicBlockInContext(lc, v_deform_fn, "adjust_unavail_cols"); |
| 155 |
b_find_start = |
| 156 |
LLVMAppendBasicBlockInContext(lc, v_deform_fn, "find_startblock"); |
| 157 |
b_out = |
| 158 |
LLVMAppendBasicBlockInContext(lc, v_deform_fn, "outblock"); |
| 159 |
b_dead = |
| 160 |
LLVMAppendBasicBlockInContext(lc, v_deform_fn, "deadblock"); |
| 161 |
|
| 162 |
b = LLVMCreateBuilderInContext(lc); |
| 163 |
|
| 164 |
attcheckattnoblocks = palloc(sizeof(LLVMBasicBlockRef) * natts); |
| 165 |
attstartblocks = palloc(sizeof(LLVMBasicBlockRef) * natts); |
| 166 |
attisnullblocks = palloc(sizeof(LLVMBasicBlockRef) * natts); |
| 167 |
attcheckalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts); |
| 168 |
attalignblocks = palloc(sizeof(LLVMBasicBlockRef) * natts); |
| 169 |
attstoreblocks = palloc(sizeof(LLVMBasicBlockRef) * natts); |
| 170 |
|
| 171 |
known_alignment = 0; |
| 172 |
|
| 173 |
LLVMPositionBuilderAtEnd(b, b_entry); |
| 174 |
|
| 175 |
/* perform allocas first, llvm only converts those to registers */ |
| 176 |
v_offp = LLVMBuildAlloca(b, TypeSizeT, "v_offp"); |
| 177 |
|
| 178 |
v_slot = LLVMGetParam(v_deform_fn, 0); |
| 179 |
|
| 180 |
v_tts_values = |
| 181 |
l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_VALUES, |
| 182 |
"tts_values"); |
| 183 |
v_tts_nulls = |
| 184 |
l_load_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_ISNULL, |
| 185 |
"tts_ISNULL"); |
| 186 |
v_flagsp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_FLAGS, ""); |
| 187 |
v_nvalidp = l_struct_gep(b, StructTupleTableSlot, v_slot, FIELDNO_TUPLETABLESLOT_NVALID, ""); |
| 188 |
|
| 189 |
if (ops == &TTSOpsHeapTuple || ops == &TTSOpsBufferHeapTuple) |
| 190 |
{ |
| 191 |
LLVMValueRef v_heapslot; |
| 192 |
|
| 193 |
v_heapslot = |
| 194 |
LLVMBuildBitCast(b, |
| 195 |
v_slot, |
| 196 |
l_ptr(StructHeapTupleTableSlot), |
| 197 |
"heapslot"); |
| 198 |
v_slotoffp = l_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_OFF, ""); |
| 199 |
v_tupleheaderp = |
| 200 |
l_load_struct_gep(b, StructHeapTupleTableSlot, v_heapslot, FIELDNO_HEAPTUPLETABLESLOT_TUPLE, |
| 201 |
"tupleheader"); |
| 202 |
} |
| 203 |
else if (ops == &TTSOpsMinimalTuple) |
| 204 |
{ |
| 205 |
LLVMValueRef v_minimalslot; |
| 206 |
|
| 207 |
v_minimalslot = |
| 208 |
LLVMBuildBitCast(b, |
| 209 |
v_slot, |
| 210 |
l_ptr(StructMinimalTupleTableSlot), |
| 211 |
"minimalslot"); |
| 212 |
v_slotoffp = l_struct_gep(b, |
| 213 |
StructMinimalTupleTableSlot, |
| 214 |
v_minimalslot, |
| 215 |
FIELDNO_MINIMALTUPLETABLESLOT_OFF, ""); |
| 216 |
v_tupleheaderp = |
| 217 |
l_load_struct_gep(b, |
| 218 |
StructMinimalTupleTableSlot, |
| 219 |
v_minimalslot, |
| 220 |
FIELDNO_MINIMALTUPLETABLESLOT_TUPLE, |
| 221 |
"tupleheader"); |
| 222 |
} |
| 223 |
else |
| 224 |
{ |
| 225 |
/* should've returned at the start of the function */ |
| 226 |
pg_unreachable(); |
| 227 |
} |
| 228 |
|
| 229 |
v_tuplep = |
| 230 |
l_load_struct_gep(b, |
| 231 |
StructHeapTupleData, |
| 232 |
v_tupleheaderp, |
| 233 |
FIELDNO_HEAPTUPLEDATA_DATA, |
| 234 |
"tuple"); |
| 235 |
v_bits = |
| 236 |
LLVMBuildBitCast(b, |
| 237 |
l_struct_gep(b, |
| 238 |
StructHeapTupleHeaderData, |
| 239 |
v_tuplep, |
| 240 |
FIELDNO_HEAPTUPLEHEADERDATA_BITS, |
| 241 |
""), |
| 242 |
l_ptr(LLVMInt8TypeInContext(lc)), |
| 243 |
"t_bits"); |
| 244 |
v_infomask1 = |
| 245 |
l_load_struct_gep(b, |
| 246 |
StructHeapTupleHeaderData, |
| 247 |
v_tuplep, |
| 248 |
FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK, |
| 249 |
"infomask1"); |
| 250 |
v_infomask2 = |
| 251 |
l_load_struct_gep(b, |
| 252 |
StructHeapTupleHeaderData, |
| 253 |
v_tuplep, FIELDNO_HEAPTUPLEHEADERDATA_INFOMASK2, |
| 254 |
"infomask2"); |
| 255 |
|
| 256 |
/* t_infomask & HEAP_HASNULL */ |
| 257 |
v_hasnulls = |
| 258 |
LLVMBuildICmp(b, LLVMIntNE, |
| 259 |
LLVMBuildAnd(b, |
| 260 |
l_int16_const(lc, HEAP_HASNULL), |
| 261 |
v_infomask1, ""), |
| 262 |
l_int16_const(lc, 0), |
| 263 |
"hasnulls"); |
| 264 |
|
| 265 |
/* t_infomask2 & HEAP_NATTS_MASK */ |
| 266 |
v_maxatt = LLVMBuildAnd(b, |
| 267 |
l_int16_const(lc, HEAP_NATTS_MASK), |
| 268 |
v_infomask2, |
| 269 |
"maxatt"); |
| 270 |
|
| 271 |
/* |
| 272 |
* Need to zext, as getelementptr otherwise treats hoff as a signed 8bit |
| 273 |
* integer, which'd yield a negative offset for t_hoff > 127. |
| 274 |
*/ |
| 275 |
v_hoff = |
| 276 |
LLVMBuildZExt(b, |
| 277 |
l_load_struct_gep(b, |
| 278 |
StructHeapTupleHeaderData, |
| 279 |
v_tuplep, |
| 280 |
FIELDNO_HEAPTUPLEHEADERDATA_HOFF, |
| 281 |
""), |
| 282 |
LLVMInt32TypeInContext(lc), "t_hoff"); |
| 283 |
|
| 284 |
v_tupdata_base = l_gep(b, |
| 285 |
LLVMInt8TypeInContext(lc), |
| 286 |
LLVMBuildBitCast(b, |
| 287 |
v_tuplep, |
| 288 |
l_ptr(LLVMInt8TypeInContext(lc)), |
| 289 |
""), |
| 290 |
&v_hoff, 1, |
| 291 |
"v_tupdata_base"); |
| 292 |
|
| 293 |
/* |
| 294 |
* Load tuple start offset from slot. Will be reset below in case there's |
| 295 |
* no existing deformed columns in slot. |
| 296 |
*/ |
| 297 |
{ |
| 298 |
LLVMValueRef v_off_start; |
| 299 |
|
| 300 |
v_off_start = l_load(b, LLVMInt32TypeInContext(lc), v_slotoffp, "v_slot_off"); |
| 301 |
v_off_start = LLVMBuildZExt(b, v_off_start, TypeSizeT, ""); |
| 302 |
LLVMBuildStore(b, v_off_start, v_offp); |
| 303 |
} |
| 304 |
|
| 305 |
/* build the basic block for each attribute, need them as jump target */ |
| 306 |
for (attnum = 0; attnum < natts; attnum++) |
| 307 |
{ |
| 308 |
attcheckattnoblocks[attnum] = |
| 309 |
l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckattno", attnum); |
| 310 |
attstartblocks[attnum] = |
| 311 |
l_bb_append_v(v_deform_fn, "block.attr.%d.start", attnum); |
| 312 |
attisnullblocks[attnum] = |
| 313 |
l_bb_append_v(v_deform_fn, "block.attr.%d.attisnull", attnum); |
| 314 |
attcheckalignblocks[attnum] = |
| 315 |
l_bb_append_v(v_deform_fn, "block.attr.%d.attcheckalign", attnum); |
| 316 |
attalignblocks[attnum] = |
| 317 |
l_bb_append_v(v_deform_fn, "block.attr.%d.align", attnum); |
| 318 |
attstoreblocks[attnum] = |
| 319 |
l_bb_append_v(v_deform_fn, "block.attr.%d.store", attnum); |
| 320 |
} |
| 321 |
|
| 322 |
/* |
| 323 |
* Check if it is guaranteed that all the desired attributes are available |
| 324 |
* in the tuple (but still possibly NULL), by dint of either the last |
| 325 |
* to-be-deformed column being NOT NULL, or subsequent ones not accessed |
| 326 |
* here being NOT NULL. If that's not guaranteed the tuple headers natt's |
| 327 |
* has to be checked, and missing attributes potentially have to be |
| 328 |
* fetched (using slot_getmissingattrs(). |
| 329 |
*/ |
| 330 |
if ((natts - 1) <= guaranteed_column_number) |
| 331 |
{ |
| 332 |
/* just skip through unnecessary blocks */ |
| 333 |
LLVMBuildBr(b, b_adjust_unavail_cols); |
| 334 |
LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols); |
| 335 |
LLVMBuildBr(b, b_find_start); |
| 336 |
} |
| 337 |
else |
| 338 |
{ |
| 339 |
LLVMValueRef v_params[3]; |
| 340 |
LLVMValueRef f; |
| 341 |
|
| 342 |
/* branch if not all columns available */ |
| 343 |
LLVMBuildCondBr(b, |
| 344 |
LLVMBuildICmp(b, LLVMIntULT, |
| 345 |
v_maxatt, |
| 346 |
l_int16_const(lc, natts), |
| 347 |
""), |
| 348 |
b_adjust_unavail_cols, |
| 349 |
b_find_start); |
| 350 |
|
| 351 |
/* if not, memset tts_isnull of relevant cols to true */ |
| 352 |
LLVMPositionBuilderAtEnd(b, b_adjust_unavail_cols); |
| 353 |
|
| 354 |
v_params[0] = v_slot; |
| 355 |
v_params[1] = LLVMBuildZExt(b, v_maxatt, LLVMInt32TypeInContext(lc), ""); |
| 356 |
v_params[2] = l_int32_const(lc, natts); |
| 357 |
f = llvm_pg_func(mod, "slot_getmissingattrs"); |
| 358 |
l_call(b, |
| 359 |
LLVMGetFunctionType(f), f, |
| 360 |
v_params, lengthof(v_params), ""); |
| 361 |
LLVMBuildBr(b, b_find_start); |
| 362 |
} |
| 363 |
|
| 364 |
LLVMPositionBuilderAtEnd(b, b_find_start); |
| 365 |
|
| 366 |
v_nvalid = l_load(b, LLVMInt16TypeInContext(lc), v_nvalidp, ""); |
| 367 |
|
| 368 |
/* |
| 369 |
* Build switch to go from nvalid to the right startblock. Callers |
| 370 |
* currently don't have the knowledge, but it'd be good for performance to |
| 371 |
* avoid this check when it's known that the slot is empty (e.g. in scan |
| 372 |
* nodes). |
| 373 |
*/ |
| 374 |
if (true) |
| 375 |
{ |
| 376 |
LLVMValueRef v_switch = LLVMBuildSwitch(b, v_nvalid, |
| 377 |
b_dead, natts); |
| 378 |
|
| 379 |
for (attnum = 0; attnum < natts; attnum++) |
| 380 |
{ |
| 381 |
LLVMValueRef v_attno = l_int16_const(lc, attnum); |
| 382 |
|
| 383 |
LLVMAddCase(v_switch, v_attno, attcheckattnoblocks[attnum]); |
| 384 |
} |
| 385 |
} |
| 386 |
else |
| 387 |
{ |
| 388 |
/* jump from entry block to first block */ |
| 389 |
LLVMBuildBr(b, attcheckattnoblocks[0]); |
| 390 |
} |
| 391 |
|
| 392 |
LLVMPositionBuilderAtEnd(b, b_dead); |
| 393 |
LLVMBuildUnreachable(b); |
| 394 |
|
| 395 |
/* |
| 396 |
* Iterate over each attribute that needs to be deformed, build code to |
| 397 |
* deform it. |
| 398 |
*/ |
| 399 |
for (attnum = 0; attnum < natts; attnum++) |
| 400 |
{ |
| 401 |
CompactAttribute *att = TupleDescCompactAttr(desc, attnum); |
| 402 |
Form_pg_attribute attr = TupleDescAttr(desc, attnum); |
| 403 |
|
| 404 |
LLVMValueRef v_incby; |
| 405 |
int alignto = att->attalignby; |
| 406 |
LLVMValueRef l_attno = l_int16_const(lc, attnum); |
| 407 |
LLVMValueRef v_attdatap; |
| 408 |
LLVMValueRef v_resultp; |
| 409 |
|
| 410 |
/* build block checking whether we did all the necessary attributes */ |
| 411 |
LLVMPositionBuilderAtEnd(b, attcheckattnoblocks[attnum]); |
| 412 |
|
| 413 |
/* |
| 414 |
* If this is the first attribute, slot->tts_nvalid was 0. Therefore |
| 415 |
* also reset offset to 0, it may be from a previous execution. |
| 416 |
*/ |
| 417 |
if (attnum == 0) |
| 418 |
{ |
| 419 |
LLVMBuildStore(b, l_sizet_const(0), v_offp); |
| 420 |
} |
| 421 |
|
| 422 |
/* |
| 423 |
* Build check whether column is available (i.e. whether the tuple has |
| 424 |
* that many columns stored). We can avoid the branch if we know |
| 425 |
* there's a subsequent NOT NULL column. |
| 426 |
*/ |
| 427 |
if (attnum <= guaranteed_column_number) |
| 428 |
{ |
| 429 |
LLVMBuildBr(b, attstartblocks[attnum]); |
| 430 |
} |
| 431 |
else |
| 432 |
{ |
| 433 |
LLVMValueRef v_islast; |
| 434 |
|
| 435 |
v_islast = LLVMBuildICmp(b, LLVMIntUGE, |
| 436 |
l_attno, |
| 437 |
v_maxatt, |
| 438 |
"heap_natts"); |
| 439 |
LLVMBuildCondBr(b, v_islast, b_out, attstartblocks[attnum]); |
| 440 |
} |
| 441 |
LLVMPositionBuilderAtEnd(b, attstartblocks[attnum]); |
| 442 |
|
| 443 |
/* |
| 444 |
* Check for nulls if necessary. No need to take missing attributes |
| 445 |
* into account, because if they're present the heaptuple's natts |
| 446 |
* would have indicated that a slot_getmissingattrs() is needed. When |
| 447 |
* present in the tuple, virtual generated columns are always stored |
| 448 |
* as NULL, so we must always perform NULL checks for these. |
| 449 |
*/ |
| 450 |
if (att->attnullability != ATTNULLABLE_VALID || |
| 451 |
attr->attgenerated == ATTRIBUTE_GENERATED_VIRTUAL) |
| 452 |
{ |
| 453 |
LLVMBasicBlockRef b_ifnotnull; |
| 454 |
LLVMBasicBlockRef b_ifnull; |
| 455 |
LLVMBasicBlockRef b_next; |
| 456 |
LLVMValueRef v_attisnull; |
| 457 |
LLVMValueRef v_nullbyteno; |
| 458 |
LLVMValueRef v_nullbytemask; |
| 459 |
LLVMValueRef v_nullbyte; |
| 460 |
LLVMValueRef v_nullbit; |
| 461 |
|
| 462 |
b_ifnotnull = attcheckalignblocks[attnum]; |
| 463 |
b_ifnull = attisnullblocks[attnum]; |
| 464 |
|
| 465 |
if (attnum + 1 == natts) |
| 466 |
b_next = b_out; |
| 467 |
else |
| 468 |
b_next = attcheckattnoblocks[attnum + 1]; |
| 469 |
|
| 470 |
v_nullbyteno = l_int32_const(lc, attnum >> 3); |
| 471 |
v_nullbytemask = l_int8_const(lc, 1 << ((attnum) & 0x07)); |
| 472 |
v_nullbyte = l_load_gep1(b, LLVMInt8TypeInContext(lc), v_bits, v_nullbyteno, "attnullbyte"); |
| 473 |
|
| 474 |
v_nullbit = LLVMBuildICmp(b, |
| 475 |
LLVMIntEQ, |
| 476 |
LLVMBuildAnd(b, v_nullbyte, v_nullbytemask, ""), |
| 477 |
l_int8_const(lc, 0), |
| 478 |
"attisnull"); |
| 479 |
|
| 480 |
v_attisnull = LLVMBuildAnd(b, v_hasnulls, v_nullbit, ""); |
| 481 |
|
| 482 |
LLVMBuildCondBr(b, v_attisnull, b_ifnull, b_ifnotnull); |
| 483 |
|
| 484 |
LLVMPositionBuilderAtEnd(b, b_ifnull); |
| 485 |
|
| 486 |
/* store null-byte */ |
| 487 |
LLVMBuildStore(b, |
| 488 |
l_int8_const(lc, 1), |
| 489 |
l_gep(b, LLVMInt8TypeInContext(lc), v_tts_nulls, &l_attno, 1, "")); |
| 490 |
/* store zero datum */ |
| 491 |
LLVMBuildStore(b, |
| 492 |
l_sizet_const(0), |
| 493 |
l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, "")); |
| 494 |
|
| 495 |
LLVMBuildBr(b, b_next); |
| 496 |
attguaranteedalign = false; |
| 497 |
} |
| 498 |
else |
| 499 |
{ |
| 500 |
/* nothing to do */ |
| 501 |
LLVMBuildBr(b, attcheckalignblocks[attnum]); |
| 502 |
LLVMPositionBuilderAtEnd(b, attisnullblocks[attnum]); |
| 503 |
LLVMBuildBr(b, attcheckalignblocks[attnum]); |
| 504 |
} |
| 505 |
LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]); |
| 506 |
|
| 507 |
/* ------ |
| 508 |
* Even if alignment is required, we can skip doing it if provably |
| 509 |
* unnecessary: |
| 510 |
* - first column is guaranteed to be aligned |
| 511 |
* - columns following a NOT NULL fixed width datum have known |
| 512 |
* alignment, can skip alignment computation if that known alignment |
| 513 |
* is compatible with current column. |
| 514 |
* ------ |
| 515 |
*/ |
| 516 |
if (alignto > 1 && |
| 517 |
(known_alignment < 0 || known_alignment != TYPEALIGN(alignto, known_alignment))) |
| 518 |
{ |
| 519 |
/* |
| 520 |
* When accessing a varlena field, we have to "peek" to see if we |
| 521 |
* are looking at a pad byte or the first byte of a 1-byte-header |
| 522 |
* datum. A zero byte must be either a pad byte, or the first |
| 523 |
* byte of a correctly aligned 4-byte length word; in either case, |
| 524 |
* we can align safely. A non-zero byte must be either a 1-byte |
| 525 |
* length word, or the first byte of a correctly aligned 4-byte |
| 526 |
* length word; in either case, we need not align. |
| 527 |
*/ |
| 528 |
if (att->attlen == -1) |
| 529 |
{ |
| 530 |
LLVMValueRef v_possible_padbyte; |
| 531 |
LLVMValueRef v_ispad; |
| 532 |
LLVMValueRef v_off; |
| 533 |
|
| 534 |
/* don't know if short varlena or not */ |
| 535 |
attguaranteedalign = false; |
| 536 |
|
| 537 |
v_off = l_load(b, TypeSizeT, v_offp, ""); |
| 538 |
|
| 539 |
v_possible_padbyte = |
| 540 |
l_load_gep1(b, LLVMInt8TypeInContext(lc), v_tupdata_base, v_off, "padbyte"); |
| 541 |
v_ispad = |
| 542 |
LLVMBuildICmp(b, LLVMIntEQ, |
| 543 |
v_possible_padbyte, l_int8_const(lc, 0), |
| 544 |
"ispadbyte"); |
| 545 |
LLVMBuildCondBr(b, v_ispad, |
| 546 |
attalignblocks[attnum], |
| 547 |
attstoreblocks[attnum]); |
| 548 |
} |
| 549 |
else |
| 550 |
{ |
| 551 |
LLVMBuildBr(b, attalignblocks[attnum]); |
| 552 |
} |
| 553 |
|
| 554 |
LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]); |
| 555 |
|
| 556 |
/* translation of alignment code (cf TYPEALIGN()) */ |
| 557 |
{ |
| 558 |
LLVMValueRef v_off_aligned; |
| 559 |
LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); |
| 560 |
|
| 561 |
/* ((ALIGNVAL) - 1) */ |
| 562 |
LLVMValueRef v_alignval = l_sizet_const(alignto - 1); |
| 563 |
|
| 564 |
/* ((uintptr_t) (LEN) + ((ALIGNVAL) - 1)) */ |
| 565 |
LLVMValueRef v_lh = LLVMBuildAdd(b, v_off, v_alignval, ""); |
| 566 |
|
| 567 |
/* ~((uintptr_t) ((ALIGNVAL) - 1)) */ |
| 568 |
LLVMValueRef v_rh = l_sizet_const(~(alignto - 1)); |
| 569 |
|
| 570 |
v_off_aligned = LLVMBuildAnd(b, v_lh, v_rh, "aligned_offset"); |
| 571 |
|
| 572 |
LLVMBuildStore(b, v_off_aligned, v_offp); |
| 573 |
} |
| 574 |
|
| 575 |
/* |
| 576 |
* As alignment either was unnecessary or has been performed, we |
| 577 |
* now know the current alignment. This is only safe because this |
| 578 |
* value isn't used for varlena and nullable columns. |
| 579 |
*/ |
| 580 |
if (known_alignment >= 0) |
| 581 |
{ |
| 582 |
Assert(known_alignment != 0); |
| 583 |
known_alignment = TYPEALIGN(alignto, known_alignment); |
| 584 |
} |
| 585 |
|
| 586 |
LLVMBuildBr(b, attstoreblocks[attnum]); |
| 587 |
LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]); |
| 588 |
} |
| 589 |
else |
| 590 |
{ |
| 591 |
LLVMPositionBuilderAtEnd(b, attcheckalignblocks[attnum]); |
| 592 |
LLVMBuildBr(b, attalignblocks[attnum]); |
| 593 |
LLVMPositionBuilderAtEnd(b, attalignblocks[attnum]); |
| 594 |
LLVMBuildBr(b, attstoreblocks[attnum]); |
| 595 |
} |
| 596 |
LLVMPositionBuilderAtEnd(b, attstoreblocks[attnum]); |
| 597 |
|
| 598 |
/* |
| 599 |
* Store the current offset if known to be constant. That allows LLVM |
| 600 |
* to generate better code. Without that LLVM can't figure out that |
| 601 |
* the offset might be constant due to the jumps for previously |
| 602 |
* decoded columns. |
| 603 |
*/ |
| 604 |
if (attguaranteedalign) |
| 605 |
{ |
| 606 |
Assert(known_alignment >= 0); |
| 607 |
LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp); |
| 608 |
} |
| 609 |
|
| 610 |
/* compute what following columns are aligned to */ |
| 611 |
if (att->attlen < 0) |
| 612 |
{ |
| 613 |
/* can't guarantee any alignment after variable length field */ |
| 614 |
known_alignment = -1; |
| 615 |
attguaranteedalign = false; |
| 616 |
} |
| 617 |
else if (att->attnullability == ATTNULLABLE_VALID && |
| 618 |
attguaranteedalign && known_alignment >= 0) |
| 619 |
{ |
| 620 |
/* |
| 621 |
* If the offset to the column was previously known, a NOT NULL & |
| 622 |
* fixed-width column guarantees that alignment is just the |
| 623 |
* previous alignment plus column width. |
| 624 |
*/ |
| 625 |
Assert(att->attlen > 0); |
| 626 |
known_alignment += att->attlen; |
| 627 |
} |
| 628 |
else if (att->attnullability == ATTNULLABLE_VALID && |
| 629 |
attr->attgenerated != ATTRIBUTE_GENERATED_VIRTUAL && |
| 630 |
(att->attlen % alignto) == 0) |
| 631 |
{ |
| 632 |
/* |
| 633 |
* After a NOT NULL (and not virtual generated) fixed-width column |
| 634 |
* with a length that is a multiple of its alignment requirement, |
| 635 |
* we know the following column is aligned to at least the current |
| 636 |
* column's alignment. |
| 637 |
*/ |
| 638 |
Assert(att->attlen > 0); |
| 639 |
known_alignment = alignto; |
| 640 |
Assert(known_alignment > 0); |
| 641 |
attguaranteedalign = false; |
| 642 |
} |
| 643 |
else |
| 644 |
{ |
| 645 |
known_alignment = -1; |
| 646 |
attguaranteedalign = false; |
| 647 |
} |
| 648 |
|
| 649 |
|
| 650 |
/* compute address to load data from */ |
| 651 |
{ |
| 652 |
LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); |
| 653 |
|
| 654 |
v_attdatap = |
| 655 |
l_gep(b, LLVMInt8TypeInContext(lc), v_tupdata_base, &v_off, 1, ""); |
| 656 |
} |
| 657 |
|
| 658 |
/* compute address to store value at */ |
| 659 |
v_resultp = l_gep(b, TypeSizeT, v_tts_values, &l_attno, 1, ""); |
| 660 |
|
| 661 |
/* store null-byte (false) */ |
| 662 |
LLVMBuildStore(b, l_int8_const(lc, 0), |
| 663 |
l_gep(b, TypeStorageBool, v_tts_nulls, &l_attno, 1, "")); |
| 664 |
|
| 665 |
/* |
| 666 |
* Store datum. For byval: datums copy the value, extend to Datum's |
| 667 |
* width, and store. For byref types: store pointer to data. |
| 668 |
*/ |
| 669 |
if (att->attbyval) |
| 670 |
{ |
| 671 |
LLVMValueRef v_tmp_loaddata; |
| 672 |
LLVMTypeRef vartype = LLVMIntTypeInContext(lc, att->attlen * 8); |
| 673 |
LLVMTypeRef vartypep = LLVMPointerType(vartype, 0); |
| 674 |
|
| 675 |
v_tmp_loaddata = |
| 676 |
LLVMBuildPointerCast(b, v_attdatap, vartypep, ""); |
| 677 |
v_tmp_loaddata = l_load(b, vartype, v_tmp_loaddata, "attr_byval"); |
| 678 |
v_tmp_loaddata = LLVMBuildSExt(b, v_tmp_loaddata, TypeSizeT, ""); |
| 679 |
|
| 680 |
LLVMBuildStore(b, v_tmp_loaddata, v_resultp); |
| 681 |
} |
| 682 |
else |
| 683 |
{ |
| 684 |
LLVMValueRef v_tmp_loaddata; |
| 685 |
|
| 686 |
/* store pointer */ |
| 687 |
v_tmp_loaddata = |
| 688 |
LLVMBuildPtrToInt(b, |
| 689 |
v_attdatap, |
| 690 |
TypeSizeT, |
| 691 |
"attr_ptr"); |
| 692 |
LLVMBuildStore(b, v_tmp_loaddata, v_resultp); |
| 693 |
} |
| 694 |
|
| 695 |
/* increment data pointer */ |
| 696 |
if (att->attlen > 0) |
| 697 |
{ |
| 698 |
v_incby = l_sizet_const(att->attlen); |
| 699 |
} |
| 700 |
else if (att->attlen == -1) |
| 701 |
{ |
| 702 |
v_incby = l_call(b, |
| 703 |
llvm_pg_var_func_type("varsize_any"), |
| 704 |
llvm_pg_func(mod, "varsize_any"), |
| 705 |
&v_attdatap, 1, |
| 706 |
"varsize_any"); |
| 707 |
l_callsite_ro(v_incby); |
| 708 |
l_callsite_alwaysinline(v_incby); |
| 709 |
} |
| 710 |
else if (att->attlen == -2) |
| 711 |
{ |
| 712 |
v_incby = l_call(b, |
| 713 |
llvm_pg_var_func_type("strlen"), |
| 714 |
llvm_pg_func(mod, "strlen"), |
| 715 |
&v_attdatap, 1, "strlen"); |
| 716 |
|
| 717 |
l_callsite_ro(v_incby); |
| 718 |
|
| 719 |
/* add 1 for NUL byte */ |
| 720 |
v_incby = LLVMBuildAdd(b, v_incby, l_sizet_const(1), ""); |
| 721 |
} |
| 722 |
else |
| 723 |
{ |
| 724 |
Assert(false); |
| 725 |
v_incby = NULL; /* silence compiler */ |
| 726 |
} |
| 727 |
|
| 728 |
if (attguaranteedalign) |
| 729 |
{ |
| 730 |
Assert(known_alignment >= 0); |
| 731 |
LLVMBuildStore(b, l_sizet_const(known_alignment), v_offp); |
| 732 |
} |
| 733 |
else |
| 734 |
{ |
| 735 |
LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); |
| 736 |
|
| 737 |
v_off = LLVMBuildAdd(b, v_off, v_incby, "increment_offset"); |
| 738 |
LLVMBuildStore(b, v_off, v_offp); |
| 739 |
} |
| 740 |
|
| 741 |
/* |
| 742 |
* jump to next block, unless last possible column, or all desired |
| 743 |
* (available) attributes have been fetched. |
| 744 |
*/ |
| 745 |
if (attnum + 1 == natts) |
| 746 |
{ |
| 747 |
/* jump out */ |
| 748 |
LLVMBuildBr(b, b_out); |
| 749 |
} |
| 750 |
else |
| 751 |
{ |
| 752 |
LLVMBuildBr(b, attcheckattnoblocks[attnum + 1]); |
| 753 |
} |
| 754 |
} |
| 755 |
|
| 756 |
|
| 757 |
/* build block that returns */ |
| 758 |
LLVMPositionBuilderAtEnd(b, b_out); |
| 759 |
|
| 760 |
{ |
| 761 |
LLVMValueRef v_off = l_load(b, TypeSizeT, v_offp, ""); |
| 762 |
LLVMValueRef v_flags; |
| 763 |
|
| 764 |
LLVMBuildStore(b, l_int16_const(lc, natts), v_nvalidp); |
| 765 |
v_off = LLVMBuildTrunc(b, v_off, LLVMInt32TypeInContext(lc), ""); |
| 766 |
LLVMBuildStore(b, v_off, v_slotoffp); |
| 767 |
v_flags = l_load(b, LLVMInt16TypeInContext(lc), v_flagsp, "tts_flags"); |
| 768 |
v_flags = LLVMBuildOr(b, v_flags, l_int16_const(lc, TTS_FLAG_SLOW), ""); |
| 769 |
LLVMBuildStore(b, v_flags, v_flagsp); |
| 770 |
LLVMBuildRetVoid(b); |
| 771 |
} |
| 772 |
|
| 773 |
LLVMDisposeBuilder(b); |
| 774 |
|
| 775 |
return v_deform_fn; |
| 776 |
} |