| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* pg_crc32c_sse42_choose.c |
| 4 |
* Choose between Intel SSE 4.2 and software CRC-32C implementation. |
| 5 |
* |
| 6 |
* On first call, checks if the CPU we're running on supports Intel SSE |
| 7 |
* 4.2. If it does, use the special SSE instructions for CRC-32C |
| 8 |
* computation. Otherwise, fall back to the pure software implementation |
| 9 |
* (slicing-by-8). |
| 10 |
* |
| 11 |
* Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group |
| 12 |
* Portions Copyright (c) 1994, Regents of the University of California |
| 13 |
* |
| 14 |
* |
| 15 |
* IDENTIFICATION |
| 16 |
* src/port/pg_crc32c_sse42_choose.c |
| 17 |
* |
| 18 |
*------------------------------------------------------------------------- |
| 19 |
*/ |
| 20 |
|
| 21 |
#include "c.h" |
| 22 |
|
| 23 |
#if defined(HAVE__GET_CPUID) || defined(HAVE__GET_CPUID_COUNT) |
| 24 |
#include <cpuid.h> |
| 25 |
#endif |
| 26 |
|
| 27 |
#if defined(HAVE__CPUID) || defined(HAVE__CPUIDEX) |
| 28 |
#include <intrin.h> |
| 29 |
#endif |
| 30 |
|
| 31 |
#ifdef HAVE_XSAVE_INTRINSICS |
| 32 |
#include <immintrin.h> |
| 33 |
#endif |
| 34 |
|
| 35 |
#include "port/pg_crc32c.h" |
| 36 |
|
| 37 |
/* |
| 38 |
* Does XGETBV say the ZMM registers are enabled? |
| 39 |
* |
| 40 |
* NB: Caller is responsible for verifying that osxsave is available |
| 41 |
* before calling this. |
| 42 |
*/ |
| 43 |
#ifdef HAVE_XSAVE_INTRINSICS |
| 44 |
pg_attribute_target("xsave") |
| 45 |
#endif |
| 46 |
static bool |
| 47 |
zmm_regs_available(void) |
| 48 |
{ |
| 49 |
#ifdef HAVE_XSAVE_INTRINSICS |
| 50 |
return (_xgetbv(0) & 0xe6) == 0xe6; |
| 51 |
#else |
| 52 |
return false; |
| 53 |
#endif |
| 54 |
} |
| 55 |
|
| 56 |
/* |
| 57 |
* This gets called on the first call. It replaces the function pointer |
| 58 |
* so that subsequent calls are routed directly to the chosen implementation. |
| 59 |
*/ |
| 60 |
static pg_crc32c |
| 61 |
pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len) |
| 62 |
{ |
| 63 |
unsigned int exx[4] = {0, 0, 0, 0}; |
| 64 |
|
| 65 |
/* |
| 66 |
* Set fallback. We must guard since slicing-by-8 is not visible |
| 67 |
* everywhere. |
| 68 |
*/ |
| 69 |
#ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK |
| 70 |
pg_comp_crc32c = pg_comp_crc32c_sb8; |
| 71 |
#endif |
| 72 |
|
| 73 |
#if defined(HAVE__GET_CPUID) |
| 74 |
__get_cpuid(1, &exx[0], &exx[1], &exx[2], &exx[3]); |
| 75 |
#elif defined(HAVE__CPUID) |
| 76 |
__cpuid(exx, 1); |
| 77 |
#else |
| 78 |
#error cpuid instruction not available |
| 79 |
#endif |
| 80 |
|
| 81 |
if ((exx[2] & (1 << 20)) != 0) /* SSE 4.2 */ |
| 82 |
{ |
| 83 |
pg_comp_crc32c = pg_comp_crc32c_sse42; |
| 84 |
|
| 85 |
if (exx[2] & (1 << 27) && /* OSXSAVE */ |
| 86 |
zmm_regs_available()) |
| 87 |
{ |
| 88 |
/* second cpuid call on leaf 7 to check extended AVX-512 support */ |
| 89 |
|
| 90 |
memset(exx, 0, 4 * sizeof(exx[0])); |
| 91 |
|
| 92 |
#if defined(HAVE__GET_CPUID_COUNT) |
| 93 |
__get_cpuid_count(7, 0, &exx[0], &exx[1], &exx[2], &exx[3]); |
| 94 |
#elif defined(HAVE__CPUIDEX) |
| 95 |
__cpuidex(exx, 7, 0); |
| 96 |
#endif |
| 97 |
|
| 98 |
#ifdef USE_AVX512_CRC32C_WITH_RUNTIME_CHECK |
| 99 |
if (exx[2] & (1 << 10) && /* VPCLMULQDQ */ |
| 100 |
exx[1] & (1 << 31)) /* AVX512-VL */ |
| 101 |
pg_comp_crc32c = pg_comp_crc32c_avx512; |
| 102 |
#endif |
| 103 |
} |
| 104 |
} |
| 105 |
|
| 106 |
return pg_comp_crc32c(crc, data, len); |
| 107 |
} |
| 108 |
|
| 109 |
pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len) = pg_comp_crc32c_choose; |