| 1 |
#ifndef AVX10MINMAX_HELPERFUNC_INCLUDED |
| 2 |
#define AVX10MINMAX_HELPERFUNC_INCLUDED |
| 3 |
|
| 4 |
#include <math.h> |
| 5 |
#include <limits.h> |
| 6 |
#include <stdbool.h> |
| 7 |
#include <float.h> |
| 8 |
#include "avx512f-helper.h" |
| 9 |
#define SNAN_float __builtin_nansf ("") |
| 10 |
#define SNAN_flag_float 0x7fa00000 |
| 11 |
#define QNAN_float __builtin_nanf ("") |
| 12 |
#define QNAN_flag_float 0x7fc00000 |
| 13 |
#define SNAN_double ((double)__builtin_nans ("")) |
| 14 |
#define SNAN_flag_double 0x7ff4000000000000 |
| 15 |
#define QNAN_double ((double)__builtin_nan ("")) |
| 16 |
#define QNAN_flag_double 0x7ff8000000000000 |
| 17 |
#define SNAN__Float16 ((_Float16)__builtin_nansf16 ("")) |
| 18 |
#define SNAN_flag__Float16 0x7d00 |
| 19 |
#define QNAN__Float16 ((_Float16)__builtin_nanf16 ("")) |
| 20 |
#define QNAN_flag__Float16 0x7e00 |
| 21 |
#define SNAN___bf16 ((__bf16)__builtin_nansf16b ("")) |
| 22 |
#define SNAN_flag___bf16 0x7fa0 |
| 23 |
#define QNAN___bf16 ((__bf16)__builtin_nanf ("")) |
| 24 |
#define QNAN_flag___bf16 0x7fc0 |
| 25 |
#define ISNAN(x) (x != x) |
| 26 |
#define ABS_float(x) fabsf (x) |
| 27 |
#define ABS_double(x) fabs (x) |
| 28 |
#define ABS__Float16(x) __builtin_fabsf16 (x) |
| 29 |
#define ABS___bf16(x) __builtin_fabsf (x) |
| 30 |
|
| 31 |
#define Union_Data(typef, typei) \ |
| 32 |
typedef union \ |
| 33 |
{ \ |
| 34 |
typef f; \ |
| 35 |
typei i; \ |
| 36 |
} union_##typef; |
| 37 |
|
| 38 |
Union_Data(float, int) |
| 39 |
Union_Data(double, long long) |
| 40 |
Union_Data(__bf16, short) |
| 41 |
Union_Data(_Float16, short) |
| 42 |
|
| 43 |
#define IS_SNAN(union_x, type) ((union_x.i & SNAN_flag_##type) == union_snan.i) |
| 44 |
|
| 45 |
#define IS_QNAN(union_x, type) ((union_x.i & QNAN_flag_##type) == union_qnan.i) |
| 46 |
|
| 47 |
#define CHECK_EXP_MINMAX(UNION_TYPE, VALUE_TYPE, INT_TYPE) \ |
| 48 |
static int \ |
| 49 |
__attribute__((noinline, unused)) \ |
| 50 |
check_minmax_##UNION_TYPE (UNION_TYPE u, const VALUE_TYPE *v) \ |
| 51 |
{ \ |
| 52 |
int i; \ |
| 53 |
int err = 0; \ |
| 54 |
for (i = 0; i < ARRAY_SIZE (u.a); i++) \ |
| 55 |
{ \ |
| 56 |
union_##VALUE_TYPE union_x, union_y; \ |
| 57 |
union_x.f = u.a[i]; \ |
| 58 |
union_y.f = v[i]; \ |
| 59 |
if (union_x.i != union_y.i) \ |
| 60 |
{ \ |
| 61 |
err++; \ |
| 62 |
PRINTF ("%i: " "%f" " != " "%f" "\n", \ |
| 63 |
i, v[i], u.a[i]); \ |
| 64 |
} \ |
| 65 |
} \ |
| 66 |
return err; \ |
| 67 |
} |
| 68 |
|
| 69 |
#if defined (AVX10_512BIT) |
| 70 |
CHECK_EXP_MINMAX (union512, float, int) |
| 71 |
CHECK_EXP_MINMAX (union512d, double, long int) |
| 72 |
CHECK_EXP_MINMAX (union512bf16_bf, __bf16, short int) |
| 73 |
CHECK_EXP_MINMAX (union512h, _Float16, short int) |
| 74 |
#endif |
| 75 |
CHECK_EXP_MINMAX (union256, float, int) |
| 76 |
CHECK_EXP_MINMAX (union256d, double, long int) |
| 77 |
CHECK_EXP_MINMAX (union128, float, int) |
| 78 |
CHECK_EXP_MINMAX (union128d, double, long int) |
| 79 |
CHECK_EXP_MINMAX (union256bf16_bf, __bf16, short int) |
| 80 |
CHECK_EXP_MINMAX (union128bf16_bf, __bf16, short int) |
| 81 |
CHECK_EXP_MINMAX (union256h, _Float16, short int) |
| 82 |
CHECK_EXP_MINMAX (union128h, _Float16, short int) |
| 83 |
|
| 84 |
#define UNION_CHECK_MINMAX(SIZE, NAME) EVAL(check_minmax_union, SIZE, NAME) |
| 85 |
|
| 86 |
#define CMP(res, x, y, type, value, op1, np, op2, zero, num, mag) \ |
| 87 |
{ \ |
| 88 |
union_##type union_a, union_b; \ |
| 89 |
union_a.f = x; \ |
| 90 |
union_b.f = y; \ |
| 91 |
union_##type union_snan, union_qnan; \ |
| 92 |
union_snan.f = SNAN_##type; \ |
| 93 |
union_qnan.f = QNAN_##type; \ |
| 94 |
bool flag = false; \ |
| 95 |
if(num) \ |
| 96 |
{ \ |
| 97 |
if(ISNAN(x) && ISNAN(y)) \ |
| 98 |
{ \ |
| 99 |
if(IS_SNAN(union_a,type) || (IS_QNAN(union_a,type) && IS_QNAN(union_b,type))) \ |
| 100 |
{ \ |
| 101 |
union_a.i |= value; \ |
| 102 |
res = union_a.f; \ |
| 103 |
flag = true; \ |
| 104 |
} \ |
| 105 |
else \ |
| 106 |
{ \ |
| 107 |
union_b.i |= value; \ |
| 108 |
res = union_b.f; \ |
| 109 |
flag = true; \ |
| 110 |
} \ |
| 111 |
} \ |
| 112 |
else if(ISNAN(x)) \ |
| 113 |
{ \ |
| 114 |
res = y; \ |
| 115 |
flag = true; \ |
| 116 |
} \ |
| 117 |
else if(ISNAN(y)) \ |
| 118 |
{ \ |
| 119 |
res = x; \ |
| 120 |
flag = true; \ |
| 121 |
} \ |
| 122 |
} \ |
| 123 |
else \ |
| 124 |
{ \ |
| 125 |
if(IS_SNAN(union_a,type) || (IS_QNAN(union_a,type) && !IS_SNAN(union_b,type))) \ |
| 126 |
{ \ |
| 127 |
union_a.i |= value; \ |
| 128 |
res = union_a.f; \ |
| 129 |
flag = true; \ |
| 130 |
} \ |
| 131 |
else if(ISNAN(y)) \ |
| 132 |
{ \ |
| 133 |
union_b.i |= value; \ |
| 134 |
res = union_b.f; \ |
| 135 |
flag = true; \ |
| 136 |
} \ |
| 137 |
} \ |
| 138 |
if(!flag) \ |
| 139 |
{ \ |
| 140 |
if(!mag) \ |
| 141 |
{ \ |
| 142 |
if((x == zero && y == - zero) || (x == - zero && y == zero)) \ |
| 143 |
res = np zero; \ |
| 144 |
else if(x op1 y) \ |
| 145 |
res = x; \ |
| 146 |
else \ |
| 147 |
res = y; \ |
| 148 |
} \ |
| 149 |
else \ |
| 150 |
{ \ |
| 151 |
if(ABS_##type(x) op2 ABS_##type(y)) \ |
| 152 |
res = x; \ |
| 153 |
else if(ABS_##type(y) op2 ABS_##type(x)) \ |
| 154 |
res = y; \ |
| 155 |
else \ |
| 156 |
{ \ |
| 157 |
if((x == zero && y == - zero) || (x == - zero && y == zero)) \ |
| 158 |
res = np zero; \ |
| 159 |
else if(x op1 y) \ |
| 160 |
res = x; \ |
| 161 |
else \ |
| 162 |
res = y; \ |
| 163 |
} \ |
| 164 |
} \ |
| 165 |
} \ |
| 166 |
} |
| 167 |
|
| 168 |
#define MINMAX(type, value, zero) \ |
| 169 |
type \ |
| 170 |
minmax_##type (type * a, type * b, int imm) \ |
| 171 |
{ \ |
| 172 |
int op_select = imm & 0x03; \ |
| 173 |
int sign_control = (imm & 0x0C) >> 2; \ |
| 174 |
int nan_prop_select = (imm & 0x10) >> 4; \ |
| 175 |
type tmp; \ |
| 176 |
if(nan_prop_select == 0) \ |
| 177 |
if(op_select == 0) \ |
| 178 |
CMP(tmp, *a, *b, type, value, <=, -, <, zero, false, false) \ |
| 179 |
else if(op_select == 1) \ |
| 180 |
CMP(tmp, *a, *b, type, value, >=, +, >, zero, false, false) \ |
| 181 |
else if(op_select == 2) \ |
| 182 |
CMP(tmp, *a, *b, type, value, <=, -, <, zero, false, true) \ |
| 183 |
else \ |
| 184 |
CMP(tmp, *a, *b, type, value, >=, +, >, zero, false, true) \ |
| 185 |
else \ |
| 186 |
if(op_select == 0) \ |
| 187 |
CMP(tmp, *a, *b, type, value, <=, -, <, zero, true, false) \ |
| 188 |
else if(op_select == 1) \ |
| 189 |
CMP(tmp, *a, *b, type, value, >=, +, >, zero, true, false) \ |
| 190 |
else if(op_select == 2) \ |
| 191 |
CMP(tmp, *a, *b, type, value, <=, -, <, zero, true, true) \ |
| 192 |
else \ |
| 193 |
CMP(tmp, *a, *b, type, value, >=, +, >, zero, true, true) \ |
| 194 |
if(!ISNAN(tmp)) \ |
| 195 |
if(sign_control == 0 && !ISNAN(*a)) \ |
| 196 |
if((tmp < 0 && *a > 0) || (tmp > 0 && *a < 0)) \ |
| 197 |
tmp = -tmp; \ |
| 198 |
else if(sign_control == 2) \ |
| 199 |
if(tmp < 0) tmp = -tmp; \ |
| 200 |
else if(sign_control == 3) \ |
| 201 |
if(tmp > 0) tmp = -tmp; \ |
| 202 |
return tmp; \ |
| 203 |
} |
| 204 |
|
| 205 |
|
| 206 |
MINMAX(double, 0x7ff8000000000000, 0.0) |
| 207 |
MINMAX(float, 0x7fc00000, 0.0f) |
| 208 |
MINMAX(_Float16, 0x7e00, 0.0f16) |
| 209 |
MINMAX(__bf16, 0x7fc0, 0.0bf16) |
| 210 |
|
| 211 |
#define UNIT_TEST(R, InsnSuffix, MaskType, type) \ |
| 212 |
sign = -1; \ |
| 213 |
for (i = 0; i < SIZE; i++) \ |
| 214 |
{ \ |
| 215 |
src1.a[i] = i % 2 ? SNAN_##type : 1.5 + 34.67 * i * sign; \ |
| 216 |
src2.a[i] = i % 3 ? QNAN_##type : -22.17 * i * sign; \ |
| 217 |
sign = sign * -1; \ |
| 218 |
} \ |
| 219 |
for (i = 0; i < SIZE; i++) \ |
| 220 |
res2.a[i] = DEFAULT_VALUE; \ |
| 221 |
res1.x = INTRINSIC(_minmax_##InsnSuffix) (src1.x, src2.x, R); \ |
| 222 |
res2.x = INTRINSIC(_mask_minmax_##InsnSuffix) (res2.x, mask, src1.x, src2.x, R); \ |
| 223 |
res3.x = INTRINSIC(_maskz_minmax_##InsnSuffix) (mask, src1.x, src2.x, R); \ |
| 224 |
CALC (res_ref, src1.a, src2.a, R); \ |
| 225 |
if (UNION_CHECK_MINMAX (AVX512F_LEN, MaskType) (res1, res_ref)) \ |
| 226 |
abort(); \ |
| 227 |
MASK_MERGE (MaskType) (res_ref, mask, SIZE); \ |
| 228 |
if (UNION_CHECK_MINMAX (AVX512F_LEN, MaskType) (res2, res_ref)) \ |
| 229 |
abort(); \ |
| 230 |
MASK_ZERO (MaskType) (res_ref, mask, SIZE); \ |
| 231 |
if (UNION_CHECK_MINMAX (AVX512F_LEN, MaskType) (res3, res_ref)) \ |
| 232 |
abort(); |
| 233 |
|
| 234 |
#define SCALAR_UNIT_TEST(R, InsnSuffix, MaskType, type) \ |
| 235 |
sign = -1; \ |
| 236 |
for (i = 0; i < SIZE; i++) \ |
| 237 |
{ \ |
| 238 |
src1.a[i] = i % 2 ? SNAN_##type : 1.5 + 34.67 * i * sign; \ |
| 239 |
src2.a[i] = i % 3 ? QNAN_##type : -22.17 * i * sign; \ |
| 240 |
sign = sign * -1; \ |
| 241 |
} \ |
| 242 |
for (i = 0; i < SIZE; i++) \ |
| 243 |
res2.a[i] = DEFAULT_VALUE; \ |
| 244 |
res1.x = _mm_minmax_##InsnSuffix (src1.x, src2.x, R); \ |
| 245 |
res2.x = _mm_mask_minmax_##InsnSuffix (res2.x, mask, src1.x, src2.x, R); \ |
| 246 |
res3.x = _mm_maskz_minmax_##InsnSuffix (mask, src1.x, src2.x, R); \ |
| 247 |
CALC (res_ref, src1.a, src2.a, R); \ |
| 248 |
if (UNION_CHECK_MINMAX (128, MaskType) (res1, res_ref)) \ |
| 249 |
abort(); \ |
| 250 |
MASK_MERGE (MaskType) (res_ref, mask, 1); \ |
| 251 |
if (UNION_CHECK_MINMAX (128, MaskType) (res2, res_ref)) \ |
| 252 |
abort(); \ |
| 253 |
MASK_ZERO (MaskType) (res_ref, mask, 1); \ |
| 254 |
if (UNION_CHECK_MINMAX (128, MaskType) (res3, res_ref)) \ |
| 255 |
abort(); |
| 256 |
|
| 257 |
#endif |