| 1 |
/* { dg-do run } */ |
| 2 |
/* { dg-require-effective-target avx512bw } */ |
| 3 |
/* { dg-require-effective-target avx512vl } */ |
| 4 |
/* { dg-options "-O2 -std=c++17 -mavx512bw -mavx512vl" } */ |
| 5 |
/* { dg-skip-if "requires hosted libstdc++ for cstdlib malloc" { ! hostedlib } } */ |
| 6 |
|
| 7 |
#include <cstdint> |
| 8 |
#include <x86intrin.h> |
| 9 |
#include <functional> |
| 10 |
#include <ostream> |
| 11 |
|
| 12 |
#define AVX512BW |
| 13 |
#define AVX512VL |
| 14 |
|
| 15 |
#include "avx512f-helper.h" |
| 16 |
|
| 17 |
struct TensorIteratorBase{ |
| 18 |
char* in; |
| 19 |
char* out; |
| 20 |
|
| 21 |
void for_each(std::function<void(char*, char*, int64_t size)> loop){ |
| 22 |
loop(out, in, 32); |
| 23 |
} |
| 24 |
}; |
| 25 |
|
| 26 |
class Vectorized { |
| 27 |
protected: |
| 28 |
__m256i values; |
| 29 |
|
| 30 |
static inline __m256i invert(const __m256i& v) { |
| 31 |
const auto ones = _mm256_set1_epi64x(-1); |
| 32 |
return _mm256_xor_si256(ones, v); |
| 33 |
} |
| 34 |
public: |
| 35 |
operator __m256i() const { |
| 36 |
return values; |
| 37 |
} |
| 38 |
|
| 39 |
static constexpr int size() { |
| 40 |
return 32; |
| 41 |
} |
| 42 |
|
| 43 |
Vectorized() {} |
| 44 |
Vectorized(__m256i v) : values(v) {} |
| 45 |
Vectorized(uint8_t v) { values = _mm256_set1_epi8(v); } |
| 46 |
static Vectorized blendv(const Vectorized& a, const Vectorized& b, |
| 47 |
const Vectorized& mask) { |
| 48 |
return _mm256_blendv_epi8(a, b, mask); |
| 49 |
} |
| 50 |
static Vectorized loadu(const void* ptr) { |
| 51 |
return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(ptr)); |
| 52 |
} |
| 53 |
void store(void* ptr) const { |
| 54 |
_mm256_storeu_si256(reinterpret_cast<__m256i*>(ptr), values); |
| 55 |
} |
| 56 |
|
| 57 |
Vectorized operator<(const Vectorized& other) const { |
| 58 |
__m256i max = _mm256_max_epu8(values, other); |
| 59 |
return invert(_mm256_cmpeq_epi8(max, values)); |
| 60 |
} |
| 61 |
Vectorized operator-(const Vectorized& b) { |
| 62 |
return _mm256_sub_epi8(values, b); |
| 63 |
} |
| 64 |
}; |
| 65 |
|
| 66 |
std::ostream& operator<<(std::ostream& stream, const Vectorized& vec) { |
| 67 |
uint8_t buf[Vectorized::size()]; |
| 68 |
vec.store(buf); |
| 69 |
stream << "vec["; |
| 70 |
for (int i = 0; i != Vectorized::size(); i++) { |
| 71 |
if (i != 0) |
| 72 |
stream << ", "; |
| 73 |
stream << buf[i]*1; |
| 74 |
} |
| 75 |
stream << "]"; |
| 76 |
return stream; |
| 77 |
} |
| 78 |
|
| 79 |
void run(TensorIteratorBase iter){ |
| 80 |
Vectorized zero_vec(0); |
| 81 |
Vectorized one_vec(1); |
| 82 |
|
| 83 |
iter.for_each([=](char* out, char* in, int64_t size) { |
| 84 |
for (int64_t i = 0; i <= size - Vectorized::size(); i += Vectorized::size()) { |
| 85 |
auto self_vec = Vectorized::loadu(in + i); |
| 86 |
auto left = Vectorized::blendv(zero_vec, one_vec, zero_vec < self_vec); |
| 87 |
auto right = Vectorized::blendv(zero_vec, one_vec, self_vec < zero_vec); |
| 88 |
auto outv = left - right; |
| 89 |
outv.store(out + i); |
| 90 |
} |
| 91 |
}); |
| 92 |
} |
| 93 |
|
| 94 |
void |
| 95 |
test_256 (){ |
| 96 |
char in[32]; |
| 97 |
char out[32]; |
| 98 |
for(auto& x: in) x = 1; |
| 99 |
run(TensorIteratorBase{in, out}); |
| 100 |
Vectorized::loadu (out); |
| 101 |
for (int i = 0; i != 32; i++) |
| 102 |
if (out[i] != 1) |
| 103 |
__builtin_abort (); |
| 104 |
} |
| 105 |
|
| 106 |
void |
| 107 |
test_128 () |
| 108 |
{ |
| 109 |
} |