Debian Codegraph @

src/f16-f32acc-rsum/neonfp16arith.c.in

raw
1 // Copyright 2023 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 $assert BATCH_TILE % 8 == 0 or BATCH_TILE == 4
7 $assert BATCH_TILE >= 4
8 $SIMD_TILE = BATCH_TILE // 4
9 $assert ACCUMULATORS <= SIMD_TILE
10 $ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
11 #include <assert.h>
12
13 #include <arm_neon.h>
14
15 #include "xnnpack/common.h"
16 #include "xnnpack/reduce.h"
17
18
19 $ACC_SUFFIX = "" if ACCUMULATORS == 1 else "_acc%d" % ACCUMULATORS
20 void xnn_f16_f32acc_rsum_ukernel__neonfp16arith_u${BATCH_TILE}${ACC_SUFFIX}(
21 size_t batch,
22 const xnn_float16* input,
23 float* output,
24 const struct xnn_f16_f32acc_scale_params params[restrict XNN_MIN_ELEMENTS(1)])
25 {
26 assert(batch != 0);
27 assert(batch % sizeof(uint16_t) == 0);
28 assert(input != NULL);
29 assert(output != NULL);
30
31 const uint16_t* i = (const uint16_t*) input;
32 $for A in range(ACCUMULATORS):
33 float32x4_t vacc${A} = vmovq_n_f32(0.0f);
34 $if BATCH_TILE > 8:
35 for (; batch >= ${BATCH_TILE} * sizeof(uint16_t); batch -= ${BATCH_TILE} * sizeof(uint16_t)) {
36 $for N in range(0, SIMD_TILE, 2):
37 const float16x8_t vh${ABC[N:N+2]} = vreinterpretq_f16_u16(vld1q_u16(i)); i += 8;
38
39 $for N in range(0, SIMD_TILE, 2):
40 const float32x4_t vt${N} = vcvt_f32_f16(vget_low_f16(vh${ABC[N:N+2]}));
41 const float32x4_t vt${N+1} = vcvt_f32_f16(vget_high_f16(vh${ABC[N:N+2]}));
42
43 $for N in range(SIMD_TILE):
44 vacc${N % ACCUMULATORS} = vaddq_f32(vacc${N % ACCUMULATORS}, vt${N});
45 }
46 $if ACCUMULATORS > 1:
47 $ACC_SLICE = 1
48 $while ACC_SLICE < ACCUMULATORS:
49 $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
50 $if A + ACC_SLICE < ACCUMULATORS:
51 vacc${A} = vaddq_f32(vacc${A}, vacc${A + ACC_SLICE});
52 $ACC_SLICE *= 2
53 for (; batch >= 4 * sizeof(uint16_t); batch -= 4 * sizeof(uint16_t)) {
54 const float16x4_t vh = vreinterpret_f16_u16(vld1_u16(i)); i += 4;
55 const float32x4_t vt = vcvt_f32_f16(vh);
56 vacc0 = vaddq_f32(vacc0, vt);
57 }
58 const float32x2_t vscale = vld1_dup_f32(&params->scalar.scale);
59 float32x2_t vacc = vadd_f32(vget_low_f32(vacc0), vget_high_f32(vacc0));
60 if XNN_UNLIKELY(batch & (2 * sizeof(uint16_t))) {
61 const float16x4_t vh = vreinterpret_f16_u32(vld1_dup_u32((const void*) i)); i += 2;
62 const float32x4_t vt = vcvt_f32_f16(vh);
63 vacc = vadd_f32(vacc, vget_low_f32(vt));
64 }
65 vacc = vpadd_f32(vacc, vacc);
66 if XNN_UNLIKELY(batch & (1 * sizeof(uint16_t))) {
67 const float16x4_t vh = vreinterpret_f16_u16(vld1_dup_u16(i));
68 const float32x4_t vt = vcvt_f32_f16(vh);
69 vacc = vadd_f32(vacc, vget_low_f32(vt));
70 }
71 vacc = vmul_f32(vacc, vscale);
72
73 float vout = vget_lane_f32(vacc, 0);
74 *output += vout;
75 }