// Copyright 2024 Google LLC // // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. $assert CHANNEL_TILE % 32 == 0 $assert CHANNEL_TILE >= 32 $SIMD_TILE = CHANNEL_TILE // 32 $assert ACCUMULATORS <= SIMD_TILE $assert AVX in [2, 10] #include #include #include "xnnpack/common.h" #include "xnnpack/reduce.h" $ACC_SUFFIX = "" if ACCUMULATORS == 1 else "_acc%d" % ACCUMULATORS $ISA = "avx2" if AVX == 2 else "avx256skx" void xnn_qs8_rsum_ukernel__${ISA}_u${CHANNEL_TILE}${ACC_SUFFIX}( size_t batch, const int8_t* input, int32_t* output, const struct xnn_qs8_rsum_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS { assert(batch != 0); assert(input != NULL); assert(output != NULL); assert(params != NULL); XNN_ALIGN(32) static const int8_t onemask_table[64] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; const __m256i vone = _mm256_load_si256((const __m256i*) &onemask_table[0]); const __m256i vone_16 = _mm256_srli_epi16(vone, 8); $for ACC in range(ACCUMULATORS): __m256i vacc${ACC} = _mm256_setzero_si256(); // 256 int8s may be summed into an int16 before overflowing. // Each register has 32 lanes and there are ${ACCUMULATORS} accumulators so batch size is ${ACCUMULATORS*256*32} for (; batch >= ${ACCUMULATORS*256*32}; batch -= ${ACCUMULATORS*256*32}) { $for ACC in range(ACCUMULATORS): __m256i vacc16_${ACC} = _mm256_setzero_si256(); for (size_t current_batch = ${ACCUMULATORS*256*32}; current_batch > 0; current_batch -= ${CHANNEL_TILE}) { $for N in range(SIMD_TILE): const __m256i vt${N} = _mm256_maddubs_epi16(vone, _mm256_loadu_si256((const __m256i*) input)); input += 32; $for N in range(SIMD_TILE): vacc16_${N % ACCUMULATORS} = _mm256_add_epi16(vacc16_${N % ACCUMULATORS}, vt${N}); } $for ACC in range(ACCUMULATORS): vacc${ACC} = _mm256_add_epi32(vacc${ACC}, _mm256_madd_epi16(vone_16, vacc16_${ACC})); } $if CHANNEL_TILE > 32: if (XNN_LIKELY(batch >= ${CHANNEL_TILE})) { assert(batch >= 1 && batch < ${ACCUMULATORS*256*32}); $for ACC in range(ACCUMULATORS): __m256i vacc16_${ACC} = _mm256_setzero_si256(); for (; batch >= ${CHANNEL_TILE}; batch -= ${CHANNEL_TILE}) { $for N in range(SIMD_TILE): const __m256i vt${N} = _mm256_maddubs_epi16(vone, _mm256_loadu_si256((const __m256i*) input)); input += 32; $for N in range(SIMD_TILE): vacc16_${N % ACCUMULATORS} = _mm256_add_epi16(vacc16_${N % ACCUMULATORS}, vt${N}); } $for ACC in range(ACCUMULATORS): vacc${ACC} = _mm256_add_epi32(vacc${ACC}, _mm256_madd_epi16(vone_16, vacc16_${ACC})); } if (XNN_UNLIKELY(batch != 0)) { assert(batch >= 1 && batch < ${256*32}); __m256i vacc16 = _mm256_setzero_si256(); for (; batch >= 32; batch -= 32) { const __m256i vt = _mm256_maddubs_epi16(vone, _mm256_loadu_si256((const __m256i*) input)); input += 32; vacc16 = _mm256_add_epi16(vacc16, vt); } $if AVX == 2: // Remainder is between 17 and 31 bytes, so process 32 bytes (overread of up to 15) if (XNN_UNLIKELY(batch >= 17)) { assert(batch >= 17 && batch <= 31); const __m256i vonemask = _mm256_loadu_si256((const __m256i*) &onemask_table[32 - batch]); const __m256i vt = _mm256_maddubs_epi16(vonemask, _mm256_loadu_si256((const __m256i*) input)); vacc16 = _mm256_add_epi16(vacc16, vt); // Remainder is between 1 and 16 bytes, so process 16 bytes (overread of up to 15) } else if (XNN_UNLIKELY(batch != 0)) { assert(batch >= 1 && batch <= 16); const __m256i vonemask = _mm256_loadu_si256((const __m256i*) &onemask_table[32 - batch]); const __m256i vt = _mm256_maddubs_epi16(vonemask, _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*) input))); vacc16 = _mm256_add_epi16(vacc16, vt); } $else: if (XNN_UNLIKELY(batch != 0)) { assert(batch >= 1 && batch <= 31); // Prepare mask for valid 8-bit elements (depends on batch). const __mmask32 vmask = _cvtu32_mask32((UINT32_C(1) << batch) - 1); const __m256i vt = _mm256_maddubs_epi16(vone, _mm256_maskz_loadu_epi8(vmask, input)); vacc16 = _mm256_add_epi16(vacc16, vt); } vacc0 = _mm256_add_epi32(vacc0, _mm256_madd_epi16(vone_16, vacc16)); } $if ACCUMULATORS > 1: $ACC_SLICE = 1 $while ACC_SLICE < ACCUMULATORS: $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): $if A + ACC_SLICE < ACCUMULATORS: vacc${A} = _mm256_add_epi32(vacc${A}, vacc${A + ACC_SLICE}); $ACC_SLICE *= 2 __m128i vacc_lo = _mm_add_epi32(_mm256_castsi256_si128(vacc0), _mm256_extractf128_si256(vacc0, 1)); vacc_lo = _mm_hadd_epi32(vacc_lo, vacc_lo); vacc_lo = _mm_hadd_epi32(vacc_lo, vacc_lo); *output += _mm_cvtsi128_si32(vacc_lo); }