| 1 |
/* |
| 2 |
* Copyright (C) 2020 Linux Studio Plugins Project <https://lsp-plug.in/> |
| 3 |
* (C) 2020 Vladimir Sadovnikov <[email protected]> |
| 4 |
* |
| 5 |
* This file is part of lsp-dsp-lib |
| 6 |
* Created on: 31 мар. 2020 г. |
| 7 |
* |
| 8 |
* lsp-dsp-lib is free software: you can redistribute it and/or modify |
| 9 |
* it under the terms of the GNU Lesser General Public License as published by |
| 10 |
* the Free Software Foundation, either version 3 of the License, or |
| 11 |
* any later version. |
| 12 |
* |
| 13 |
* lsp-dsp-lib is distributed in the hope that it will be useful, |
| 14 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 |
* GNU Lesser General Public License for more details. |
| 17 |
* |
| 18 |
* You should have received a copy of the GNU Lesser General Public License |
| 19 |
* along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>. |
| 20 |
*/ |
| 21 |
|
| 22 |
#ifndef PRIVATE_DSP_ARCH_ARM_NEON_D32_CONVOLUTION_H_ |
| 23 |
#define PRIVATE_DSP_ARCH_ARM_NEON_D32_CONVOLUTION_H_ |
| 24 |
|
| 25 |
#ifndef PRIVATE_DSP_ARCH_ARM_NEON_D32_IMPL |
| 26 |
#error "This header should not be included directly" |
| 27 |
#endif /* PRIVATE_DSP_ARCH_ARM_NEON_D32_IMPL */ |
| 28 |
|
| 29 |
namespace lsp |
| 30 |
{ |
| 31 |
namespace neon_d32 |
| 32 |
{ |
| 33 |
void convolve(float *dst, const float *src, const float *conv, size_t length, size_t count) |
| 34 |
{ |
| 35 |
IF_ARCH_ARM( |
| 36 |
const float *c; // , *cc; |
| 37 |
float *d; //, *dd; |
| 38 |
size_t clen; |
| 39 |
) |
| 40 |
|
| 41 |
ARCH_ARM_ASM( |
| 42 |
__ASM_EMIT("subs %[count], #4") |
| 43 |
__ASM_EMIT("blo 20f") |
| 44 |
|
| 45 |
__ASM_EMIT("10:") |
| 46 |
__ASM_EMIT("veor q15, q15, q15") // q15 = p0 p1 p2 p3 = 0 0 0 0 (history) |
| 47 |
__ASM_EMIT("mov %[d], %[dst]") |
| 48 |
__ASM_EMIT("mov %[c], %[conv]") |
| 49 |
__ASM_EMIT("subs %[clen], %[length], #4") |
| 50 |
__ASM_EMIT("blo 12f") |
| 51 |
|
| 52 |
// Load convolution coefficients |
| 53 |
__ASM_EMIT("vld2.32 {d0[], d2[]}, [%[k]]") // q0 = k0 k0 ? ?, q1 = k1 k1 ? ? |
| 54 |
__ASM_EMIT("vld2.32 {d1[], d3[]}, [%[k]]!") // q0 = k0 k0 k0 k0, q1 = k1 k1 k1 k1, k += 8 |
| 55 |
__ASM_EMIT("vld2.32 {d4[], d6[]}, [%[k]]") // q2 = k2 k2 ? ?, q3 = k3 k3 ? ? |
| 56 |
__ASM_EMIT("vld2.32 {d5[], d7[]}, [%[k]]") // q2 = k2 k2 k2 k2, q3 = k3 k3 k3 k3 |
| 57 |
__ASM_EMIT("sub %[k], #8") // k -= 8 |
| 58 |
|
| 59 |
// Load convolution kernel |
| 60 |
__ASM_EMIT("11:") |
| 61 |
__ASM_EMIT("vld1.32 {q4}, [%[c]]!") // q4 = c0 c1 c2 c3, c += 4 |
| 62 |
__ASM_EMIT("vld1.32 {q8}, [%[d]]") // q8 = d0 d1 d2 d3 |
| 63 |
__ASM_EMIT("vext.8 q5, q15, q4, #12") // q5 = p3 c0 c1 c2 |
| 64 |
__ASM_EMIT("vext.8 q6, q15, q4, #8") // q6 = p2 p3 c0 c1 |
| 65 |
__ASM_EMIT("vext.8 q7, q15, q4, #4") // q7 = p1 p2 p3 c0 |
| 66 |
__ASM_EMIT("vmla.f32 q8, q5, q1") // q8 = d0 + k0*c0 + k1*p3 ... |
| 67 |
__ASM_EMIT("vmla.f32 q8, q4, q0") // q8 = d0+k0*c0 d1+k0*c1 d2+k0*c2 d3+k0*c3 |
| 68 |
__ASM_EMIT("vmla.f32 q8, q6, q2") // q8 = d0 + k0*c0 + k1*p3 + k2*p2 ... |
| 69 |
__ASM_EMIT("vmla.f32 q8, q7, q3") // q8 = d0 + k0*c0 + k1*p3 + k2*p2 + k3*p1 ... |
| 70 |
__ASM_EMIT("vmov q15, q4") // q15 = c0 c1 c2 c3 |
| 71 |
__ASM_EMIT("vst1.32 {q8}, [%[d]]!") // d += 4 |
| 72 |
__ASM_EMIT("subs %[clen], #4") // clen -= 4 |
| 73 |
__ASM_EMIT("bhs 11b") |
| 74 |
|
| 75 |
// Apply tail: q15 = p0 p1 p2 p3 |
| 76 |
__ASM_EMIT("vld1.32 {q1}, [%[k]]") // s4 = k0, s5 = k1, s6 = k2, s7 = k3 |
| 77 |
__ASM_EMIT("vmov q7, q15") // s28 = p0, s29 = p1, s30 = p2, s31 = p3 |
| 78 |
__ASM_EMIT("vldm.32 %[d], {s0-s2}") // s0 = d0, s1 = d1, s2 = d2 |
| 79 |
__ASM_EMIT("vmla.f32 s0, s7, s29") // s0 = d0 + k3*p1 |
| 80 |
__ASM_EMIT("vmla.f32 s1, s7, s30") // s1 = d1 + k3*p2 |
| 81 |
__ASM_EMIT("vmla.f32 s2, s7, s31") // s2 = d2 + k3*p3 |
| 82 |
__ASM_EMIT("vmla.f32 s0, s6, s30") // s0 = d0 + k2*p2 + k3*p1 |
| 83 |
__ASM_EMIT("vmla.f32 s1, s6, s31") // s1 = d1 + k2*p3 + k3*p2 |
| 84 |
__ASM_EMIT("vmla.f32 s0, s5, s31") // s0 = d0 + k1*p3 + k2*p2 + k3*p1 |
| 85 |
__ASM_EMIT("vstm.f32 %[d], {s0-s2}") |
| 86 |
|
| 87 |
// Apply tail |
| 88 |
__ASM_EMIT("12:") |
| 89 |
__ASM_EMIT("adds %[clen], #3") // while (clen >= 0) |
| 90 |
__ASM_EMIT("blt 14f") |
| 91 |
__ASM_EMIT("vld1.32 {q0}, [%[k]]") // q0 = k0 k1 k2 k3 k += 4 |
| 92 |
__ASM_EMIT("15:") |
| 93 |
__ASM_EMIT("vld1.32 {d2[], d3[]}, [%[c]]!") // q1 = c0 c0 c0 c0, c++ |
| 94 |
__ASM_EMIT("vld1.32 {q2}, [%[d]]") // q2 = d0 d1 d2 d3 |
| 95 |
__ASM_EMIT("vmla.f32 q2, q0, q1") // q2 = d0+k0*c0 d1+k1*c0 d2+k2*c0 d3+k3*c0 |
| 96 |
__ASM_EMIT("vst1.32 {q2}, [%[d]]") |
| 97 |
__ASM_EMIT("subs %[clen], #1") |
| 98 |
__ASM_EMIT("add %[d], #4") // d++ |
| 99 |
__ASM_EMIT("bge 15b") |
| 100 |
|
| 101 |
__ASM_EMIT("14:") |
| 102 |
__ASM_EMIT("add %[dst], #0x10") // dst += 4 |
| 103 |
__ASM_EMIT("add %[k], #0x10") // k += 4 |
| 104 |
__ASM_EMIT("subs %[count], #4") // count -= 4 |
| 105 |
__ASM_EMIT("bge 10b") |
| 106 |
|
| 107 |
__ASM_EMIT("20:") |
| 108 |
__ASM_EMIT("adds %[count], #3") |
| 109 |
__ASM_EMIT("blt 40f") |
| 110 |
__ASM_EMIT("21:") |
| 111 |
__ASM_EMIT("mov %[d], %[dst]") |
| 112 |
__ASM_EMIT("mov %[c], %[conv]") |
| 113 |
__ASM_EMIT("vld1.32 {d0[], d1[]}, [%[k]]") // q0 = k0 k0 k0 k0 |
| 114 |
__ASM_EMIT("subs %[clen], %[length], #8") |
| 115 |
__ASM_EMIT("vmov q1, q0") // q1 = k0 k0 k0 k0 |
| 116 |
__ASM_EMIT("blo 22f") |
| 117 |
__ASM_EMIT("23:") |
| 118 |
__ASM_EMIT("vld1.32 {q2-q3}, [%[c]]!") // q2 = c0 c1 c2 c3, c+= 8 |
| 119 |
__ASM_EMIT("vld1.32 {q8-q9}, [%[d]]") // q8 = d0 d1 d2 d3 |
| 120 |
__ASM_EMIT("vmla.f32 q8, q2, q0") // q8 = d0+k0*c0 d1+k0*c1 d2+k0*c2 d3+k0*c3 |
| 121 |
__ASM_EMIT("vmla.f32 q9, q3, q1") |
| 122 |
__ASM_EMIT("subs %[clen], #8") // clen -= 8 |
| 123 |
__ASM_EMIT("vst1.32 {q8-q9}, [%[d]]!") // d += 8 |
| 124 |
__ASM_EMIT("bge 23b") |
| 125 |
|
| 126 |
// Apply convolution |
| 127 |
__ASM_EMIT("22:") |
| 128 |
__ASM_EMIT("adds %[clen], #4") |
| 129 |
__ASM_EMIT("blt 24f") |
| 130 |
__ASM_EMIT("vld1.32 {q2}, [%[c]]!") // q2 = c0 c1 c2 c3, c+= 8 |
| 131 |
__ASM_EMIT("vld1.32 {q8}, [%[d]]") // q8 = d0 d1 d2 d3 |
| 132 |
__ASM_EMIT("vmla.f32 q8, q2, q0") // q8 = d0+k0*c0 d1+k0*c1 d2+k0*c2 d3+k0*c3 |
| 133 |
__ASM_EMIT("subs %[clen], #4") // clen -= 4 |
| 134 |
__ASM_EMIT("vst1.32 {q8}, [%[d]]!") // d += 4 |
| 135 |
|
| 136 |
// Apply tail |
| 137 |
__ASM_EMIT("24:") |
| 138 |
__ASM_EMIT("adds %[clen], #3") |
| 139 |
__ASM_EMIT("blt 26f") |
| 140 |
__ASM_EMIT("25:") |
| 141 |
__ASM_EMIT("vldm.32 %[c]!, {s2}") // s2 = c0, c++ |
| 142 |
__ASM_EMIT("vldm.32 %[d], {s8}") // s8 = d0 |
| 143 |
__ASM_EMIT("vmla.f32 s8, s2, s0") // s8 = d0+k0*c0 |
| 144 |
__ASM_EMIT("subs %[clen], #1") // clen -- |
| 145 |
__ASM_EMIT("vstm.32 %[d]!, {s8}") // d ++ |
| 146 |
__ASM_EMIT("bge 25b") |
| 147 |
|
| 148 |
__ASM_EMIT("26:") |
| 149 |
__ASM_EMIT("add %[k], #0x04") |
| 150 |
__ASM_EMIT("add %[dst], #0x04") // dst++ |
| 151 |
__ASM_EMIT("subs %[count], #1") |
| 152 |
__ASM_EMIT("bge 21b") |
| 153 |
|
| 154 |
__ASM_EMIT("40:") |
| 155 |
: [dst] "+r" (dst), |
| 156 |
[k] "+r" (src), [count] "+r" (count), |
| 157 |
[c] "=&r" (c), [d] "=&r" (d), |
| 158 |
[clen] "=&r" (clen) |
| 159 |
: [conv] "r" (conv), [length] "r" (length) |
| 160 |
: "cc", "memory", |
| 161 |
"q0", "q1", "q2", "q3" , "q4", "q5", "q6", "q7", |
| 162 |
"q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" |
| 163 |
); |
| 164 |
} |
| 165 |
} |
| 166 |
} |
| 167 |
|
| 168 |
#endif /* PRIVATE_DSP_ARCH_ARM_NEON_D32_CONVOLUTION_H_ */ |