| 1 |
/* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 |
/* |
| 3 |
* arch/arm/include/asm/xor.h |
| 4 |
* |
| 5 |
* Copyright (C) 2001 Russell King |
| 6 |
*/ |
| 7 |
#include <linux/hardirq.h> |
| 8 |
#include <asm-generic/xor.h> |
| 9 |
#include <asm/hwcap.h> |
| 10 |
#include <asm/neon.h> |
| 11 |
|
| 12 |
#define __XOR(a1, a2) a1 ^= a2 |
| 13 |
|
| 14 |
#define GET_BLOCK_2(dst) \ |
| 15 |
__asm__("ldmia %0, {%1, %2}" \ |
| 16 |
: "=r" (dst), "=r" (a1), "=r" (a2) \ |
| 17 |
: "0" (dst)) |
| 18 |
|
| 19 |
#define GET_BLOCK_4(dst) \ |
| 20 |
__asm__("ldmia %0, {%1, %2, %3, %4}" \ |
| 21 |
: "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \ |
| 22 |
: "0" (dst)) |
| 23 |
|
| 24 |
#define XOR_BLOCK_2(src) \ |
| 25 |
__asm__("ldmia %0!, {%1, %2}" \ |
| 26 |
: "=r" (src), "=r" (b1), "=r" (b2) \ |
| 27 |
: "0" (src)); \ |
| 28 |
__XOR(a1, b1); __XOR(a2, b2); |
| 29 |
|
| 30 |
#define XOR_BLOCK_4(src) \ |
| 31 |
__asm__("ldmia %0!, {%1, %2, %3, %4}" \ |
| 32 |
: "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \ |
| 33 |
: "0" (src)); \ |
| 34 |
__XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4) |
| 35 |
|
| 36 |
#define PUT_BLOCK_2(dst) \ |
| 37 |
__asm__ __volatile__("stmia %0!, {%2, %3}" \ |
| 38 |
: "=r" (dst) \ |
| 39 |
: "0" (dst), "r" (a1), "r" (a2)) |
| 40 |
|
| 41 |
#define PUT_BLOCK_4(dst) \ |
| 42 |
__asm__ __volatile__("stmia %0!, {%2, %3, %4, %5}" \ |
| 43 |
: "=r" (dst) \ |
| 44 |
: "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4)) |
| 45 |
|
| 46 |
static void |
| 47 |
xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1, |
| 48 |
const unsigned long * __restrict p2) |
| 49 |
{ |
| 50 |
unsigned int lines = bytes / sizeof(unsigned long) / 4; |
| 51 |
register unsigned int a1 __asm__("r4"); |
| 52 |
register unsigned int a2 __asm__("r5"); |
| 53 |
register unsigned int a3 __asm__("r6"); |
| 54 |
register unsigned int a4 __asm__("r10"); |
| 55 |
register unsigned int b1 __asm__("r8"); |
| 56 |
register unsigned int b2 __asm__("r9"); |
| 57 |
register unsigned int b3 __asm__("ip"); |
| 58 |
register unsigned int b4 __asm__("lr"); |
| 59 |
|
| 60 |
do { |
| 61 |
GET_BLOCK_4(p1); |
| 62 |
XOR_BLOCK_4(p2); |
| 63 |
PUT_BLOCK_4(p1); |
| 64 |
} while (--lines); |
| 65 |
} |
| 66 |
|
| 67 |
static void |
| 68 |
xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1, |
| 69 |
const unsigned long * __restrict p2, |
| 70 |
const unsigned long * __restrict p3) |
| 71 |
{ |
| 72 |
unsigned int lines = bytes / sizeof(unsigned long) / 4; |
| 73 |
register unsigned int a1 __asm__("r4"); |
| 74 |
register unsigned int a2 __asm__("r5"); |
| 75 |
register unsigned int a3 __asm__("r6"); |
| 76 |
register unsigned int a4 __asm__("r10"); |
| 77 |
register unsigned int b1 __asm__("r8"); |
| 78 |
register unsigned int b2 __asm__("r9"); |
| 79 |
register unsigned int b3 __asm__("ip"); |
| 80 |
register unsigned int b4 __asm__("lr"); |
| 81 |
|
| 82 |
do { |
| 83 |
GET_BLOCK_4(p1); |
| 84 |
XOR_BLOCK_4(p2); |
| 85 |
XOR_BLOCK_4(p3); |
| 86 |
PUT_BLOCK_4(p1); |
| 87 |
} while (--lines); |
| 88 |
} |
| 89 |
|
| 90 |
static void |
| 91 |
xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1, |
| 92 |
const unsigned long * __restrict p2, |
| 93 |
const unsigned long * __restrict p3, |
| 94 |
const unsigned long * __restrict p4) |
| 95 |
{ |
| 96 |
unsigned int lines = bytes / sizeof(unsigned long) / 2; |
| 97 |
register unsigned int a1 __asm__("r8"); |
| 98 |
register unsigned int a2 __asm__("r9"); |
| 99 |
register unsigned int b1 __asm__("ip"); |
| 100 |
register unsigned int b2 __asm__("lr"); |
| 101 |
|
| 102 |
do { |
| 103 |
GET_BLOCK_2(p1); |
| 104 |
XOR_BLOCK_2(p2); |
| 105 |
XOR_BLOCK_2(p3); |
| 106 |
XOR_BLOCK_2(p4); |
| 107 |
PUT_BLOCK_2(p1); |
| 108 |
} while (--lines); |
| 109 |
} |
| 110 |
|
| 111 |
static void |
| 112 |
xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1, |
| 113 |
const unsigned long * __restrict p2, |
| 114 |
const unsigned long * __restrict p3, |
| 115 |
const unsigned long * __restrict p4, |
| 116 |
const unsigned long * __restrict p5) |
| 117 |
{ |
| 118 |
unsigned int lines = bytes / sizeof(unsigned long) / 2; |
| 119 |
register unsigned int a1 __asm__("r8"); |
| 120 |
register unsigned int a2 __asm__("r9"); |
| 121 |
register unsigned int b1 __asm__("ip"); |
| 122 |
register unsigned int b2 __asm__("lr"); |
| 123 |
|
| 124 |
do { |
| 125 |
GET_BLOCK_2(p1); |
| 126 |
XOR_BLOCK_2(p2); |
| 127 |
XOR_BLOCK_2(p3); |
| 128 |
XOR_BLOCK_2(p4); |
| 129 |
XOR_BLOCK_2(p5); |
| 130 |
PUT_BLOCK_2(p1); |
| 131 |
} while (--lines); |
| 132 |
} |
| 133 |
|
| 134 |
static struct xor_block_template xor_block_arm4regs = { |
| 135 |
.name = "arm4regs", |
| 136 |
.do_2 = xor_arm4regs_2, |
| 137 |
.do_3 = xor_arm4regs_3, |
| 138 |
.do_4 = xor_arm4regs_4, |
| 139 |
.do_5 = xor_arm4regs_5, |
| 140 |
}; |
| 141 |
|
| 142 |
#undef XOR_TRY_TEMPLATES |
| 143 |
#define XOR_TRY_TEMPLATES \ |
| 144 |
do { \ |
| 145 |
xor_speed(&xor_block_arm4regs); \ |
| 146 |
xor_speed(&xor_block_8regs); \ |
| 147 |
xor_speed(&xor_block_32regs); \ |
| 148 |
NEON_TEMPLATES; \ |
| 149 |
} while (0) |
| 150 |
|
| 151 |
#ifdef CONFIG_KERNEL_MODE_NEON |
| 152 |
|
| 153 |
extern struct xor_block_template const xor_block_neon_inner; |
| 154 |
|
| 155 |
static void |
| 156 |
xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, |
| 157 |
const unsigned long * __restrict p2) |
| 158 |
{ |
| 159 |
if (in_interrupt()) { |
| 160 |
xor_arm4regs_2(bytes, p1, p2); |
| 161 |
} else { |
| 162 |
kernel_neon_begin(); |
| 163 |
xor_block_neon_inner.do_2(bytes, p1, p2); |
| 164 |
kernel_neon_end(); |
| 165 |
} |
| 166 |
} |
| 167 |
|
| 168 |
static void |
| 169 |
xor_neon_3(unsigned long bytes, unsigned long * __restrict p1, |
| 170 |
const unsigned long * __restrict p2, |
| 171 |
const unsigned long * __restrict p3) |
| 172 |
{ |
| 173 |
if (in_interrupt()) { |
| 174 |
xor_arm4regs_3(bytes, p1, p2, p3); |
| 175 |
} else { |
| 176 |
kernel_neon_begin(); |
| 177 |
xor_block_neon_inner.do_3(bytes, p1, p2, p3); |
| 178 |
kernel_neon_end(); |
| 179 |
} |
| 180 |
} |
| 181 |
|
| 182 |
static void |
| 183 |
xor_neon_4(unsigned long bytes, unsigned long * __restrict p1, |
| 184 |
const unsigned long * __restrict p2, |
| 185 |
const unsigned long * __restrict p3, |
| 186 |
const unsigned long * __restrict p4) |
| 187 |
{ |
| 188 |
if (in_interrupt()) { |
| 189 |
xor_arm4regs_4(bytes, p1, p2, p3, p4); |
| 190 |
} else { |
| 191 |
kernel_neon_begin(); |
| 192 |
xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4); |
| 193 |
kernel_neon_end(); |
| 194 |
} |
| 195 |
} |
| 196 |
|
| 197 |
static void |
| 198 |
xor_neon_5(unsigned long bytes, unsigned long * __restrict p1, |
| 199 |
const unsigned long * __restrict p2, |
| 200 |
const unsigned long * __restrict p3, |
| 201 |
const unsigned long * __restrict p4, |
| 202 |
const unsigned long * __restrict p5) |
| 203 |
{ |
| 204 |
if (in_interrupt()) { |
| 205 |
xor_arm4regs_5(bytes, p1, p2, p3, p4, p5); |
| 206 |
} else { |
| 207 |
kernel_neon_begin(); |
| 208 |
xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5); |
| 209 |
kernel_neon_end(); |
| 210 |
} |
| 211 |
} |
| 212 |
|
| 213 |
static struct xor_block_template xor_block_neon = { |
| 214 |
.name = "neon", |
| 215 |
.do_2 = xor_neon_2, |
| 216 |
.do_3 = xor_neon_3, |
| 217 |
.do_4 = xor_neon_4, |
| 218 |
.do_5 = xor_neon_5 |
| 219 |
}; |
| 220 |
|
| 221 |
#define NEON_TEMPLATES \ |
| 222 |
do { if (cpu_has_neon()) xor_speed(&xor_block_neon); } while (0) |
| 223 |
#else |
| 224 |
#define NEON_TEMPLATES |
| 225 |
#endif |