| 1 |
/* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 |
/* |
| 3 |
* linux/arch/arm/lib/csumpartialcopygeneric.S |
| 4 |
* |
| 5 |
* Copyright (C) 1995-2001 Russell King |
| 6 |
*/ |
| 7 |
#include <asm/assembler.h> |
| 8 |
|
| 9 |
/* |
| 10 |
* unsigned int |
| 11 |
* csum_partial_copy_xxx(const char *src, char *dst, int len, int sum, ) |
| 12 |
* r0 = src, r1 = dst, r2 = len, r3 = sum |
| 13 |
* Returns : r0 = checksum |
| 14 |
* |
| 15 |
* Note that 'tst' and 'teq' preserve the carry flag. |
| 16 |
*/ |
| 17 |
|
| 18 |
src .req r0 |
| 19 |
dst .req r1 |
| 20 |
len .req r2 |
| 21 |
sum .req r3 |
| 22 |
|
| 23 |
.Lzero: mov r0, sum |
| 24 |
load_regs |
| 25 |
|
| 26 |
/* |
| 27 |
* Align an unaligned destination pointer. We know that |
| 28 |
* we have >= 8 bytes here, so we don't need to check |
| 29 |
* the length. Note that the source pointer hasn't been |
| 30 |
* aligned yet. |
| 31 |
*/ |
| 32 |
.Ldst_unaligned: |
| 33 |
tst dst, #1 |
| 34 |
beq .Ldst_16bit |
| 35 |
|
| 36 |
load1b ip |
| 37 |
sub len, len, #1 |
| 38 |
adcs sum, sum, ip, put_byte_1 @ update checksum |
| 39 |
strb ip, [dst], #1 |
| 40 |
tst dst, #2 |
| 41 |
reteq lr @ dst is now 32bit aligned |
| 42 |
|
| 43 |
.Ldst_16bit: load2b r8, ip |
| 44 |
sub len, len, #2 |
| 45 |
adcs sum, sum, r8, put_byte_0 |
| 46 |
strb r8, [dst], #1 |
| 47 |
adcs sum, sum, ip, put_byte_1 |
| 48 |
strb ip, [dst], #1 |
| 49 |
ret lr @ dst is now 32bit aligned |
| 50 |
|
| 51 |
/* |
| 52 |
* Handle 0 to 7 bytes, with any alignment of source and |
| 53 |
* destination pointers. Note that when we get here, C = 0 |
| 54 |
*/ |
| 55 |
.Lless8: teq len, #0 @ check for zero count |
| 56 |
beq .Lzero |
| 57 |
|
| 58 |
/* we must have at least one byte. */ |
| 59 |
tst dst, #1 @ dst 16-bit aligned |
| 60 |
beq .Lless8_aligned |
| 61 |
|
| 62 |
/* Align dst */ |
| 63 |
load1b ip |
| 64 |
sub len, len, #1 |
| 65 |
adcs sum, sum, ip, put_byte_1 @ update checksum |
| 66 |
strb ip, [dst], #1 |
| 67 |
tst len, #6 |
| 68 |
beq .Lless8_byteonly |
| 69 |
|
| 70 |
1: load2b r8, ip |
| 71 |
sub len, len, #2 |
| 72 |
adcs sum, sum, r8, put_byte_0 |
| 73 |
strb r8, [dst], #1 |
| 74 |
adcs sum, sum, ip, put_byte_1 |
| 75 |
strb ip, [dst], #1 |
| 76 |
.Lless8_aligned: |
| 77 |
tst len, #6 |
| 78 |
bne 1b |
| 79 |
.Lless8_byteonly: |
| 80 |
tst len, #1 |
| 81 |
beq .Ldone |
| 82 |
load1b r8 |
| 83 |
adcs sum, sum, r8, put_byte_0 @ update checksum |
| 84 |
strb r8, [dst], #1 |
| 85 |
b .Ldone |
| 86 |
|
| 87 |
FN_ENTRY |
| 88 |
save_regs |
| 89 |
mov sum, #-1 |
| 90 |
|
| 91 |
cmp len, #8 @ Ensure that we have at least |
| 92 |
blo .Lless8 @ 8 bytes to copy. |
| 93 |
|
| 94 |
adds sum, sum, #0 @ C = 0 |
| 95 |
tst dst, #3 @ Test destination alignment |
| 96 |
blne .Ldst_unaligned @ align destination, return here |
| 97 |
|
| 98 |
/* |
| 99 |
* Ok, the dst pointer is now 32bit aligned, and we know |
| 100 |
* that we must have more than 4 bytes to copy. Note |
| 101 |
* that C contains the carry from the dst alignment above. |
| 102 |
*/ |
| 103 |
|
| 104 |
tst src, #3 @ Test source alignment |
| 105 |
bne .Lsrc_not_aligned |
| 106 |
|
| 107 |
/* Routine for src & dst aligned */ |
| 108 |
|
| 109 |
bics ip, len, #15 |
| 110 |
beq 2f |
| 111 |
|
| 112 |
1: load4l r4, r5, r6, r7 |
| 113 |
stmia dst!, {r4, r5, r6, r7} |
| 114 |
adcs sum, sum, r4 |
| 115 |
adcs sum, sum, r5 |
| 116 |
adcs sum, sum, r6 |
| 117 |
adcs sum, sum, r7 |
| 118 |
sub ip, ip, #16 |
| 119 |
teq ip, #0 |
| 120 |
bne 1b |
| 121 |
|
| 122 |
2: ands ip, len, #12 |
| 123 |
beq 4f |
| 124 |
tst ip, #8 |
| 125 |
beq 3f |
| 126 |
load2l r4, r5 |
| 127 |
stmia dst!, {r4, r5} |
| 128 |
adcs sum, sum, r4 |
| 129 |
adcs sum, sum, r5 |
| 130 |
tst ip, #4 |
| 131 |
beq 4f |
| 132 |
|
| 133 |
3: load1l r4 |
| 134 |
str r4, [dst], #4 |
| 135 |
adcs sum, sum, r4 |
| 136 |
|
| 137 |
4: ands len, len, #3 |
| 138 |
beq .Ldone |
| 139 |
load1l r4 |
| 140 |
tst len, #2 |
| 141 |
mov r5, r4, get_byte_0 |
| 142 |
beq .Lexit |
| 143 |
adcs sum, sum, r4, lspush #16 |
| 144 |
strb r5, [dst], #1 |
| 145 |
mov r5, r4, get_byte_1 |
| 146 |
strb r5, [dst], #1 |
| 147 |
mov r5, r4, get_byte_2 |
| 148 |
.Lexit: tst len, #1 |
| 149 |
strbne r5, [dst], #1 |
| 150 |
andne r5, r5, #255 |
| 151 |
adcsne sum, sum, r5, put_byte_0 |
| 152 |
|
| 153 |
/* |
| 154 |
* If the dst pointer was not 16-bit aligned, we |
| 155 |
* need to rotate the checksum here to get around |
| 156 |
* the inefficient byte manipulations in the |
| 157 |
* architecture independent code. |
| 158 |
*/ |
| 159 |
.Ldone: adc r0, sum, #0 |
| 160 |
ldr sum, [sp, #0] @ dst |
| 161 |
tst sum, #1 |
| 162 |
movne r0, r0, ror #8 |
| 163 |
load_regs |
| 164 |
|
| 165 |
.Lsrc_not_aligned: |
| 166 |
adc sum, sum, #0 @ include C from dst alignment |
| 167 |
and ip, src, #3 |
| 168 |
bic src, src, #3 |
| 169 |
load1l r5 |
| 170 |
cmp ip, #2 |
| 171 |
beq .Lsrc2_aligned |
| 172 |
bhi .Lsrc3_aligned |
| 173 |
mov r4, r5, lspull #8 @ C = 0 |
| 174 |
bics ip, len, #15 |
| 175 |
beq 2f |
| 176 |
1: load4l r5, r6, r7, r8 |
| 177 |
orr r4, r4, r5, lspush #24 |
| 178 |
mov r5, r5, lspull #8 |
| 179 |
orr r5, r5, r6, lspush #24 |
| 180 |
mov r6, r6, lspull #8 |
| 181 |
orr r6, r6, r7, lspush #24 |
| 182 |
mov r7, r7, lspull #8 |
| 183 |
orr r7, r7, r8, lspush #24 |
| 184 |
stmia dst!, {r4, r5, r6, r7} |
| 185 |
adcs sum, sum, r4 |
| 186 |
adcs sum, sum, r5 |
| 187 |
adcs sum, sum, r6 |
| 188 |
adcs sum, sum, r7 |
| 189 |
mov r4, r8, lspull #8 |
| 190 |
sub ip, ip, #16 |
| 191 |
teq ip, #0 |
| 192 |
bne 1b |
| 193 |
2: ands ip, len, #12 |
| 194 |
beq 4f |
| 195 |
tst ip, #8 |
| 196 |
beq 3f |
| 197 |
load2l r5, r6 |
| 198 |
orr r4, r4, r5, lspush #24 |
| 199 |
mov r5, r5, lspull #8 |
| 200 |
orr r5, r5, r6, lspush #24 |
| 201 |
stmia dst!, {r4, r5} |
| 202 |
adcs sum, sum, r4 |
| 203 |
adcs sum, sum, r5 |
| 204 |
mov r4, r6, lspull #8 |
| 205 |
tst ip, #4 |
| 206 |
beq 4f |
| 207 |
3: load1l r5 |
| 208 |
orr r4, r4, r5, lspush #24 |
| 209 |
str r4, [dst], #4 |
| 210 |
adcs sum, sum, r4 |
| 211 |
mov r4, r5, lspull #8 |
| 212 |
4: ands len, len, #3 |
| 213 |
beq .Ldone |
| 214 |
mov r5, r4, get_byte_0 |
| 215 |
tst len, #2 |
| 216 |
beq .Lexit |
| 217 |
adcs sum, sum, r4, lspush #16 |
| 218 |
strb r5, [dst], #1 |
| 219 |
mov r5, r4, get_byte_1 |
| 220 |
strb r5, [dst], #1 |
| 221 |
mov r5, r4, get_byte_2 |
| 222 |
b .Lexit |
| 223 |
|
| 224 |
.Lsrc2_aligned: mov r4, r5, lspull #16 |
| 225 |
adds sum, sum, #0 |
| 226 |
bics ip, len, #15 |
| 227 |
beq 2f |
| 228 |
1: load4l r5, r6, r7, r8 |
| 229 |
orr r4, r4, r5, lspush #16 |
| 230 |
mov r5, r5, lspull #16 |
| 231 |
orr r5, r5, r6, lspush #16 |
| 232 |
mov r6, r6, lspull #16 |
| 233 |
orr r6, r6, r7, lspush #16 |
| 234 |
mov r7, r7, lspull #16 |
| 235 |
orr r7, r7, r8, lspush #16 |
| 236 |
stmia dst!, {r4, r5, r6, r7} |
| 237 |
adcs sum, sum, r4 |
| 238 |
adcs sum, sum, r5 |
| 239 |
adcs sum, sum, r6 |
| 240 |
adcs sum, sum, r7 |
| 241 |
mov r4, r8, lspull #16 |
| 242 |
sub ip, ip, #16 |
| 243 |
teq ip, #0 |
| 244 |
bne 1b |
| 245 |
2: ands ip, len, #12 |
| 246 |
beq 4f |
| 247 |
tst ip, #8 |
| 248 |
beq 3f |
| 249 |
load2l r5, r6 |
| 250 |
orr r4, r4, r5, lspush #16 |
| 251 |
mov r5, r5, lspull #16 |
| 252 |
orr r5, r5, r6, lspush #16 |
| 253 |
stmia dst!, {r4, r5} |
| 254 |
adcs sum, sum, r4 |
| 255 |
adcs sum, sum, r5 |
| 256 |
mov r4, r6, lspull #16 |
| 257 |
tst ip, #4 |
| 258 |
beq 4f |
| 259 |
3: load1l r5 |
| 260 |
orr r4, r4, r5, lspush #16 |
| 261 |
str r4, [dst], #4 |
| 262 |
adcs sum, sum, r4 |
| 263 |
mov r4, r5, lspull #16 |
| 264 |
4: ands len, len, #3 |
| 265 |
beq .Ldone |
| 266 |
mov r5, r4, get_byte_0 |
| 267 |
tst len, #2 |
| 268 |
beq .Lexit |
| 269 |
adcs sum, sum, r4 |
| 270 |
strb r5, [dst], #1 |
| 271 |
mov r5, r4, get_byte_1 |
| 272 |
strb r5, [dst], #1 |
| 273 |
tst len, #1 |
| 274 |
beq .Ldone |
| 275 |
load1b r5 |
| 276 |
b .Lexit |
| 277 |
|
| 278 |
.Lsrc3_aligned: mov r4, r5, lspull #24 |
| 279 |
adds sum, sum, #0 |
| 280 |
bics ip, len, #15 |
| 281 |
beq 2f |
| 282 |
1: load4l r5, r6, r7, r8 |
| 283 |
orr r4, r4, r5, lspush #8 |
| 284 |
mov r5, r5, lspull #24 |
| 285 |
orr r5, r5, r6, lspush #8 |
| 286 |
mov r6, r6, lspull #24 |
| 287 |
orr r6, r6, r7, lspush #8 |
| 288 |
mov r7, r7, lspull #24 |
| 289 |
orr r7, r7, r8, lspush #8 |
| 290 |
stmia dst!, {r4, r5, r6, r7} |
| 291 |
adcs sum, sum, r4 |
| 292 |
adcs sum, sum, r5 |
| 293 |
adcs sum, sum, r6 |
| 294 |
adcs sum, sum, r7 |
| 295 |
mov r4, r8, lspull #24 |
| 296 |
sub ip, ip, #16 |
| 297 |
teq ip, #0 |
| 298 |
bne 1b |
| 299 |
2: ands ip, len, #12 |
| 300 |
beq 4f |
| 301 |
tst ip, #8 |
| 302 |
beq 3f |
| 303 |
load2l r5, r6 |
| 304 |
orr r4, r4, r5, lspush #8 |
| 305 |
mov r5, r5, lspull #24 |
| 306 |
orr r5, r5, r6, lspush #8 |
| 307 |
stmia dst!, {r4, r5} |
| 308 |
adcs sum, sum, r4 |
| 309 |
adcs sum, sum, r5 |
| 310 |
mov r4, r6, lspull #24 |
| 311 |
tst ip, #4 |
| 312 |
beq 4f |
| 313 |
3: load1l r5 |
| 314 |
orr r4, r4, r5, lspush #8 |
| 315 |
str r4, [dst], #4 |
| 316 |
adcs sum, sum, r4 |
| 317 |
mov r4, r5, lspull #24 |
| 318 |
4: ands len, len, #3 |
| 319 |
beq .Ldone |
| 320 |
mov r5, r4, get_byte_0 |
| 321 |
tst len, #2 |
| 322 |
beq .Lexit |
| 323 |
strb r5, [dst], #1 |
| 324 |
adcs sum, sum, r4 |
| 325 |
load1l r4 |
| 326 |
mov r5, r4, get_byte_0 |
| 327 |
strb r5, [dst], #1 |
| 328 |
adcs sum, sum, r4, lspush #24 |
| 329 |
mov r5, r4, get_byte_1 |
| 330 |
b .Lexit |
| 331 |
FN_EXIT |