| 1 |
/* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 |
/* |
| 3 |
* linux/arch/arm/lib/memmove.S |
| 4 |
* |
| 5 |
* Author: Nicolas Pitre |
| 6 |
* Created: Sep 28, 2005 |
| 7 |
* Copyright: (C) MontaVista Software Inc. |
| 8 |
*/ |
| 9 |
|
| 10 |
#include <linux/linkage.h> |
| 11 |
#include <asm/assembler.h> |
| 12 |
#include <asm/unwind.h> |
| 13 |
|
| 14 |
.text |
| 15 |
|
| 16 |
/* |
| 17 |
* Prototype: void *memmove(void *dest, const void *src, size_t n); |
| 18 |
* |
| 19 |
* Note: |
| 20 |
* |
| 21 |
* If the memory regions don't overlap, we simply branch to memcpy which is |
| 22 |
* normally a bit faster. Otherwise the copy is done going downwards. This |
| 23 |
* is a transposition of the code from copy_template.S but with the copy |
| 24 |
* occurring in the opposite direction. |
| 25 |
*/ |
| 26 |
|
| 27 |
ENTRY(__memmove) |
| 28 |
WEAK(memmove) |
| 29 |
UNWIND( .fnstart ) |
| 30 |
|
| 31 |
subs ip, r0, r1 |
| 32 |
cmphi r2, ip |
| 33 |
bls __memcpy |
| 34 |
UNWIND( .fnend ) |
| 35 |
|
| 36 |
UNWIND( .fnstart ) |
| 37 |
UNWIND( .save {r0, r4, fpreg, lr} ) |
| 38 |
stmfd sp!, {r0, r4, UNWIND(fpreg,) lr} |
| 39 |
UNWIND( .setfp fpreg, sp ) |
| 40 |
UNWIND( mov fpreg, sp ) |
| 41 |
add r1, r1, r2 |
| 42 |
add r0, r0, r2 |
| 43 |
subs r2, r2, #4 |
| 44 |
blt 8f |
| 45 |
ands ip, r0, #3 |
| 46 |
PLD( pld [r1, #-4] ) |
| 47 |
bne 9f |
| 48 |
ands ip, r1, #3 |
| 49 |
bne 10f |
| 50 |
|
| 51 |
1: subs r2, r2, #(28) |
| 52 |
stmfd sp!, {r5, r6, r8, r9} |
| 53 |
blt 5f |
| 54 |
|
| 55 |
CALGN( ands ip, r0, #31 ) |
| 56 |
CALGN( sbcsne r4, ip, r2 ) @ C is always set here |
| 57 |
CALGN( bcs 2f ) |
| 58 |
CALGN( adr r4, 6f ) |
| 59 |
CALGN( subs r2, r2, ip ) @ C is set here |
| 60 |
CALGN( rsb ip, ip, #32 ) |
| 61 |
CALGN( add pc, r4, ip ) |
| 62 |
|
| 63 |
PLD( pld [r1, #-4] ) |
| 64 |
2: PLD( subs r2, r2, #96 ) |
| 65 |
PLD( pld [r1, #-32] ) |
| 66 |
PLD( blt 4f ) |
| 67 |
PLD( pld [r1, #-64] ) |
| 68 |
PLD( pld [r1, #-96] ) |
| 69 |
|
| 70 |
3: PLD( pld [r1, #-128] ) |
| 71 |
4: ldmdb r1!, {r3, r4, r5, r6, r8, r9, ip, lr} |
| 72 |
subs r2, r2, #32 |
| 73 |
stmdb r0!, {r3, r4, r5, r6, r8, r9, ip, lr} |
| 74 |
bge 3b |
| 75 |
PLD( cmn r2, #96 ) |
| 76 |
PLD( bge 4b ) |
| 77 |
|
| 78 |
5: ands ip, r2, #28 |
| 79 |
rsb ip, ip, #32 |
| 80 |
addne pc, pc, ip @ C is always clear here |
| 81 |
b 7f |
| 82 |
6: W(nop) |
| 83 |
W(ldr) r3, [r1, #-4]! |
| 84 |
W(ldr) r4, [r1, #-4]! |
| 85 |
W(ldr) r5, [r1, #-4]! |
| 86 |
W(ldr) r6, [r1, #-4]! |
| 87 |
W(ldr) r8, [r1, #-4]! |
| 88 |
W(ldr) r9, [r1, #-4]! |
| 89 |
W(ldr) lr, [r1, #-4]! |
| 90 |
|
| 91 |
add pc, pc, ip |
| 92 |
nop |
| 93 |
W(nop) |
| 94 |
W(str) r3, [r0, #-4]! |
| 95 |
W(str) r4, [r0, #-4]! |
| 96 |
W(str) r5, [r0, #-4]! |
| 97 |
W(str) r6, [r0, #-4]! |
| 98 |
W(str) r8, [r0, #-4]! |
| 99 |
W(str) r9, [r0, #-4]! |
| 100 |
W(str) lr, [r0, #-4]! |
| 101 |
|
| 102 |
CALGN( bcs 2b ) |
| 103 |
|
| 104 |
7: ldmfd sp!, {r5, r6, r8, r9} |
| 105 |
|
| 106 |
8: movs r2, r2, lsl #31 |
| 107 |
ldrbne r3, [r1, #-1]! |
| 108 |
ldrbcs r4, [r1, #-1]! |
| 109 |
ldrbcs ip, [r1, #-1] |
| 110 |
strbne r3, [r0, #-1]! |
| 111 |
strbcs r4, [r0, #-1]! |
| 112 |
strbcs ip, [r0, #-1] |
| 113 |
ldmfd sp!, {r0, r4, UNWIND(fpreg,) pc} |
| 114 |
|
| 115 |
9: cmp ip, #2 |
| 116 |
ldrbgt r3, [r1, #-1]! |
| 117 |
ldrbge r4, [r1, #-1]! |
| 118 |
ldrb lr, [r1, #-1]! |
| 119 |
strbgt r3, [r0, #-1]! |
| 120 |
strbge r4, [r0, #-1]! |
| 121 |
subs r2, r2, ip |
| 122 |
strb lr, [r0, #-1]! |
| 123 |
blt 8b |
| 124 |
ands ip, r1, #3 |
| 125 |
beq 1b |
| 126 |
|
| 127 |
10: bic r1, r1, #3 |
| 128 |
cmp ip, #2 |
| 129 |
ldr r3, [r1, #0] |
| 130 |
beq 17f |
| 131 |
blt 18f |
| 132 |
|
| 133 |
|
| 134 |
.macro backward_copy_shift push pull |
| 135 |
|
| 136 |
subs r2, r2, #28 |
| 137 |
blt 14f |
| 138 |
|
| 139 |
CALGN( ands ip, r0, #31 ) |
| 140 |
CALGN( sbcsne r4, ip, r2 ) @ C is always set here |
| 141 |
CALGN( subcc r2, r2, ip ) |
| 142 |
CALGN( bcc 15f ) |
| 143 |
|
| 144 |
11: stmfd sp!, {r5, r6, r8 - r10} |
| 145 |
|
| 146 |
PLD( pld [r1, #-4] ) |
| 147 |
PLD( subs r2, r2, #96 ) |
| 148 |
PLD( pld [r1, #-32] ) |
| 149 |
PLD( blt 13f ) |
| 150 |
PLD( pld [r1, #-64] ) |
| 151 |
PLD( pld [r1, #-96] ) |
| 152 |
|
| 153 |
12: PLD( pld [r1, #-128] ) |
| 154 |
13: ldmdb r1!, {r8, r9, r10, ip} |
| 155 |
mov lr, r3, lspush #\push |
| 156 |
subs r2, r2, #32 |
| 157 |
ldmdb r1!, {r3, r4, r5, r6} |
| 158 |
orr lr, lr, ip, lspull #\pull |
| 159 |
mov ip, ip, lspush #\push |
| 160 |
orr ip, ip, r10, lspull #\pull |
| 161 |
mov r10, r10, lspush #\push |
| 162 |
orr r10, r10, r9, lspull #\pull |
| 163 |
mov r9, r9, lspush #\push |
| 164 |
orr r9, r9, r8, lspull #\pull |
| 165 |
mov r8, r8, lspush #\push |
| 166 |
orr r8, r8, r6, lspull #\pull |
| 167 |
mov r6, r6, lspush #\push |
| 168 |
orr r6, r6, r5, lspull #\pull |
| 169 |
mov r5, r5, lspush #\push |
| 170 |
orr r5, r5, r4, lspull #\pull |
| 171 |
mov r4, r4, lspush #\push |
| 172 |
orr r4, r4, r3, lspull #\pull |
| 173 |
stmdb r0!, {r4 - r6, r8 - r10, ip, lr} |
| 174 |
bge 12b |
| 175 |
PLD( cmn r2, #96 ) |
| 176 |
PLD( bge 13b ) |
| 177 |
|
| 178 |
ldmfd sp!, {r5, r6, r8 - r10} |
| 179 |
|
| 180 |
14: ands ip, r2, #28 |
| 181 |
beq 16f |
| 182 |
|
| 183 |
15: mov lr, r3, lspush #\push |
| 184 |
ldr r3, [r1, #-4]! |
| 185 |
subs ip, ip, #4 |
| 186 |
orr lr, lr, r3, lspull #\pull |
| 187 |
str lr, [r0, #-4]! |
| 188 |
bgt 15b |
| 189 |
CALGN( cmp r2, #0 ) |
| 190 |
CALGN( bge 11b ) |
| 191 |
|
| 192 |
16: add r1, r1, #(\pull / 8) |
| 193 |
b 8b |
| 194 |
|
| 195 |
.endm |
| 196 |
|
| 197 |
|
| 198 |
backward_copy_shift push=8 pull=24 |
| 199 |
|
| 200 |
17: backward_copy_shift push=16 pull=16 |
| 201 |
|
| 202 |
18: backward_copy_shift push=24 pull=8 |
| 203 |
|
| 204 |
UNWIND( .fnend ) |
| 205 |
ENDPROC(memmove) |
| 206 |
ENDPROC(__memmove) |