| 1 |
/* |
| 2 |
* Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved. |
| 3 |
* |
| 4 |
* SPDX-License-Identifier: BSD-3-Clause |
| 5 |
*/ |
| 6 |
|
| 7 |
#include <arch.h> |
| 8 |
#include <asm_macros.S> |
| 9 |
|
| 10 |
.globl flush_dcache_range |
| 11 |
.globl clean_dcache_range |
| 12 |
.globl inv_dcache_range |
| 13 |
.globl dcsw_op_louis |
| 14 |
.globl dcsw_op_all |
| 15 |
.globl dcsw_op_level1 |
| 16 |
.globl dcsw_op_level2 |
| 17 |
.globl dcsw_op_level3 |
| 18 |
|
| 19 |
/* |
| 20 |
* This macro can be used for implementing various data cache operations `op` |
| 21 |
*/ |
| 22 |
.macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2 |
| 23 |
/* Exit early if size is zero */ |
| 24 |
cmp r1, #0 |
| 25 |
beq exit_loop_\op |
| 26 |
dcache_line_size r2, r3 |
| 27 |
add r1, r0, r1 |
| 28 |
sub r3, r2, #1 |
| 29 |
bic r0, r0, r3 |
| 30 |
loop_\op: |
| 31 |
stcopr r0, \coproc, \opc1, \CRn, \CRm, \opc2 |
| 32 |
add r0, r0, r2 |
| 33 |
cmp r0, r1 |
| 34 |
blo loop_\op |
| 35 |
dsb sy |
| 36 |
exit_loop_\op: |
| 37 |
bx lr |
| 38 |
.endm |
| 39 |
|
| 40 |
/* ------------------------------------------ |
| 41 |
* Clean+Invalidate from base address till |
| 42 |
* size. 'r0' = addr, 'r1' = size |
| 43 |
* ------------------------------------------ |
| 44 |
*/ |
| 45 |
func flush_dcache_range |
| 46 |
do_dcache_maintenance_by_mva cimvac, DCCIMVAC |
| 47 |
endfunc flush_dcache_range |
| 48 |
|
| 49 |
/* ------------------------------------------ |
| 50 |
* Clean from base address till size. |
| 51 |
* 'r0' = addr, 'r1' = size |
| 52 |
* ------------------------------------------ |
| 53 |
*/ |
| 54 |
func clean_dcache_range |
| 55 |
do_dcache_maintenance_by_mva cmvac, DCCMVAC |
| 56 |
endfunc clean_dcache_range |
| 57 |
|
| 58 |
/* ------------------------------------------ |
| 59 |
* Invalidate from base address till |
| 60 |
* size. 'r0' = addr, 'r1' = size |
| 61 |
* ------------------------------------------ |
| 62 |
*/ |
| 63 |
func inv_dcache_range |
| 64 |
do_dcache_maintenance_by_mva imvac, DCIMVAC |
| 65 |
endfunc inv_dcache_range |
| 66 |
|
| 67 |
/* ---------------------------------------------------------------- |
| 68 |
* Data cache operations by set/way to the level specified |
| 69 |
* |
| 70 |
* The main function, do_dcsw_op requires: |
| 71 |
* r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 72 |
* as defined in arch.h |
| 73 |
* r1: The cache level to begin operation from |
| 74 |
* r2: clidr_el1 |
| 75 |
* r3: The last cache level to operate on |
| 76 |
* and will carry out the operation on each data cache from level 0 |
| 77 |
* to the level in r3 in sequence |
| 78 |
* |
| 79 |
* The dcsw_op macro sets up the r2 and r3 parameters based on |
| 80 |
* clidr_el1 cache information before invoking the main function |
| 81 |
* ---------------------------------------------------------------- |
| 82 |
*/ |
| 83 |
|
| 84 |
.macro dcsw_op shift, fw, ls |
| 85 |
ldcopr r2, CLIDR |
| 86 |
ubfx r3, r2, \shift, \fw |
| 87 |
lsl r3, r3, \ls |
| 88 |
mov r1, #0 |
| 89 |
b do_dcsw_op |
| 90 |
.endm |
| 91 |
|
| 92 |
func do_dcsw_op |
| 93 |
push {r4-r12, lr} |
| 94 |
ldcopr r8, ID_MMFR4 // stash FEAT_CCIDX identifier in r8 |
| 95 |
ubfx r8, r8, #ID_MMFR4_CCIDX_SHIFT, #ID_MMFR4_CCIDX_LENGTH |
| 96 |
adr r11, dcsw_loop_table // compute cache op based on the operation type |
| 97 |
add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions |
| 98 |
loop1: |
| 99 |
add r10, r1, r1, LSR #1 // Work out 3x current cache level |
| 100 |
mov r12, r2, LSR r10 // extract cache type bits from clidr |
| 101 |
and r12, r12, #7 // mask the bits for current cache only |
| 102 |
cmp r12, #2 // see what cache we have at this level |
| 103 |
blo level_done // no cache or only instruction cache at this level |
| 104 |
|
| 105 |
stcopr r1, CSSELR // select current cache level in csselr |
| 106 |
isb // isb to sych the new cssr&csidr |
| 107 |
ldcopr r12, CCSIDR // read the new ccsidr |
| 108 |
and r10, r12, #7 // extract the length of the cache lines |
| 109 |
add r10, r10, #4 // add 4 (r10 = line length offset) |
| 110 |
|
| 111 |
cmp r8, #0 // check for FEAT_CCIDX for Associativity |
| 112 |
beq 1f |
| 113 |
ubfx r4, r12, #3, #21 // r4 = associativity CCSIDR[23:3] |
| 114 |
b 2f |
| 115 |
1: |
| 116 |
ubfx r4, r12, #3, #10 // r4 = associativity CCSIDR[12:3] |
| 117 |
2: |
| 118 |
clz r5, r4 // r5 = the bit position of the way size increment |
| 119 |
mov r9, r4 // r9 working copy of the aligned max way number |
| 120 |
|
| 121 |
loop2: |
| 122 |
cmp r8, #0 // check for FEAT_CCIDX for NumSets |
| 123 |
beq 3f |
| 124 |
ldcopr r12, CCSIDR2 // FEAT_CCIDX numsets is in CCSIDR2 |
| 125 |
ubfx r7, r12, #0, #24 // r7 = numsets CCSIDR2[23:0] |
| 126 |
b loop3 |
| 127 |
3: |
| 128 |
ubfx r7, r12, #13, #15 // r7 = numsets CCSIDR[27:13] |
| 129 |
loop3: |
| 130 |
orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0 |
| 131 |
orr r0, r0, r7, LSL r10 // factor in the set number |
| 132 |
|
| 133 |
blx r6 |
| 134 |
subs r7, r7, #1 // decrement the set number |
| 135 |
bhs loop3 |
| 136 |
subs r9, r9, #1 // decrement the way number |
| 137 |
bhs loop2 |
| 138 |
level_done: |
| 139 |
add r1, r1, #2 // increment the cache number |
| 140 |
cmp r3, r1 |
| 141 |
// Ensure completion of previous cache maintenance instruction. Note |
| 142 |
// this also mitigates erratum 814220 on Cortex-A7 |
| 143 |
dsb sy |
| 144 |
bhi loop1 |
| 145 |
|
| 146 |
mov r6, #0 |
| 147 |
stcopr r6, CSSELR //select cache level 0 in csselr |
| 148 |
dsb sy |
| 149 |
isb |
| 150 |
pop {r4-r12, pc} |
| 151 |
|
| 152 |
dcsw_loop_table: |
| 153 |
stcopr r0, DCISW |
| 154 |
bx lr |
| 155 |
stcopr r0, DCCISW |
| 156 |
bx lr |
| 157 |
stcopr r0, DCCSW |
| 158 |
bx lr |
| 159 |
|
| 160 |
endfunc do_dcsw_op |
| 161 |
|
| 162 |
/* --------------------------------------------------------------- |
| 163 |
* Data cache operations by set/way till PoU. |
| 164 |
* |
| 165 |
* The function requires : |
| 166 |
* r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 167 |
* as defined in arch.h |
| 168 |
* --------------------------------------------------------------- |
| 169 |
*/ |
| 170 |
func dcsw_op_louis |
| 171 |
dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT |
| 172 |
endfunc dcsw_op_louis |
| 173 |
|
| 174 |
/* --------------------------------------------------------------- |
| 175 |
* Data cache operations by set/way till PoC. |
| 176 |
* |
| 177 |
* The function requires : |
| 178 |
* r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 179 |
* as defined in arch.h |
| 180 |
* --------------------------------------------------------------- |
| 181 |
*/ |
| 182 |
func dcsw_op_all |
| 183 |
dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT |
| 184 |
endfunc dcsw_op_all |
| 185 |
|
| 186 |
|
| 187 |
/* --------------------------------------------------------------- |
| 188 |
* Helper macro for data cache operations by set/way for the |
| 189 |
* level specified |
| 190 |
* --------------------------------------------------------------- |
| 191 |
*/ |
| 192 |
.macro dcsw_op_level level |
| 193 |
ldcopr r2, CLIDR |
| 194 |
mov r3, \level |
| 195 |
sub r1, r3, #2 |
| 196 |
b do_dcsw_op |
| 197 |
.endm |
| 198 |
|
| 199 |
/* --------------------------------------------------------------- |
| 200 |
* Data cache operations by set/way for level 1 cache |
| 201 |
* |
| 202 |
* The main function, do_dcsw_op requires: |
| 203 |
* r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 204 |
* as defined in arch.h |
| 205 |
* --------------------------------------------------------------- |
| 206 |
*/ |
| 207 |
func dcsw_op_level1 |
| 208 |
dcsw_op_level #(1 << LEVEL_SHIFT) |
| 209 |
endfunc dcsw_op_level1 |
| 210 |
|
| 211 |
/* --------------------------------------------------------------- |
| 212 |
* Data cache operations by set/way for level 2 cache |
| 213 |
* |
| 214 |
* The main function, do_dcsw_op requires: |
| 215 |
* r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 216 |
* as defined in arch.h |
| 217 |
* --------------------------------------------------------------- |
| 218 |
*/ |
| 219 |
func dcsw_op_level2 |
| 220 |
dcsw_op_level #(2 << LEVEL_SHIFT) |
| 221 |
endfunc dcsw_op_level2 |
| 222 |
|
| 223 |
/* --------------------------------------------------------------- |
| 224 |
* Data cache operations by set/way for level 3 cache |
| 225 |
* |
| 226 |
* The main function, do_dcsw_op requires: |
| 227 |
* r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), |
| 228 |
* as defined in arch.h |
| 229 |
* --------------------------------------------------------------- |
| 230 |
*/ |
| 231 |
func dcsw_op_level3 |
| 232 |
dcsw_op_level #(3 << LEVEL_SHIFT) |
| 233 |
endfunc dcsw_op_level3 |