Debian Codegraph @

lib/aarch32/cache_helpers.S

raw
1 /*
2 * Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved.
3 *
4 * SPDX-License-Identifier: BSD-3-Clause
5 */
6
7 #include <arch.h>
8 #include <asm_macros.S>
9
10 .globl flush_dcache_range
11 .globl clean_dcache_range
12 .globl inv_dcache_range
13 .globl dcsw_op_louis
14 .globl dcsw_op_all
15 .globl dcsw_op_level1
16 .globl dcsw_op_level2
17 .globl dcsw_op_level3
18
19 /*
20 * This macro can be used for implementing various data cache operations `op`
21 */
22 .macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2
23 /* Exit early if size is zero */
24 cmp r1, #0
25 beq exit_loop_\op
26 dcache_line_size r2, r3
27 add r1, r0, r1
28 sub r3, r2, #1
29 bic r0, r0, r3
30 loop_\op:
31 stcopr r0, \coproc, \opc1, \CRn, \CRm, \opc2
32 add r0, r0, r2
33 cmp r0, r1
34 blo loop_\op
35 dsb sy
36 exit_loop_\op:
37 bx lr
38 .endm
39
40 /* ------------------------------------------
41 * Clean+Invalidate from base address till
42 * size. 'r0' = addr, 'r1' = size
43 * ------------------------------------------
44 */
45 func flush_dcache_range
46 do_dcache_maintenance_by_mva cimvac, DCCIMVAC
47 endfunc flush_dcache_range
48
49 /* ------------------------------------------
50 * Clean from base address till size.
51 * 'r0' = addr, 'r1' = size
52 * ------------------------------------------
53 */
54 func clean_dcache_range
55 do_dcache_maintenance_by_mva cmvac, DCCMVAC
56 endfunc clean_dcache_range
57
58 /* ------------------------------------------
59 * Invalidate from base address till
60 * size. 'r0' = addr, 'r1' = size
61 * ------------------------------------------
62 */
63 func inv_dcache_range
64 do_dcache_maintenance_by_mva imvac, DCIMVAC
65 endfunc inv_dcache_range
66
67 /* ----------------------------------------------------------------
68 * Data cache operations by set/way to the level specified
69 *
70 * The main function, do_dcsw_op requires:
71 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
72 * as defined in arch.h
73 * r1: The cache level to begin operation from
74 * r2: clidr_el1
75 * r3: The last cache level to operate on
76 * and will carry out the operation on each data cache from level 0
77 * to the level in r3 in sequence
78 *
79 * The dcsw_op macro sets up the r2 and r3 parameters based on
80 * clidr_el1 cache information before invoking the main function
81 * ----------------------------------------------------------------
82 */
83
84 .macro dcsw_op shift, fw, ls
85 ldcopr r2, CLIDR
86 ubfx r3, r2, \shift, \fw
87 lsl r3, r3, \ls
88 mov r1, #0
89 b do_dcsw_op
90 .endm
91
92 func do_dcsw_op
93 push {r4-r12, lr}
94 ldcopr r8, ID_MMFR4 // stash FEAT_CCIDX identifier in r8
95 ubfx r8, r8, #ID_MMFR4_CCIDX_SHIFT, #ID_MMFR4_CCIDX_LENGTH
96 adr r11, dcsw_loop_table // compute cache op based on the operation type
97 add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions
98 loop1:
99 add r10, r1, r1, LSR #1 // Work out 3x current cache level
100 mov r12, r2, LSR r10 // extract cache type bits from clidr
101 and r12, r12, #7 // mask the bits for current cache only
102 cmp r12, #2 // see what cache we have at this level
103 blo level_done // no cache or only instruction cache at this level
104
105 stcopr r1, CSSELR // select current cache level in csselr
106 isb // isb to sych the new cssr&csidr
107 ldcopr r12, CCSIDR // read the new ccsidr
108 and r10, r12, #7 // extract the length of the cache lines
109 add r10, r10, #4 // add 4 (r10 = line length offset)
110
111 cmp r8, #0 // check for FEAT_CCIDX for Associativity
112 beq 1f
113 ubfx r4, r12, #3, #21 // r4 = associativity CCSIDR[23:3]
114 b 2f
115 1:
116 ubfx r4, r12, #3, #10 // r4 = associativity CCSIDR[12:3]
117 2:
118 clz r5, r4 // r5 = the bit position of the way size increment
119 mov r9, r4 // r9 working copy of the aligned max way number
120
121 loop2:
122 cmp r8, #0 // check for FEAT_CCIDX for NumSets
123 beq 3f
124 ldcopr r12, CCSIDR2 // FEAT_CCIDX numsets is in CCSIDR2
125 ubfx r7, r12, #0, #24 // r7 = numsets CCSIDR2[23:0]
126 b loop3
127 3:
128 ubfx r7, r12, #13, #15 // r7 = numsets CCSIDR[27:13]
129 loop3:
130 orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0
131 orr r0, r0, r7, LSL r10 // factor in the set number
132
133 blx r6
134 subs r7, r7, #1 // decrement the set number
135 bhs loop3
136 subs r9, r9, #1 // decrement the way number
137 bhs loop2
138 level_done:
139 add r1, r1, #2 // increment the cache number
140 cmp r3, r1
141 // Ensure completion of previous cache maintenance instruction. Note
142 // this also mitigates erratum 814220 on Cortex-A7
143 dsb sy
144 bhi loop1
145
146 mov r6, #0
147 stcopr r6, CSSELR //select cache level 0 in csselr
148 dsb sy
149 isb
150 pop {r4-r12, pc}
151
152 dcsw_loop_table:
153 stcopr r0, DCISW
154 bx lr
155 stcopr r0, DCCISW
156 bx lr
157 stcopr r0, DCCSW
158 bx lr
159
160 endfunc do_dcsw_op
161
162 /* ---------------------------------------------------------------
163 * Data cache operations by set/way till PoU.
164 *
165 * The function requires :
166 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
167 * as defined in arch.h
168 * ---------------------------------------------------------------
169 */
170 func dcsw_op_louis
171 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
172 endfunc dcsw_op_louis
173
174 /* ---------------------------------------------------------------
175 * Data cache operations by set/way till PoC.
176 *
177 * The function requires :
178 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
179 * as defined in arch.h
180 * ---------------------------------------------------------------
181 */
182 func dcsw_op_all
183 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT
184 endfunc dcsw_op_all
185
186
187 /* ---------------------------------------------------------------
188 * Helper macro for data cache operations by set/way for the
189 * level specified
190 * ---------------------------------------------------------------
191 */
192 .macro dcsw_op_level level
193 ldcopr r2, CLIDR
194 mov r3, \level
195 sub r1, r3, #2
196 b do_dcsw_op
197 .endm
198
199 /* ---------------------------------------------------------------
200 * Data cache operations by set/way for level 1 cache
201 *
202 * The main function, do_dcsw_op requires:
203 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
204 * as defined in arch.h
205 * ---------------------------------------------------------------
206 */
207 func dcsw_op_level1
208 dcsw_op_level #(1 << LEVEL_SHIFT)
209 endfunc dcsw_op_level1
210
211 /* ---------------------------------------------------------------
212 * Data cache operations by set/way for level 2 cache
213 *
214 * The main function, do_dcsw_op requires:
215 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
216 * as defined in arch.h
217 * ---------------------------------------------------------------
218 */
219 func dcsw_op_level2
220 dcsw_op_level #(2 << LEVEL_SHIFT)
221 endfunc dcsw_op_level2
222
223 /* ---------------------------------------------------------------
224 * Data cache operations by set/way for level 3 cache
225 *
226 * The main function, do_dcsw_op requires:
227 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW),
228 * as defined in arch.h
229 * ---------------------------------------------------------------
230 */
231 func dcsw_op_level3
232 dcsw_op_level #(3 << LEVEL_SHIFT)
233 endfunc dcsw_op_level3