| 1 |
/* |
| 2 |
* Copyright (c) 2013-2024, Arm Limited and Contributors. All rights reserved. |
| 3 |
* |
| 4 |
* SPDX-License-Identifier: BSD-3-Clause |
| 5 |
*/ |
| 6 |
|
| 7 |
#include <arch.h> |
| 8 |
#include <asm_macros.S> |
| 9 |
#include <assert_macros.S> |
| 10 |
#include <context.h> |
| 11 |
#include <el3_common_macros.S> |
| 12 |
#include <platform_def.h> |
| 13 |
|
| 14 |
#if CTX_INCLUDE_FPREGS |
| 15 |
.global fpregs_context_save |
| 16 |
.global fpregs_context_restore |
| 17 |
#endif /* CTX_INCLUDE_FPREGS */ |
| 18 |
|
| 19 |
#if CTX_INCLUDE_SVE_REGS |
| 20 |
.global sve_context_save |
| 21 |
.global sve_context_restore |
| 22 |
#endif /* CTX_INCLUDE_SVE_REGS */ |
| 23 |
|
| 24 |
#if ERRATA_SPECULATIVE_AT |
| 25 |
.global save_and_update_ptw_el1_sys_regs |
| 26 |
#endif /* ERRATA_SPECULATIVE_AT */ |
| 27 |
|
| 28 |
.global prepare_el3_entry |
| 29 |
.global restore_gp_pmcr_pauth_regs |
| 30 |
.global el3_exit |
| 31 |
|
| 32 |
/* Following macros will be used if any of CTX_INCLUDE_FPREGS or CTX_INCLUDE_SVE_REGS is enabled */ |
| 33 |
#if CTX_INCLUDE_FPREGS || CTX_INCLUDE_SVE_REGS |
| 34 |
.macro fpregs_state_save base:req hold:req |
| 35 |
mrs \hold, fpsr |
| 36 |
str \hold, [\base, #CTX_SIMD_FPSR] |
| 37 |
|
| 38 |
mrs \hold, fpcr |
| 39 |
str \hold, [\base, #CTX_SIMD_FPCR] |
| 40 |
|
| 41 |
#if CTX_INCLUDE_AARCH32_REGS && CTX_INCLUDE_FPREGS |
| 42 |
mrs \hold, fpexc32_el2 |
| 43 |
str \hold, [\base, #CTX_SIMD_FPEXC32] |
| 44 |
#endif |
| 45 |
.endm |
| 46 |
|
| 47 |
.macro fpregs_state_restore base:req hold:req |
| 48 |
ldr \hold, [\base, #CTX_SIMD_FPSR] |
| 49 |
msr fpsr, \hold |
| 50 |
|
| 51 |
ldr \hold, [\base, #CTX_SIMD_FPCR] |
| 52 |
msr fpcr, \hold |
| 53 |
|
| 54 |
#if CTX_INCLUDE_AARCH32_REGS && CTX_INCLUDE_FPREGS |
| 55 |
ldr \hold, [\base, #CTX_SIMD_FPEXC32] |
| 56 |
msr fpexc32_el2, \hold |
| 57 |
#endif |
| 58 |
.endm |
| 59 |
|
| 60 |
#endif /* CTX_INCLUDE_FPREGS || CTX_INCLUDE_SVE_REGS */ |
| 61 |
|
| 62 |
/* ------------------------------------------------------------------ |
| 63 |
* The following function follows the aapcs_64 strictly to use |
| 64 |
* x9-x17 (temporary caller-saved registers according to AArch64 PCS) |
| 65 |
* to save floating point register context. It assumes that 'x0' is |
| 66 |
* pointing to a 'fp_regs' structure where the register context will |
| 67 |
* be saved. |
| 68 |
* |
| 69 |
* Access to VFP registers will trap if CPTR_EL3.TFP is set. |
| 70 |
* However currently we don't use VFP registers nor set traps in |
| 71 |
* Trusted Firmware, and assume it's cleared. |
| 72 |
* |
| 73 |
* TODO: Revisit when VFP is used in secure world |
| 74 |
* ------------------------------------------------------------------ |
| 75 |
*/ |
| 76 |
#if CTX_INCLUDE_FPREGS |
| 77 |
func fpregs_context_save |
| 78 |
stp q0, q1, [x0], #32 |
| 79 |
stp q2, q3, [x0], #32 |
| 80 |
stp q4, q5, [x0], #32 |
| 81 |
stp q6, q7, [x0], #32 |
| 82 |
stp q8, q9, [x0], #32 |
| 83 |
stp q10, q11, [x0], #32 |
| 84 |
stp q12, q13, [x0], #32 |
| 85 |
stp q14, q15, [x0], #32 |
| 86 |
stp q16, q17, [x0], #32 |
| 87 |
stp q18, q19, [x0], #32 |
| 88 |
stp q20, q21, [x0], #32 |
| 89 |
stp q22, q23, [x0], #32 |
| 90 |
stp q24, q25, [x0], #32 |
| 91 |
stp q26, q27, [x0], #32 |
| 92 |
stp q28, q29, [x0], #32 |
| 93 |
stp q30, q31, [x0], #32 |
| 94 |
|
| 95 |
fpregs_state_save x0, x9 |
| 96 |
|
| 97 |
ret |
| 98 |
endfunc fpregs_context_save |
| 99 |
|
| 100 |
/* ------------------------------------------------------------------ |
| 101 |
* The following function follows the aapcs_64 strictly to use x9-x17 |
| 102 |
* (temporary caller-saved registers according to AArch64 PCS) to |
| 103 |
* restore floating point register context. It assumes that 'x0' is |
| 104 |
* pointing to a 'fp_regs' structure from where the register context |
| 105 |
* will be restored. |
| 106 |
* |
| 107 |
* Access to VFP registers will trap if CPTR_EL3.TFP is set. |
| 108 |
* However currently we don't use VFP registers nor set traps in |
| 109 |
* Trusted Firmware, and assume it's cleared. |
| 110 |
* |
| 111 |
* TODO: Revisit when VFP is used in secure world |
| 112 |
* ------------------------------------------------------------------ |
| 113 |
*/ |
| 114 |
func fpregs_context_restore |
| 115 |
ldp q0, q1, [x0], #32 |
| 116 |
ldp q2, q3, [x0], #32 |
| 117 |
ldp q4, q5, [x0], #32 |
| 118 |
ldp q6, q7, [x0], #32 |
| 119 |
ldp q8, q9, [x0], #32 |
| 120 |
ldp q10, q11, [x0], #32 |
| 121 |
ldp q12, q13, [x0], #32 |
| 122 |
ldp q14, q15, [x0], #32 |
| 123 |
ldp q16, q17, [x0], #32 |
| 124 |
ldp q18, q19, [x0], #32 |
| 125 |
ldp q20, q21, [x0], #32 |
| 126 |
ldp q22, q23, [x0], #32 |
| 127 |
ldp q24, q25, [x0], #32 |
| 128 |
ldp q26, q27, [x0], #32 |
| 129 |
ldp q28, q29, [x0], #32 |
| 130 |
ldp q30, q31, [x0], #32 |
| 131 |
|
| 132 |
fpregs_state_restore x0, x9 |
| 133 |
|
| 134 |
ret |
| 135 |
endfunc fpregs_context_restore |
| 136 |
#endif /* CTX_INCLUDE_FPREGS */ |
| 137 |
|
| 138 |
#if CTX_INCLUDE_SVE_REGS |
| 139 |
/* |
| 140 |
* Helper macros for SVE predicates save/restore operations. |
| 141 |
*/ |
| 142 |
.macro sve_predicate_op op:req reg:req |
| 143 |
\op p0, [\reg, #0, MUL VL] |
| 144 |
\op p1, [\reg, #1, MUL VL] |
| 145 |
\op p2, [\reg, #2, MUL VL] |
| 146 |
\op p3, [\reg, #3, MUL VL] |
| 147 |
\op p4, [\reg, #4, MUL VL] |
| 148 |
\op p5, [\reg, #5, MUL VL] |
| 149 |
\op p6, [\reg, #6, MUL VL] |
| 150 |
\op p7, [\reg, #7, MUL VL] |
| 151 |
\op p8, [\reg, #8, MUL VL] |
| 152 |
\op p9, [\reg, #9, MUL VL] |
| 153 |
\op p10, [\reg, #10, MUL VL] |
| 154 |
\op p11, [\reg, #11, MUL VL] |
| 155 |
\op p12, [\reg, #12, MUL VL] |
| 156 |
\op p13, [\reg, #13, MUL VL] |
| 157 |
\op p14, [\reg, #14, MUL VL] |
| 158 |
\op p15, [\reg, #15, MUL VL] |
| 159 |
.endm |
| 160 |
|
| 161 |
.macro sve_vectors_op op:req reg:req |
| 162 |
\op z0, [\reg, #0, MUL VL] |
| 163 |
\op z1, [\reg, #1, MUL VL] |
| 164 |
\op z2, [\reg, #2, MUL VL] |
| 165 |
\op z3, [\reg, #3, MUL VL] |
| 166 |
\op z4, [\reg, #4, MUL VL] |
| 167 |
\op z5, [\reg, #5, MUL VL] |
| 168 |
\op z6, [\reg, #6, MUL VL] |
| 169 |
\op z7, [\reg, #7, MUL VL] |
| 170 |
\op z8, [\reg, #8, MUL VL] |
| 171 |
\op z9, [\reg, #9, MUL VL] |
| 172 |
\op z10, [\reg, #10, MUL VL] |
| 173 |
\op z11, [\reg, #11, MUL VL] |
| 174 |
\op z12, [\reg, #12, MUL VL] |
| 175 |
\op z13, [\reg, #13, MUL VL] |
| 176 |
\op z14, [\reg, #14, MUL VL] |
| 177 |
\op z15, [\reg, #15, MUL VL] |
| 178 |
\op z16, [\reg, #16, MUL VL] |
| 179 |
\op z17, [\reg, #17, MUL VL] |
| 180 |
\op z18, [\reg, #18, MUL VL] |
| 181 |
\op z19, [\reg, #19, MUL VL] |
| 182 |
\op z20, [\reg, #20, MUL VL] |
| 183 |
\op z21, [\reg, #21, MUL VL] |
| 184 |
\op z22, [\reg, #22, MUL VL] |
| 185 |
\op z23, [\reg, #23, MUL VL] |
| 186 |
\op z24, [\reg, #24, MUL VL] |
| 187 |
\op z25, [\reg, #25, MUL VL] |
| 188 |
\op z26, [\reg, #26, MUL VL] |
| 189 |
\op z27, [\reg, #27, MUL VL] |
| 190 |
\op z28, [\reg, #28, MUL VL] |
| 191 |
\op z29, [\reg, #29, MUL VL] |
| 192 |
\op z30, [\reg, #30, MUL VL] |
| 193 |
\op z31, [\reg, #31, MUL VL] |
| 194 |
.endm |
| 195 |
|
| 196 |
/* ------------------------------------------------------------------ |
| 197 |
* The following function follows the aapcs_64 strictly to use x9-x17 |
| 198 |
* (temporary caller-saved registers according to AArch64 PCS) to |
| 199 |
* restore SVE register context. It assumes that 'x0' is |
| 200 |
* pointing to a 'sve_regs_t' structure to which the register context |
| 201 |
* will be saved. |
| 202 |
* ------------------------------------------------------------------ |
| 203 |
*/ |
| 204 |
func sve_context_save |
| 205 |
.arch_extension sve |
| 206 |
/* Temporarily enable SVE */ |
| 207 |
mrs x10, cptr_el3 |
| 208 |
orr x11, x10, #CPTR_EZ_BIT |
| 209 |
bic x11, x11, #TFP_BIT |
| 210 |
msr cptr_el3, x11 |
| 211 |
isb |
| 212 |
|
| 213 |
/* zcr_el3 */ |
| 214 |
mrs x12, S3_6_C1_C2_0 |
| 215 |
mov x13, #((SVE_VECTOR_LEN >> 7) - 1) |
| 216 |
msr S3_6_C1_C2_0, x13 |
| 217 |
isb |
| 218 |
|
| 219 |
/* Predicate registers */ |
| 220 |
mov x13, #CTX_SIMD_PREDICATES |
| 221 |
add x9, x0, x13 |
| 222 |
sve_predicate_op str, x9 |
| 223 |
|
| 224 |
/* Save FFR after predicates */ |
| 225 |
mov x13, #CTX_SIMD_FFR |
| 226 |
add x9, x0, x13 |
| 227 |
rdffr p0.b |
| 228 |
str p0, [x9] |
| 229 |
|
| 230 |
/* Save vector registers */ |
| 231 |
mov x13, #CTX_SIMD_VECTORS |
| 232 |
add x9, x0, x13 |
| 233 |
sve_vectors_op str, x9 |
| 234 |
|
| 235 |
/* Restore SVE enablement */ |
| 236 |
msr S3_6_C1_C2_0, x12 /* zcr_el3 */ |
| 237 |
msr cptr_el3, x10 |
| 238 |
isb |
| 239 |
.arch_extension nosve |
| 240 |
|
| 241 |
/* Save FPSR, FPCR and FPEXC32 */ |
| 242 |
fpregs_state_save x0, x9 |
| 243 |
|
| 244 |
ret |
| 245 |
endfunc sve_context_save |
| 246 |
|
| 247 |
/* ------------------------------------------------------------------ |
| 248 |
* The following function follows the aapcs_64 strictly to use x9-x17 |
| 249 |
* (temporary caller-saved registers according to AArch64 PCS) to |
| 250 |
* restore SVE register context. It assumes that 'x0' is pointing to |
| 251 |
* a 'sve_regs_t' structure from where the register context will be |
| 252 |
* restored. |
| 253 |
* ------------------------------------------------------------------ |
| 254 |
*/ |
| 255 |
func sve_context_restore |
| 256 |
.arch_extension sve |
| 257 |
/* Temporarily enable SVE for EL3 */ |
| 258 |
mrs x10, cptr_el3 |
| 259 |
orr x11, x10, #CPTR_EZ_BIT |
| 260 |
bic x11, x11, #TFP_BIT |
| 261 |
msr cptr_el3, x11 |
| 262 |
isb |
| 263 |
|
| 264 |
/* zcr_el3 */ |
| 265 |
mrs x12, S3_6_C1_C2_0 |
| 266 |
mov x13, #((SVE_VECTOR_LEN >> 7) - 1) |
| 267 |
msr S3_6_C1_C2_0, x13 |
| 268 |
isb |
| 269 |
|
| 270 |
/* Restore FFR register before predicates */ |
| 271 |
mov x13, #CTX_SIMD_FFR |
| 272 |
add x9, x0, x13 |
| 273 |
ldr p0, [x9] |
| 274 |
wrffr p0.b |
| 275 |
|
| 276 |
/* Restore predicate registers */ |
| 277 |
mov x13, #CTX_SIMD_PREDICATES |
| 278 |
add x9, x0, x13 |
| 279 |
sve_predicate_op ldr, x9 |
| 280 |
|
| 281 |
/* Restore vector registers */ |
| 282 |
mov x13, #CTX_SIMD_VECTORS |
| 283 |
add x9, x0, x13 |
| 284 |
sve_vectors_op ldr, x9 |
| 285 |
|
| 286 |
/* Restore SVE enablement */ |
| 287 |
msr S3_6_C1_C2_0, x12 /* zcr_el3 */ |
| 288 |
msr cptr_el3, x10 |
| 289 |
isb |
| 290 |
.arch_extension nosve |
| 291 |
|
| 292 |
/* Restore FPSR, FPCR and FPEXC32 */ |
| 293 |
fpregs_state_restore x0, x9 |
| 294 |
ret |
| 295 |
endfunc sve_context_restore |
| 296 |
#endif /* CTX_INCLUDE_SVE_REGS */ |
| 297 |
|
| 298 |
/* |
| 299 |
* Set SCR_EL3.EA bit to enable SErrors at EL3 |
| 300 |
*/ |
| 301 |
.macro enable_serror_at_el3 |
| 302 |
mrs x8, scr_el3 |
| 303 |
orr x8, x8, #SCR_EA_BIT |
| 304 |
msr scr_el3, x8 |
| 305 |
.endm |
| 306 |
|
| 307 |
/* |
| 308 |
* Set the PSTATE bits not set when the exception was taken as |
| 309 |
* described in the AArch64.TakeException() pseudocode function |
| 310 |
* in ARM DDI 0487F.c page J1-7635 to a default value. |
| 311 |
*/ |
| 312 |
.macro set_unset_pstate_bits |
| 313 |
/* |
| 314 |
* If Data Independent Timing (DIT) functionality is implemented, |
| 315 |
* always enable DIT in EL3 |
| 316 |
*/ |
| 317 |
#if ENABLE_FEAT_DIT |
| 318 |
#if ENABLE_FEAT_DIT >= 2 |
| 319 |
mrs x8, id_aa64pfr0_el1 |
| 320 |
and x8, x8, #(ID_AA64PFR0_DIT_MASK << ID_AA64PFR0_DIT_SHIFT) |
| 321 |
cbz x8, 1f |
| 322 |
#endif |
| 323 |
mov x8, #DIT_BIT |
| 324 |
msr DIT, x8 |
| 325 |
1: |
| 326 |
#endif /* ENABLE_FEAT_DIT */ |
| 327 |
.endm /* set_unset_pstate_bits */ |
| 328 |
|
| 329 |
/*------------------------------------------------------------------------- |
| 330 |
* This macro checks the ENABLE_FEAT_MPAM state, performs ID register |
| 331 |
* check to see if the platform supports MPAM extension and restores MPAM3 |
| 332 |
* register value if it is FEAT_STATE_ENABLED/FEAT_STATE_CHECKED. |
| 333 |
* |
| 334 |
* This is particularly more complicated because we can't check |
| 335 |
* if the platform supports MPAM by looking for status of a particular bit |
| 336 |
* in the MDCR_EL3 or CPTR_EL3 register like other extensions. |
| 337 |
* ------------------------------------------------------------------------ |
| 338 |
*/ |
| 339 |
|
| 340 |
.macro restore_mpam3_el3 |
| 341 |
#if ENABLE_FEAT_MPAM |
| 342 |
#if ENABLE_FEAT_MPAM >= 2 |
| 343 |
mrs x8, id_aa64pfr0_el1 |
| 344 |
lsr x8, x8, #(ID_AA64PFR0_MPAM_SHIFT) |
| 345 |
and x8, x8, #(ID_AA64PFR0_MPAM_MASK) |
| 346 |
mrs x7, id_aa64pfr1_el1 |
| 347 |
lsr x7, x7, #(ID_AA64PFR1_MPAM_FRAC_SHIFT) |
| 348 |
and x7, x7, #(ID_AA64PFR1_MPAM_FRAC_MASK) |
| 349 |
orr x7, x7, x8 |
| 350 |
cbz x7, no_mpam |
| 351 |
#endif |
| 352 |
/* ----------------------------------------------------------- |
| 353 |
* Restore MPAM3_EL3 register as per context state |
| 354 |
* Currently we only enable MPAM for NS world and trap to EL3 |
| 355 |
* for MPAM access in lower ELs of Secure and Realm world |
| 356 |
* x9 holds address of the per_world context |
| 357 |
* ----------------------------------------------------------- |
| 358 |
*/ |
| 359 |
|
| 360 |
ldr x17, [x9, #CTX_MPAM3_EL3] |
| 361 |
msr S3_6_C10_C5_0, x17 /* mpam3_el3 */ |
| 362 |
|
| 363 |
no_mpam: |
| 364 |
#endif |
| 365 |
.endm /* restore_mpam3_el3 */ |
| 366 |
|
| 367 |
/* ------------------------------------------------------------------ |
| 368 |
* The following macro is used to save and restore all the general |
| 369 |
* purpose and ARMv8.3-PAuth (if enabled) registers. |
| 370 |
* It also checks if the Secure Cycle Counter (PMCCNTR_EL0) |
| 371 |
* is disabled in EL3/Secure (ARMv8.5-PMU), wherein PMCCNTR_EL0 |
| 372 |
* needs not to be saved/restored during world switch. |
| 373 |
* |
| 374 |
* Ideally we would only save and restore the callee saved registers |
| 375 |
* when a world switch occurs but that type of implementation is more |
| 376 |
* complex. So currently we will always save and restore these |
| 377 |
* registers on entry and exit of EL3. |
| 378 |
* clobbers: x18 |
| 379 |
* ------------------------------------------------------------------ |
| 380 |
*/ |
| 381 |
.macro save_gp_pmcr_pauth_regs |
| 382 |
stp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] |
| 383 |
stp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] |
| 384 |
stp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] |
| 385 |
stp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] |
| 386 |
stp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] |
| 387 |
stp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10] |
| 388 |
stp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12] |
| 389 |
stp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14] |
| 390 |
stp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16] |
| 391 |
stp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18] |
| 392 |
stp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20] |
| 393 |
stp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22] |
| 394 |
stp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24] |
| 395 |
stp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26] |
| 396 |
stp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28] |
| 397 |
mrs x18, sp_el0 |
| 398 |
str x18, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_SP_EL0] |
| 399 |
|
| 400 |
/* PMUv3 is presumed to be always present */ |
| 401 |
mrs x9, pmcr_el0 |
| 402 |
str x9, [sp, #CTX_EL3STATE_OFFSET + CTX_PMCR_EL0] |
| 403 |
isb |
| 404 |
#if CTX_INCLUDE_PAUTH_REGS |
| 405 |
/* ---------------------------------------------------------- |
| 406 |
* Save the ARMv8.3-PAuth keys as they are not banked |
| 407 |
* by exception level |
| 408 |
* ---------------------------------------------------------- |
| 409 |
*/ |
| 410 |
add x19, sp, #CTX_PAUTH_REGS_OFFSET |
| 411 |
|
| 412 |
mrs x20, APIAKeyLo_EL1 /* x21:x20 = APIAKey */ |
| 413 |
mrs x21, APIAKeyHi_EL1 |
| 414 |
mrs x22, APIBKeyLo_EL1 /* x23:x22 = APIBKey */ |
| 415 |
mrs x23, APIBKeyHi_EL1 |
| 416 |
mrs x24, APDAKeyLo_EL1 /* x25:x24 = APDAKey */ |
| 417 |
mrs x25, APDAKeyHi_EL1 |
| 418 |
mrs x26, APDBKeyLo_EL1 /* x27:x26 = APDBKey */ |
| 419 |
mrs x27, APDBKeyHi_EL1 |
| 420 |
mrs x28, APGAKeyLo_EL1 /* x29:x28 = APGAKey */ |
| 421 |
mrs x29, APGAKeyHi_EL1 |
| 422 |
|
| 423 |
stp x20, x21, [x19, #CTX_PACIAKEY_LO] |
| 424 |
stp x22, x23, [x19, #CTX_PACIBKEY_LO] |
| 425 |
stp x24, x25, [x19, #CTX_PACDAKEY_LO] |
| 426 |
stp x26, x27, [x19, #CTX_PACDBKEY_LO] |
| 427 |
stp x28, x29, [x19, #CTX_PACGAKEY_LO] |
| 428 |
#endif /* CTX_INCLUDE_PAUTH_REGS */ |
| 429 |
.endm /* save_gp_pmcr_pauth_regs */ |
| 430 |
|
| 431 |
/* ----------------------------------------------------------------- |
| 432 |
* This function saves the context and sets the PSTATE to a known |
| 433 |
* state, preparing entry to el3. |
| 434 |
* Save all the general purpose and ARMv8.3-PAuth (if enabled) |
| 435 |
* registers. |
| 436 |
* Then set any of the PSTATE bits that are not set by hardware |
| 437 |
* according to the Aarch64.TakeException pseudocode in the Arm |
| 438 |
* Architecture Reference Manual to a default value for EL3. |
| 439 |
* clobbers: x17 |
| 440 |
* ----------------------------------------------------------------- |
| 441 |
*/ |
| 442 |
func prepare_el3_entry |
| 443 |
save_gp_pmcr_pauth_regs |
| 444 |
setup_el3_execution_context |
| 445 |
ret |
| 446 |
endfunc prepare_el3_entry |
| 447 |
|
| 448 |
/* ------------------------------------------------------------------ |
| 449 |
* This function restores ARMv8.3-PAuth (if enabled) and all general |
| 450 |
* purpose registers except x30 from the CPU context. |
| 451 |
* x30 register must be explicitly restored by the caller. |
| 452 |
* ------------------------------------------------------------------ |
| 453 |
*/ |
| 454 |
func restore_gp_pmcr_pauth_regs |
| 455 |
#if CTX_INCLUDE_PAUTH_REGS |
| 456 |
/* Restore the ARMv8.3 PAuth keys */ |
| 457 |
add x10, sp, #CTX_PAUTH_REGS_OFFSET |
| 458 |
|
| 459 |
ldp x0, x1, [x10, #CTX_PACIAKEY_LO] /* x1:x0 = APIAKey */ |
| 460 |
ldp x2, x3, [x10, #CTX_PACIBKEY_LO] /* x3:x2 = APIBKey */ |
| 461 |
ldp x4, x5, [x10, #CTX_PACDAKEY_LO] /* x5:x4 = APDAKey */ |
| 462 |
ldp x6, x7, [x10, #CTX_PACDBKEY_LO] /* x7:x6 = APDBKey */ |
| 463 |
ldp x8, x9, [x10, #CTX_PACGAKEY_LO] /* x9:x8 = APGAKey */ |
| 464 |
|
| 465 |
msr APIAKeyLo_EL1, x0 |
| 466 |
msr APIAKeyHi_EL1, x1 |
| 467 |
msr APIBKeyLo_EL1, x2 |
| 468 |
msr APIBKeyHi_EL1, x3 |
| 469 |
msr APDAKeyLo_EL1, x4 |
| 470 |
msr APDAKeyHi_EL1, x5 |
| 471 |
msr APDBKeyLo_EL1, x6 |
| 472 |
msr APDBKeyHi_EL1, x7 |
| 473 |
msr APGAKeyLo_EL1, x8 |
| 474 |
msr APGAKeyHi_EL1, x9 |
| 475 |
#endif /* CTX_INCLUDE_PAUTH_REGS */ |
| 476 |
|
| 477 |
/* PMUv3 is presumed to be always present */ |
| 478 |
ldr x0, [sp, #CTX_EL3STATE_OFFSET + CTX_PMCR_EL0] |
| 479 |
msr pmcr_el0, x0 |
| 480 |
ldp x0, x1, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X0] |
| 481 |
ldp x2, x3, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X2] |
| 482 |
ldp x4, x5, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X4] |
| 483 |
ldp x6, x7, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X6] |
| 484 |
ldp x8, x9, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X8] |
| 485 |
ldp x10, x11, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X10] |
| 486 |
ldp x12, x13, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X12] |
| 487 |
ldp x14, x15, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X14] |
| 488 |
ldp x16, x17, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X16] |
| 489 |
ldp x18, x19, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X18] |
| 490 |
ldp x20, x21, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X20] |
| 491 |
ldp x22, x23, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X22] |
| 492 |
ldp x24, x25, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X24] |
| 493 |
ldp x26, x27, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X26] |
| 494 |
ldr x28, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_SP_EL0] |
| 495 |
msr sp_el0, x28 |
| 496 |
ldp x28, x29, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_X28] |
| 497 |
ret |
| 498 |
endfunc restore_gp_pmcr_pauth_regs |
| 499 |
|
| 500 |
#if ERRATA_SPECULATIVE_AT |
| 501 |
/* -------------------------------------------------------------------- |
| 502 |
* In case of ERRATA_SPECULATIVE_AT, save SCTLR_EL1 and TCR_EL1 |
| 503 |
* registers and update EL1 registers to disable stage1 and stage2 |
| 504 |
* page table walk. |
| 505 |
* -------------------------------------------------------------------- |
| 506 |
*/ |
| 507 |
func save_and_update_ptw_el1_sys_regs |
| 508 |
/* ---------------------------------------------------------- |
| 509 |
* Save only sctlr_el1 and tcr_el1 registers |
| 510 |
* ---------------------------------------------------------- |
| 511 |
*/ |
| 512 |
mrs x29, sctlr_el1 |
| 513 |
str x29, [sp, #(CTX_ERRATA_SPEC_AT_OFFSET + CTX_ERRATA_SPEC_AT_SCTLR_EL1)] |
| 514 |
mrs x29, tcr_el1 |
| 515 |
str x29, [sp, #(CTX_ERRATA_SPEC_AT_OFFSET + CTX_ERRATA_SPEC_AT_TCR_EL1)] |
| 516 |
|
| 517 |
/* ------------------------------------------------------------ |
| 518 |
* Must follow below order in order to disable page table |
| 519 |
* walk for lower ELs (EL1 and EL0). First step ensures that |
| 520 |
* page table walk is disabled for stage1 and second step |
| 521 |
* ensures that page table walker should use TCR_EL1.EPDx |
| 522 |
* bits to perform address translation. ISB ensures that CPU |
| 523 |
* does these 2 steps in order. |
| 524 |
* |
| 525 |
* 1. Update TCR_EL1.EPDx bits to disable page table walk by |
| 526 |
* stage1. |
| 527 |
* 2. Enable MMU bit to avoid identity mapping via stage2 |
| 528 |
* and force TCR_EL1.EPDx to be used by the page table |
| 529 |
* walker. |
| 530 |
* ------------------------------------------------------------ |
| 531 |
*/ |
| 532 |
orr x29, x29, #(TCR_EPD0_BIT) |
| 533 |
orr x29, x29, #(TCR_EPD1_BIT) |
| 534 |
msr tcr_el1, x29 |
| 535 |
isb |
| 536 |
mrs x29, sctlr_el1 |
| 537 |
orr x29, x29, #SCTLR_M_BIT |
| 538 |
msr sctlr_el1, x29 |
| 539 |
isb |
| 540 |
ret |
| 541 |
endfunc save_and_update_ptw_el1_sys_regs |
| 542 |
|
| 543 |
#endif /* ERRATA_SPECULATIVE_AT */ |
| 544 |
|
| 545 |
/* ----------------------------------------------------------------- |
| 546 |
* The below macro returns the address of the per_world context for |
| 547 |
* the security state, retrieved through "get_security_state" macro. |
| 548 |
* The per_world context address is returned in the register argument. |
| 549 |
* Clobbers: x9, x10 |
| 550 |
* ------------------------------------------------------------------ |
| 551 |
*/ |
| 552 |
|
| 553 |
.macro get_per_world_context _reg:req |
| 554 |
ldr x10, [sp, #CTX_EL3STATE_OFFSET + CTX_SCR_EL3] |
| 555 |
get_security_state x9, x10 |
| 556 |
mov_imm x10, (CTX_PERWORLD_EL3STATE_END - CTX_CPTR_EL3) |
| 557 |
mul x9, x9, x10 |
| 558 |
adrp x10, per_world_context |
| 559 |
add x10, x10, :lo12:per_world_context |
| 560 |
add x9, x9, x10 |
| 561 |
mov \_reg, x9 |
| 562 |
.endm |
| 563 |
|
| 564 |
/* ------------------------------------------------------------------ |
| 565 |
* This routine assumes that the SP_EL3 is pointing to a valid |
| 566 |
* context structure from where the gp regs and other special |
| 567 |
* registers can be retrieved. |
| 568 |
* ------------------------------------------------------------------ |
| 569 |
*/ |
| 570 |
func el3_exit |
| 571 |
#if ENABLE_ASSERTIONS |
| 572 |
/* el3_exit assumes SP_EL0 on entry */ |
| 573 |
mrs x17, spsel |
| 574 |
cmp x17, #MODE_SP_EL0 |
| 575 |
ASM_ASSERT(eq) |
| 576 |
#endif /* ENABLE_ASSERTIONS */ |
| 577 |
|
| 578 |
/* ---------------------------------------------------------- |
| 579 |
* Save the current SP_EL0 i.e. the EL3 runtime stack which |
| 580 |
* will be used for handling the next SMC. |
| 581 |
* Then switch to SP_EL3. |
| 582 |
* ---------------------------------------------------------- |
| 583 |
*/ |
| 584 |
mov x17, sp |
| 585 |
msr spsel, #MODE_SP_ELX |
| 586 |
str x17, [sp, #CTX_EL3STATE_OFFSET + CTX_RUNTIME_SP] |
| 587 |
|
| 588 |
/* ---------------------------------------------------------- |
| 589 |
* Restore CPTR_EL3. |
| 590 |
* ZCR is only restored if SVE is supported and enabled. |
| 591 |
* Synchronization is required before zcr_el3 is addressed. |
| 592 |
* ---------------------------------------------------------- |
| 593 |
*/ |
| 594 |
|
| 595 |
/* The address of the per_world context is stored in x9 */ |
| 596 |
get_per_world_context x9 |
| 597 |
|
| 598 |
ldp x19, x20, [x9, #CTX_CPTR_EL3] |
| 599 |
msr cptr_el3, x19 |
| 600 |
|
| 601 |
#if IMAGE_BL31 |
| 602 |
ands x19, x19, #CPTR_EZ_BIT |
| 603 |
beq sve_not_enabled |
| 604 |
|
| 605 |
isb |
| 606 |
msr S3_6_C1_C2_0, x20 /* zcr_el3 */ |
| 607 |
sve_not_enabled: |
| 608 |
|
| 609 |
restore_mpam3_el3 |
| 610 |
|
| 611 |
#endif /* IMAGE_BL31 */ |
| 612 |
|
| 613 |
#if IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639 |
| 614 |
/* ---------------------------------------------------------- |
| 615 |
* Restore mitigation state as it was on entry to EL3 |
| 616 |
* ---------------------------------------------------------- |
| 617 |
*/ |
| 618 |
ldr x17, [sp, #CTX_CVE_2018_3639_OFFSET + CTX_CVE_2018_3639_DISABLE] |
| 619 |
cbz x17, 1f |
| 620 |
blr x17 |
| 621 |
1: |
| 622 |
#endif /* IMAGE_BL31 && DYNAMIC_WORKAROUND_CVE_2018_3639 */ |
| 623 |
|
| 624 |
#if IMAGE_BL31 |
| 625 |
synchronize_errors |
| 626 |
#endif /* IMAGE_BL31 */ |
| 627 |
|
| 628 |
/* -------------------------------------------------------------- |
| 629 |
* Restore MDCR_EL3, SPSR_EL3, ELR_EL3 and SCR_EL3 prior to ERET |
| 630 |
* -------------------------------------------------------------- |
| 631 |
*/ |
| 632 |
ldp x16, x17, [sp, #CTX_EL3STATE_OFFSET + CTX_SPSR_EL3] |
| 633 |
ldr x18, [sp, #CTX_EL3STATE_OFFSET + CTX_SCR_EL3] |
| 634 |
ldr x19, [sp, #CTX_EL3STATE_OFFSET + CTX_MDCR_EL3] |
| 635 |
msr spsr_el3, x16 |
| 636 |
msr elr_el3, x17 |
| 637 |
msr scr_el3, x18 |
| 638 |
msr mdcr_el3, x19 |
| 639 |
|
| 640 |
restore_ptw_el1_sys_regs |
| 641 |
|
| 642 |
/* ---------------------------------------------------------- |
| 643 |
* Restore general purpose (including x30), PMCR_EL0 and |
| 644 |
* ARMv8.3-PAuth registers. |
| 645 |
* Exit EL3 via ERET to a lower exception level. |
| 646 |
* ---------------------------------------------------------- |
| 647 |
*/ |
| 648 |
bl restore_gp_pmcr_pauth_regs |
| 649 |
ldr x30, [sp, #CTX_GPREGS_OFFSET + CTX_GPREG_LR] |
| 650 |
|
| 651 |
#ifdef IMAGE_BL31 |
| 652 |
/* Clear the EL3 flag as we are exiting el3 */ |
| 653 |
str xzr, [sp, #CTX_EL3STATE_OFFSET + CTX_NESTED_EA_FLAG] |
| 654 |
#endif /* IMAGE_BL31 */ |
| 655 |
|
| 656 |
exception_return |
| 657 |
|
| 658 |
endfunc el3_exit |