| 1 |
/* |
| 2 |
* Copyright (c) 2016-2018, Arm Limited and Contributors. All rights reserved. |
| 3 |
* |
| 4 |
* SPDX-License-Identifier: BSD-3-Clause |
| 5 |
*/ |
| 6 |
|
| 7 |
#include <assert.h> |
| 8 |
#include <stdbool.h> |
| 9 |
#include <stdint.h> |
| 10 |
#include <string.h> |
| 11 |
|
| 12 |
#include <platform_def.h> |
| 13 |
|
| 14 |
#include <arch.h> |
| 15 |
#include <arch_helpers.h> |
| 16 |
#include <common/debug.h> |
| 17 |
#include <lib/cassert.h> |
| 18 |
#include <lib/utils.h> |
| 19 |
#include <lib/xlat_tables/xlat_tables.h> |
| 20 |
#include <plat/common/common_def.h> |
| 21 |
|
| 22 |
#include "xlat_tables_private.h" |
| 23 |
|
| 24 |
#if LOG_LEVEL >= LOG_LEVEL_VERBOSE |
| 25 |
#define LVL0_SPACER "" |
| 26 |
#define LVL1_SPACER " " |
| 27 |
#define LVL2_SPACER " " |
| 28 |
#define LVL3_SPACER " " |
| 29 |
#define get_level_spacer(level) \ |
| 30 |
(((level) == U(0)) ? LVL0_SPACER : \ |
| 31 |
(((level) == U(1)) ? LVL1_SPACER : \ |
| 32 |
(((level) == U(2)) ? LVL2_SPACER : LVL3_SPACER))) |
| 33 |
#define debug_print(...) printf(__VA_ARGS__) |
| 34 |
#else |
| 35 |
#define debug_print(...) ((void)0) |
| 36 |
#endif |
| 37 |
|
| 38 |
#define UNSET_DESC ~0ULL |
| 39 |
#define MT_UNKNOWN ~0U |
| 40 |
|
| 41 |
static uint64_t xlat_tables[MAX_XLAT_TABLES][XLAT_TABLE_ENTRIES] |
| 42 |
__aligned(XLAT_TABLE_SIZE) __section(".xlat_table"); |
| 43 |
|
| 44 |
static unsigned int next_xlat; |
| 45 |
static unsigned long long xlat_max_pa; |
| 46 |
static uintptr_t xlat_max_va; |
| 47 |
|
| 48 |
static uint64_t execute_never_mask; |
| 49 |
static uint64_t ap1_mask; |
| 50 |
|
| 51 |
/* |
| 52 |
* Array of all memory regions stored in order of ascending base address. |
| 53 |
* The list is terminated by the first entry with size == 0. |
| 54 |
*/ |
| 55 |
static mmap_region_t mmap[MAX_MMAP_REGIONS + 1]; |
| 56 |
|
| 57 |
|
| 58 |
void print_mmap(void) |
| 59 |
{ |
| 60 |
#if LOG_LEVEL >= LOG_LEVEL_VERBOSE |
| 61 |
debug_print("mmap:\n"); |
| 62 |
mmap_region_t *mm = mmap; |
| 63 |
while (mm->size != 0U) { |
| 64 |
debug_print(" VA:%p PA:0x%llx size:0x%zx attr:0x%x\n", |
| 65 |
(void *)mm->base_va, mm->base_pa, |
| 66 |
mm->size, mm->attr); |
| 67 |
++mm; |
| 68 |
}; |
| 69 |
debug_print("\n"); |
| 70 |
#endif |
| 71 |
} |
| 72 |
|
| 73 |
void mmap_add_region(unsigned long long base_pa, uintptr_t base_va, |
| 74 |
size_t size, unsigned int attr) |
| 75 |
{ |
| 76 |
mmap_region_t *mm = mmap; |
| 77 |
const mmap_region_t *mm_last = mm + ARRAY_SIZE(mmap) - 1U; |
| 78 |
unsigned long long end_pa = base_pa + size - 1U; |
| 79 |
uintptr_t end_va = base_va + size - 1U; |
| 80 |
|
| 81 |
assert(IS_PAGE_ALIGNED(base_pa)); |
| 82 |
assert(IS_PAGE_ALIGNED(base_va)); |
| 83 |
assert(IS_PAGE_ALIGNED(size)); |
| 84 |
|
| 85 |
if (size == 0U) |
| 86 |
return; |
| 87 |
|
| 88 |
assert(base_pa < end_pa); /* Check for overflows */ |
| 89 |
assert(base_va < end_va); |
| 90 |
|
| 91 |
assert((base_va + (uintptr_t)size - (uintptr_t)1) <= |
| 92 |
(PLAT_VIRT_ADDR_SPACE_SIZE - 1U)); |
| 93 |
assert((base_pa + (unsigned long long)size - 1ULL) <= |
| 94 |
(PLAT_PHY_ADDR_SPACE_SIZE - 1U)); |
| 95 |
|
| 96 |
#if ENABLE_ASSERTIONS |
| 97 |
|
| 98 |
/* Check for PAs and VAs overlaps with all other regions */ |
| 99 |
for (mm = mmap; mm->size; ++mm) { |
| 100 |
|
| 101 |
uintptr_t mm_end_va = mm->base_va + mm->size - 1U; |
| 102 |
|
| 103 |
/* |
| 104 |
* Check if one of the regions is completely inside the other |
| 105 |
* one. |
| 106 |
*/ |
| 107 |
bool fully_overlapped_va = |
| 108 |
((base_va >= mm->base_va) && (end_va <= mm_end_va)) || |
| 109 |
((mm->base_va >= base_va) && (mm_end_va <= end_va)); |
| 110 |
|
| 111 |
/* |
| 112 |
* Full VA overlaps are only allowed if both regions are |
| 113 |
* identity mapped (zero offset) or have the same VA to PA |
| 114 |
* offset. Also, make sure that it's not the exact same area. |
| 115 |
*/ |
| 116 |
if (fully_overlapped_va) { |
| 117 |
assert((mm->base_va - mm->base_pa) == |
| 118 |
(base_va - base_pa)); |
| 119 |
assert((base_va != mm->base_va) || (size != mm->size)); |
| 120 |
} else { |
| 121 |
/* |
| 122 |
* If the regions do not have fully overlapping VAs, |
| 123 |
* then they must have fully separated VAs and PAs. |
| 124 |
* Partial overlaps are not allowed |
| 125 |
*/ |
| 126 |
|
| 127 |
unsigned long long mm_end_pa = |
| 128 |
mm->base_pa + mm->size - 1; |
| 129 |
|
| 130 |
bool separated_pa = (end_pa < mm->base_pa) || |
| 131 |
(base_pa > mm_end_pa); |
| 132 |
bool separated_va = (end_va < mm->base_va) || |
| 133 |
(base_va > mm_end_va); |
| 134 |
|
| 135 |
assert(separated_va && separated_pa); |
| 136 |
} |
| 137 |
} |
| 138 |
|
| 139 |
mm = mmap; /* Restore pointer to the start of the array */ |
| 140 |
|
| 141 |
#endif /* ENABLE_ASSERTIONS */ |
| 142 |
|
| 143 |
/* Find correct place in mmap to insert new region */ |
| 144 |
while ((mm->base_va < base_va) && (mm->size != 0U)) |
| 145 |
++mm; |
| 146 |
|
| 147 |
/* |
| 148 |
* If a section is contained inside another one with the same base |
| 149 |
* address, it must be placed after the one it is contained in: |
| 150 |
* |
| 151 |
* 1st |-----------------------| |
| 152 |
* 2nd |------------| |
| 153 |
* 3rd |------| |
| 154 |
* |
| 155 |
* This is required for mmap_region_attr() to get the attributes of the |
| 156 |
* small region correctly. |
| 157 |
*/ |
| 158 |
while ((mm->base_va == base_va) && (mm->size > size)) |
| 159 |
++mm; |
| 160 |
|
| 161 |
/* Make room for new region by moving other regions up by one place */ |
| 162 |
(void)memmove(mm + 1, mm, (uintptr_t)mm_last - (uintptr_t)mm); |
| 163 |
|
| 164 |
/* Check we haven't lost the empty sentinel from the end of the array */ |
| 165 |
assert(mm_last->size == 0U); |
| 166 |
|
| 167 |
mm->base_pa = base_pa; |
| 168 |
mm->base_va = base_va; |
| 169 |
mm->size = size; |
| 170 |
mm->attr = attr; |
| 171 |
|
| 172 |
if (end_pa > xlat_max_pa) |
| 173 |
xlat_max_pa = end_pa; |
| 174 |
if (end_va > xlat_max_va) |
| 175 |
xlat_max_va = end_va; |
| 176 |
} |
| 177 |
|
| 178 |
void mmap_add(const mmap_region_t *mm) |
| 179 |
{ |
| 180 |
const mmap_region_t *mm_cursor = mm; |
| 181 |
|
| 182 |
while ((mm_cursor->size != 0U) || (mm_cursor->attr != 0U)) { |
| 183 |
mmap_add_region(mm_cursor->base_pa, mm_cursor->base_va, |
| 184 |
mm_cursor->size, mm_cursor->attr); |
| 185 |
mm_cursor++; |
| 186 |
} |
| 187 |
} |
| 188 |
|
| 189 |
static uint64_t mmap_desc(unsigned int attr, unsigned long long addr_pa, |
| 190 |
unsigned int level) |
| 191 |
{ |
| 192 |
uint64_t desc; |
| 193 |
int mem_type; |
| 194 |
|
| 195 |
/* Make sure that the granularity is fine enough to map this address. */ |
| 196 |
assert((addr_pa & XLAT_BLOCK_MASK(level)) == 0U); |
| 197 |
|
| 198 |
desc = addr_pa; |
| 199 |
/* |
| 200 |
* There are different translation table descriptors for level 3 and the |
| 201 |
* rest. |
| 202 |
*/ |
| 203 |
desc |= (level == XLAT_TABLE_LEVEL_MAX) ? PAGE_DESC : BLOCK_DESC; |
| 204 |
desc |= ((attr & MT_NS) != 0U) ? LOWER_ATTRS(NS) : 0U; |
| 205 |
desc |= ((attr & MT_RW) != 0U) ? LOWER_ATTRS(AP_RW) : LOWER_ATTRS(AP_RO); |
| 206 |
/* |
| 207 |
* Always set the access flag, as this library assumes access flag |
| 208 |
* faults aren't managed. |
| 209 |
*/ |
| 210 |
desc |= LOWER_ATTRS(ACCESS_FLAG); |
| 211 |
desc |= ap1_mask; |
| 212 |
|
| 213 |
/* |
| 214 |
* Deduce shareability domain and executability of the memory region |
| 215 |
* from the memory type. |
| 216 |
* |
| 217 |
* Data accesses to device memory and non-cacheable normal memory are |
| 218 |
* coherent for all observers in the system, and correspondingly are |
| 219 |
* always treated as being Outer Shareable. Therefore, for these 2 types |
| 220 |
* of memory, it is not strictly needed to set the shareability field |
| 221 |
* in the translation tables. |
| 222 |
*/ |
| 223 |
mem_type = MT_TYPE(attr); |
| 224 |
if (mem_type == MT_DEVICE) { |
| 225 |
desc |= LOWER_ATTRS(ATTR_DEVICE_INDEX | OSH); |
| 226 |
/* |
| 227 |
* Always map device memory as execute-never. |
| 228 |
* This is to avoid the possibility of a speculative instruction |
| 229 |
* fetch, which could be an issue if this memory region |
| 230 |
* corresponds to a read-sensitive peripheral. |
| 231 |
*/ |
| 232 |
desc |= execute_never_mask; |
| 233 |
|
| 234 |
} else { /* Normal memory */ |
| 235 |
/* |
| 236 |
* Always map read-write normal memory as execute-never. |
| 237 |
* This library assumes that it is used by software that does |
| 238 |
* not self-modify its code, therefore R/W memory is reserved |
| 239 |
* for data storage, which must not be executable. |
| 240 |
* |
| 241 |
* Note that setting the XN bit here is for consistency only. |
| 242 |
* The function that enables the MMU sets the SCTLR_ELx.WXN bit, |
| 243 |
* which makes any writable memory region to be treated as |
| 244 |
* execute-never, regardless of the value of the XN bit in the |
| 245 |
* translation table. |
| 246 |
* |
| 247 |
* For read-only memory, rely on the MT_EXECUTE/MT_EXECUTE_NEVER |
| 248 |
* attribute to figure out the value of the XN bit. |
| 249 |
*/ |
| 250 |
if (((attr & MT_RW) != 0U) || ((attr & MT_EXECUTE_NEVER) != 0U)) { |
| 251 |
desc |= execute_never_mask; |
| 252 |
} |
| 253 |
|
| 254 |
if (mem_type == MT_MEMORY) { |
| 255 |
desc |= LOWER_ATTRS(ATTR_IWBWA_OWBWA_NTR_INDEX | ISH); |
| 256 |
} else { |
| 257 |
assert(mem_type == MT_NON_CACHEABLE); |
| 258 |
desc |= LOWER_ATTRS(ATTR_NON_CACHEABLE_INDEX | OSH); |
| 259 |
} |
| 260 |
} |
| 261 |
|
| 262 |
debug_print((mem_type == MT_MEMORY) ? "MEM" : |
| 263 |
((mem_type == MT_NON_CACHEABLE) ? "NC" : "DEV")); |
| 264 |
debug_print(((attr & MT_RW) != 0U) ? "-RW" : "-RO"); |
| 265 |
debug_print(((attr & MT_NS) != 0U) ? "-NS" : "-S"); |
| 266 |
debug_print(((attr & MT_EXECUTE_NEVER) != 0U) ? "-XN" : "-EXEC"); |
| 267 |
return desc; |
| 268 |
} |
| 269 |
|
| 270 |
/* |
| 271 |
* Look for the innermost region that contains the area at `base_va` with size |
| 272 |
* `size`. Populate *attr with the attributes of this region. |
| 273 |
* |
| 274 |
* On success, this function returns 0. |
| 275 |
* If there are partial overlaps (meaning that a smaller size is needed) or if |
| 276 |
* the region can't be found in the given area, it returns MT_UNKNOWN. In this |
| 277 |
* case the value pointed by attr should be ignored by the caller. |
| 278 |
*/ |
| 279 |
static unsigned int mmap_region_attr(const mmap_region_t *mm, uintptr_t base_va, |
| 280 |
size_t size, unsigned int *attr) |
| 281 |
{ |
| 282 |
/* Don't assume that the area is contained in the first region */ |
| 283 |
unsigned int ret = MT_UNKNOWN; |
| 284 |
|
| 285 |
/* |
| 286 |
* Get attributes from last (innermost) region that contains the |
| 287 |
* requested area. Don't stop as soon as one region doesn't contain it |
| 288 |
* because there may be other internal regions that contain this area: |
| 289 |
* |
| 290 |
* |-----------------------------1-----------------------------| |
| 291 |
* |----2----| |-------3-------| |----5----| |
| 292 |
* |--4--| |
| 293 |
* |
| 294 |
* |---| <- Area we want the attributes of. |
| 295 |
* |
| 296 |
* In this example, the area is contained in regions 1, 3 and 4 but not |
| 297 |
* in region 2. The loop shouldn't stop at region 2 as inner regions |
| 298 |
* have priority over outer regions, it should stop at region 5. |
| 299 |
*/ |
| 300 |
for ( ; ; ++mm) { |
| 301 |
|
| 302 |
if (mm->size == 0U) |
| 303 |
return ret; /* Reached end of list */ |
| 304 |
|
| 305 |
if (mm->base_va > (base_va + size - 1U)) |
| 306 |
return ret; /* Next region is after area so end */ |
| 307 |
|
| 308 |
if ((mm->base_va + mm->size - 1U) < base_va) |
| 309 |
continue; /* Next region has already been overtaken */ |
| 310 |
|
| 311 |
if ((ret == 0U) && (mm->attr == *attr)) |
| 312 |
continue; /* Region doesn't override attribs so skip */ |
| 313 |
|
| 314 |
if ((mm->base_va > base_va) || |
| 315 |
((mm->base_va + mm->size - 1U) < (base_va + size - 1U))) |
| 316 |
return MT_UNKNOWN; /* Region doesn't fully cover area */ |
| 317 |
|
| 318 |
*attr = mm->attr; |
| 319 |
ret = 0U; |
| 320 |
} |
| 321 |
return ret; |
| 322 |
} |
| 323 |
|
| 324 |
static mmap_region_t *init_xlation_table_inner(mmap_region_t *mm, |
| 325 |
uintptr_t base_va, |
| 326 |
uint64_t *table, |
| 327 |
unsigned int level) |
| 328 |
{ |
| 329 |
assert((level >= XLAT_TABLE_LEVEL_MIN) && |
| 330 |
(level <= XLAT_TABLE_LEVEL_MAX)); |
| 331 |
|
| 332 |
unsigned int level_size_shift = |
| 333 |
L0_XLAT_ADDRESS_SHIFT - level * XLAT_TABLE_ENTRIES_SHIFT; |
| 334 |
u_register_t level_size = (u_register_t)1 << level_size_shift; |
| 335 |
u_register_t level_index_mask = |
| 336 |
((u_register_t)XLAT_TABLE_ENTRIES_MASK) << level_size_shift; |
| 337 |
|
| 338 |
debug_print("New xlat table:\n"); |
| 339 |
|
| 340 |
do { |
| 341 |
uint64_t desc = UNSET_DESC; |
| 342 |
|
| 343 |
if (mm->size == 0U) { |
| 344 |
/* Done mapping regions; finish zeroing the table */ |
| 345 |
desc = INVALID_DESC; |
| 346 |
} else if ((mm->base_va + mm->size - 1U) < base_va) { |
| 347 |
/* This area is after the region so get next region */ |
| 348 |
++mm; |
| 349 |
continue; |
| 350 |
} |
| 351 |
|
| 352 |
debug_print("%s VA:%p size:0x%llx ", get_level_spacer(level), |
| 353 |
(void *)base_va, (unsigned long long)level_size); |
| 354 |
|
| 355 |
if (mm->base_va > (base_va + level_size - 1U)) { |
| 356 |
/* Next region is after this area. Nothing to map yet */ |
| 357 |
desc = INVALID_DESC; |
| 358 |
/* Make sure that the current level allows block descriptors */ |
| 359 |
} else if (level >= XLAT_BLOCK_LEVEL_MIN) { |
| 360 |
/* |
| 361 |
* Try to get attributes of this area. It will fail if |
| 362 |
* there are partially overlapping regions. On success, |
| 363 |
* it will return the innermost region's attributes. |
| 364 |
*/ |
| 365 |
unsigned int attr; |
| 366 |
unsigned int r = mmap_region_attr(mm, base_va, |
| 367 |
level_size, &attr); |
| 368 |
|
| 369 |
if (r == 0U) { |
| 370 |
desc = mmap_desc(attr, |
| 371 |
base_va - mm->base_va + mm->base_pa, |
| 372 |
level); |
| 373 |
} |
| 374 |
} |
| 375 |
|
| 376 |
if (desc == UNSET_DESC) { |
| 377 |
/* Area not covered by a region so need finer table */ |
| 378 |
uint64_t *new_table = xlat_tables[next_xlat]; |
| 379 |
|
| 380 |
next_xlat++; |
| 381 |
assert(next_xlat <= MAX_XLAT_TABLES); |
| 382 |
desc = TABLE_DESC | (uintptr_t)new_table; |
| 383 |
|
| 384 |
/* Recurse to fill in new table */ |
| 385 |
mm = init_xlation_table_inner(mm, base_va, |
| 386 |
new_table, level + 1U); |
| 387 |
} |
| 388 |
|
| 389 |
debug_print("\n"); |
| 390 |
|
| 391 |
*table++ = desc; |
| 392 |
base_va += level_size; |
| 393 |
} while ((base_va & level_index_mask) && |
| 394 |
((base_va - 1U) < (PLAT_VIRT_ADDR_SPACE_SIZE - 1U))); |
| 395 |
|
| 396 |
return mm; |
| 397 |
} |
| 398 |
|
| 399 |
void init_xlation_table(uintptr_t base_va, uint64_t *table, |
| 400 |
unsigned int level, uintptr_t *max_va, |
| 401 |
unsigned long long *max_pa) |
| 402 |
{ |
| 403 |
unsigned int el = xlat_arch_current_el(); |
| 404 |
|
| 405 |
execute_never_mask = xlat_arch_get_xn_desc(el); |
| 406 |
|
| 407 |
if (el == 3U) { |
| 408 |
ap1_mask = LOWER_ATTRS(AP_ONE_VA_RANGE_RES1); |
| 409 |
} else { |
| 410 |
assert(el == 1U); |
| 411 |
ap1_mask = 0ULL; |
| 412 |
} |
| 413 |
|
| 414 |
init_xlation_table_inner(mmap, base_va, table, level); |
| 415 |
*max_va = xlat_max_va; |
| 416 |
*max_pa = xlat_max_pa; |
| 417 |
} |