| 1 |
/* |
| 2 |
* This file is subject to the terms and conditions of the GNU General Public |
| 3 |
* License. See the file "COPYING" in the main directory of this archive |
| 4 |
* for more details. |
| 5 |
* |
| 6 |
* Copyright (C) 1994, 95, 96, 97, 98, 99, 2000, 2003 Ralf Baechle |
| 7 |
* Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc. |
| 8 |
*/ |
| 9 |
#ifndef _ASM_PGTABLE_64_H |
| 10 |
#define _ASM_PGTABLE_64_H |
| 11 |
|
| 12 |
#include <linux/compiler.h> |
| 13 |
#include <linux/linkage.h> |
| 14 |
|
| 15 |
#include <asm/addrspace.h> |
| 16 |
#include <asm/page.h> |
| 17 |
#include <asm/cachectl.h> |
| 18 |
#include <asm/fixmap.h> |
| 19 |
|
| 20 |
#if CONFIG_PGTABLE_LEVELS == 2 |
| 21 |
#include <asm-generic/pgtable-nopmd.h> |
| 22 |
#elif CONFIG_PGTABLE_LEVELS == 3 |
| 23 |
#include <asm-generic/pgtable-nopud.h> |
| 24 |
#else |
| 25 |
#include <asm-generic/pgtable-nop4d.h> |
| 26 |
#endif |
| 27 |
|
| 28 |
/* |
| 29 |
* Each address space has 2 4K pages as its page directory, giving 1024 |
| 30 |
* (== PTRS_PER_PGD) 8 byte pointers to pmd tables. Each pmd table is a |
| 31 |
* single 4K page, giving 512 (== PTRS_PER_PMD) 8 byte pointers to page |
| 32 |
* tables. Each page table is also a single 4K page, giving 512 (== |
| 33 |
* PTRS_PER_PTE) 8 byte ptes. Each pud entry is initialized to point to |
| 34 |
* invalid_pmd_table, each pmd entry is initialized to point to |
| 35 |
* invalid_pte_table, each pte is initialized to 0. |
| 36 |
* |
| 37 |
* Kernel mappings: kernel mappings are held in the swapper_pg_table. |
| 38 |
* The layout is identical to userspace except it's indexed with the |
| 39 |
* fault address - VMALLOC_START. |
| 40 |
*/ |
| 41 |
|
| 42 |
|
| 43 |
/* PGDIR_SHIFT determines what a third-level page table entry can map */ |
| 44 |
#ifdef __PAGETABLE_PMD_FOLDED |
| 45 |
#define PGDIR_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3) |
| 46 |
#else |
| 47 |
|
| 48 |
/* PMD_SHIFT determines the size of the area a second-level page table can map */ |
| 49 |
#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - 3)) |
| 50 |
#define PMD_SIZE (1UL << PMD_SHIFT) |
| 51 |
#define PMD_MASK (~(PMD_SIZE-1)) |
| 52 |
|
| 53 |
# ifdef __PAGETABLE_PUD_FOLDED |
| 54 |
# define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_TABLE_ORDER - 3)) |
| 55 |
# endif |
| 56 |
#endif |
| 57 |
|
| 58 |
#ifndef __PAGETABLE_PUD_FOLDED |
| 59 |
#define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT + PMD_TABLE_ORDER - 3)) |
| 60 |
#define PUD_SIZE (1UL << PUD_SHIFT) |
| 61 |
#define PUD_MASK (~(PUD_SIZE-1)) |
| 62 |
#define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT + PUD_TABLE_ORDER - 3)) |
| 63 |
#endif |
| 64 |
|
| 65 |
#define PGDIR_SIZE (1UL << PGDIR_SHIFT) |
| 66 |
#define PGDIR_MASK (~(PGDIR_SIZE-1)) |
| 67 |
|
| 68 |
/* |
| 69 |
* For 4kB page size we use a 3 level page tree and an 8kB pud, which |
| 70 |
* permits us mapping 40 bits of virtual address space. |
| 71 |
* |
| 72 |
* We used to implement 41 bits by having an order 1 pmd level but that seemed |
| 73 |
* rather pointless. |
| 74 |
* |
| 75 |
* For 8kB page size we use a 3 level page tree which permits a total of |
| 76 |
* 8TB of address space. Alternatively a 33-bit / 8GB organization using |
| 77 |
* two levels would be easy to implement. |
| 78 |
* |
| 79 |
* For 16kB page size we use a 2 level page tree which permits a total of |
| 80 |
* 36 bits of virtual address space. We could add a third level but it seems |
| 81 |
* like at the moment there's no need for this. |
| 82 |
* |
| 83 |
* For 64kB page size we use a 2 level page table tree for a total of 42 bits |
| 84 |
* of virtual address space. |
| 85 |
*/ |
| 86 |
#ifdef CONFIG_PAGE_SIZE_4KB |
| 87 |
# ifdef CONFIG_MIPS_VA_BITS_48 |
| 88 |
# define PGD_TABLE_ORDER 0 |
| 89 |
# define PUD_TABLE_ORDER 0 |
| 90 |
# else |
| 91 |
# define PGD_TABLE_ORDER 1 |
| 92 |
# define PUD_TABLE_ORDER aieeee_attempt_to_allocate_pud |
| 93 |
# endif |
| 94 |
#define PMD_TABLE_ORDER 0 |
| 95 |
#endif |
| 96 |
#ifdef CONFIG_PAGE_SIZE_8KB |
| 97 |
#define PGD_TABLE_ORDER 0 |
| 98 |
#define PUD_TABLE_ORDER aieeee_attempt_to_allocate_pud |
| 99 |
#define PMD_TABLE_ORDER 0 |
| 100 |
#endif |
| 101 |
#ifdef CONFIG_PAGE_SIZE_16KB |
| 102 |
#ifdef CONFIG_MIPS_VA_BITS_48 |
| 103 |
#define PGD_TABLE_ORDER 1 |
| 104 |
#else |
| 105 |
#define PGD_TABLE_ORDER 0 |
| 106 |
#endif |
| 107 |
#define PUD_TABLE_ORDER aieeee_attempt_to_allocate_pud |
| 108 |
#define PMD_TABLE_ORDER 0 |
| 109 |
#endif |
| 110 |
#ifdef CONFIG_PAGE_SIZE_32KB |
| 111 |
#define PGD_TABLE_ORDER 0 |
| 112 |
#define PUD_TABLE_ORDER aieeee_attempt_to_allocate_pud |
| 113 |
#define PMD_TABLE_ORDER 0 |
| 114 |
#endif |
| 115 |
#ifdef CONFIG_PAGE_SIZE_64KB |
| 116 |
#define PGD_TABLE_ORDER 0 |
| 117 |
#define PUD_TABLE_ORDER aieeee_attempt_to_allocate_pud |
| 118 |
#ifdef CONFIG_MIPS_VA_BITS_48 |
| 119 |
#define PMD_TABLE_ORDER 0 |
| 120 |
#else |
| 121 |
#define PMD_TABLE_ORDER aieeee_attempt_to_allocate_pmd |
| 122 |
#endif |
| 123 |
#endif |
| 124 |
|
| 125 |
#define PTRS_PER_PGD ((PAGE_SIZE << PGD_TABLE_ORDER) / sizeof(pgd_t)) |
| 126 |
#ifndef __PAGETABLE_PUD_FOLDED |
| 127 |
#define PTRS_PER_PUD ((PAGE_SIZE << PUD_TABLE_ORDER) / sizeof(pud_t)) |
| 128 |
#endif |
| 129 |
#ifndef __PAGETABLE_PMD_FOLDED |
| 130 |
#define PTRS_PER_PMD ((PAGE_SIZE << PMD_TABLE_ORDER) / sizeof(pmd_t)) |
| 131 |
#endif |
| 132 |
#define PTRS_PER_PTE (PAGE_SIZE / sizeof(pte_t)) |
| 133 |
|
| 134 |
#define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1) |
| 135 |
|
| 136 |
/* |
| 137 |
* TLB refill handlers also map the vmalloc area into xuseg. Avoid |
| 138 |
* the first couple of pages so NULL pointer dereferences will still |
| 139 |
* reliably trap. |
| 140 |
*/ |
| 141 |
#define VMALLOC_START (MAP_BASE + (2 * PAGE_SIZE)) |
| 142 |
#define VMALLOC_END \ |
| 143 |
(MAP_BASE + \ |
| 144 |
min(PTRS_PER_PGD * PTRS_PER_PUD * PTRS_PER_PMD * PTRS_PER_PTE * PAGE_SIZE, \ |
| 145 |
(1UL << cpu_vmbits)) - (1UL << 32)) |
| 146 |
|
| 147 |
#if defined(CONFIG_MODULES) && defined(KBUILD_64BIT_SYM32) && \ |
| 148 |
VMALLOC_START != CKSSEG |
| 149 |
/* Load modules into 32bit-compatible segment. */ |
| 150 |
#define MODULES_VADDR CKSSEG |
| 151 |
#define MODULES_END (FIXADDR_START-2*PAGE_SIZE) |
| 152 |
#endif |
| 153 |
|
| 154 |
#define pte_ERROR(e) \ |
| 155 |
printk("%s:%d: bad pte %016lx.\n", __FILE__, __LINE__, pte_val(e)) |
| 156 |
#ifndef __PAGETABLE_PMD_FOLDED |
| 157 |
#define pmd_ERROR(e) \ |
| 158 |
printk("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) |
| 159 |
#endif |
| 160 |
#ifndef __PAGETABLE_PUD_FOLDED |
| 161 |
#define pud_ERROR(e) \ |
| 162 |
printk("%s:%d: bad pud %016lx.\n", __FILE__, __LINE__, pud_val(e)) |
| 163 |
#endif |
| 164 |
#define pgd_ERROR(e) \ |
| 165 |
printk("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) |
| 166 |
|
| 167 |
extern pte_t invalid_pte_table[PTRS_PER_PTE]; |
| 168 |
|
| 169 |
#ifndef __PAGETABLE_PUD_FOLDED |
| 170 |
/* |
| 171 |
* For 4-level pagetables we defines these ourselves, for 3-level the |
| 172 |
* definitions are below, for 2-level the |
| 173 |
* definitions are supplied by <asm-generic/pgtable-nopmd.h>. |
| 174 |
*/ |
| 175 |
typedef struct { unsigned long pud; } pud_t; |
| 176 |
#define pud_val(x) ((x).pud) |
| 177 |
#define __pud(x) ((pud_t) { (x) }) |
| 178 |
|
| 179 |
extern pud_t invalid_pud_table[PTRS_PER_PUD]; |
| 180 |
|
| 181 |
/* |
| 182 |
* Empty pgd entries point to the invalid_pud_table. |
| 183 |
*/ |
| 184 |
static inline int p4d_none(p4d_t p4d) |
| 185 |
{ |
| 186 |
return p4d_val(p4d) == (unsigned long)invalid_pud_table; |
| 187 |
} |
| 188 |
|
| 189 |
static inline int p4d_bad(p4d_t p4d) |
| 190 |
{ |
| 191 |
if (unlikely(p4d_val(p4d) & ~PAGE_MASK)) |
| 192 |
return 1; |
| 193 |
|
| 194 |
return 0; |
| 195 |
} |
| 196 |
|
| 197 |
static inline int p4d_present(p4d_t p4d) |
| 198 |
{ |
| 199 |
return p4d_val(p4d) != (unsigned long)invalid_pud_table; |
| 200 |
} |
| 201 |
|
| 202 |
static inline void p4d_clear(p4d_t *p4dp) |
| 203 |
{ |
| 204 |
p4d_val(*p4dp) = (unsigned long)invalid_pud_table; |
| 205 |
} |
| 206 |
|
| 207 |
static inline pud_t *p4d_pgtable(p4d_t p4d) |
| 208 |
{ |
| 209 |
return (pud_t *)p4d_val(p4d); |
| 210 |
} |
| 211 |
|
| 212 |
#define p4d_phys(p4d) virt_to_phys((void *)p4d_val(p4d)) |
| 213 |
#define p4d_page(p4d) (pfn_to_page(p4d_phys(p4d) >> PAGE_SHIFT)) |
| 214 |
|
| 215 |
#define p4d_index(address) (((address) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) |
| 216 |
|
| 217 |
static inline void set_p4d(p4d_t *p4d, p4d_t p4dval) |
| 218 |
{ |
| 219 |
*p4d = p4dval; |
| 220 |
} |
| 221 |
|
| 222 |
#endif |
| 223 |
|
| 224 |
#ifndef __PAGETABLE_PMD_FOLDED |
| 225 |
/* |
| 226 |
* For 3-level pagetables we defines these ourselves, for 2-level the |
| 227 |
* definitions are supplied by <asm-generic/pgtable-nopmd.h>. |
| 228 |
*/ |
| 229 |
typedef struct { unsigned long pmd; } pmd_t; |
| 230 |
#define pmd_val(x) ((x).pmd) |
| 231 |
#define __pmd(x) ((pmd_t) { (x) } ) |
| 232 |
|
| 233 |
|
| 234 |
extern pmd_t invalid_pmd_table[PTRS_PER_PMD]; |
| 235 |
#endif |
| 236 |
|
| 237 |
/* |
| 238 |
* Empty pgd/pmd entries point to the invalid_pte_table. |
| 239 |
*/ |
| 240 |
static inline int pmd_none(pmd_t pmd) |
| 241 |
{ |
| 242 |
return pmd_val(pmd) == (unsigned long) invalid_pte_table; |
| 243 |
} |
| 244 |
|
| 245 |
static inline int pmd_bad(pmd_t pmd) |
| 246 |
{ |
| 247 |
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT |
| 248 |
/* pmd_leaf(pmd) but inline */ |
| 249 |
if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) |
| 250 |
return 0; |
| 251 |
#endif |
| 252 |
|
| 253 |
if (unlikely(pmd_val(pmd) & ~PAGE_MASK)) |
| 254 |
return 1; |
| 255 |
|
| 256 |
return 0; |
| 257 |
} |
| 258 |
|
| 259 |
static inline int pmd_present(pmd_t pmd) |
| 260 |
{ |
| 261 |
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT |
| 262 |
if (unlikely(pmd_val(pmd) & _PAGE_HUGE)) |
| 263 |
return pmd_val(pmd) & _PAGE_PRESENT; |
| 264 |
#endif |
| 265 |
|
| 266 |
return pmd_val(pmd) != (unsigned long) invalid_pte_table; |
| 267 |
} |
| 268 |
|
| 269 |
static inline void pmd_clear(pmd_t *pmdp) |
| 270 |
{ |
| 271 |
pmd_val(*pmdp) = ((unsigned long) invalid_pte_table); |
| 272 |
} |
| 273 |
#ifndef __PAGETABLE_PMD_FOLDED |
| 274 |
|
| 275 |
/* |
| 276 |
* Empty pud entries point to the invalid_pmd_table. |
| 277 |
*/ |
| 278 |
static inline int pud_none(pud_t pud) |
| 279 |
{ |
| 280 |
return pud_val(pud) == (unsigned long) invalid_pmd_table; |
| 281 |
} |
| 282 |
|
| 283 |
static inline int pud_bad(pud_t pud) |
| 284 |
{ |
| 285 |
return pud_val(pud) & ~PAGE_MASK; |
| 286 |
} |
| 287 |
|
| 288 |
static inline int pud_present(pud_t pud) |
| 289 |
{ |
| 290 |
return pud_val(pud) != (unsigned long) invalid_pmd_table; |
| 291 |
} |
| 292 |
|
| 293 |
static inline void pud_clear(pud_t *pudp) |
| 294 |
{ |
| 295 |
pud_val(*pudp) = ((unsigned long) invalid_pmd_table); |
| 296 |
} |
| 297 |
#endif |
| 298 |
|
| 299 |
#define pte_page(x) pfn_to_page(pte_pfn(x)) |
| 300 |
|
| 301 |
#define pte_pfn(x) ((unsigned long)((x).pte >> PFN_PTE_SHIFT)) |
| 302 |
#define pfn_pte(pfn, prot) __pte(((pfn) << PFN_PTE_SHIFT) | pgprot_val(prot)) |
| 303 |
#define pfn_pmd(pfn, prot) __pmd(((pfn) << PFN_PTE_SHIFT) | pgprot_val(prot)) |
| 304 |
|
| 305 |
#ifndef __PAGETABLE_PMD_FOLDED |
| 306 |
static inline pmd_t *pud_pgtable(pud_t pud) |
| 307 |
{ |
| 308 |
return (pmd_t *)pud_val(pud); |
| 309 |
} |
| 310 |
#define pud_phys(pud) virt_to_phys((void *)pud_val(pud)) |
| 311 |
#define pud_page(pud) (pfn_to_page(pud_phys(pud) >> PAGE_SHIFT)) |
| 312 |
|
| 313 |
#endif |
| 314 |
|
| 315 |
/* |
| 316 |
* Initialize a new pgd / pud / pmd table with invalid pointers. |
| 317 |
*/ |
| 318 |
extern void pgd_init(void *addr); |
| 319 |
extern void pud_init(void *addr); |
| 320 |
#define pud_init pud_init |
| 321 |
extern void pmd_init(void *addr); |
| 322 |
#define pmd_init pmd_init |
| 323 |
|
| 324 |
/* |
| 325 |
* Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that |
| 326 |
* are !pte_none() && !pte_present(). |
| 327 |
* |
| 328 |
* Format of swap PTEs: |
| 329 |
* |
| 330 |
* 6 6 6 6 5 5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3 |
| 331 |
* 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 |
| 332 |
* <--------------------------- offset --------------------------- |
| 333 |
* |
| 334 |
* 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 |
| 335 |
* 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 |
| 336 |
* --------------> E <-- type ---> <---------- zeroes -----------> |
| 337 |
* |
| 338 |
* E is the exclusive marker that is not stored in swap entries. |
| 339 |
*/ |
| 340 |
static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) |
| 341 |
{ pte_t pte; pte_val(pte) = ((type & 0x7f) << 16) | (offset << 24); return pte; } |
| 342 |
|
| 343 |
#define __swp_type(x) (((x).val >> 16) & 0x7f) |
| 344 |
#define __swp_offset(x) ((x).val >> 24) |
| 345 |
#define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) }) |
| 346 |
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) |
| 347 |
#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) |
| 348 |
|
| 349 |
/* We borrow bit 23 to store the exclusive marker in swap PTEs. */ |
| 350 |
#define _PAGE_SWP_EXCLUSIVE (1 << 23) |
| 351 |
|
| 352 |
#endif /* _ASM_PGTABLE_64_H */ |