| 1 |
/* |
| 2 |
* This file is subject to the terms and conditions of the GNU General Public |
| 3 |
* License. See the file "COPYING" in the main directory of this archive |
| 4 |
* for more details. |
| 5 |
* |
| 6 |
* Copyright (C) 1996 David S. Miller ([email protected]) |
| 7 |
* Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle [email protected] |
| 8 |
* Carsten Langgaard, [email protected] |
| 9 |
* Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved. |
| 10 |
*/ |
| 11 |
#include <linux/cpu_pm.h> |
| 12 |
#include <linux/init.h> |
| 13 |
#include <linux/sched.h> |
| 14 |
#include <linux/smp.h> |
| 15 |
#include <linux/memblock.h> |
| 16 |
#include <linux/minmax.h> |
| 17 |
#include <linux/mm.h> |
| 18 |
#include <linux/hugetlb.h> |
| 19 |
#include <linux/export.h> |
| 20 |
#include <linux/sort.h> |
| 21 |
|
| 22 |
#include <asm/cpu.h> |
| 23 |
#include <asm/cpu-type.h> |
| 24 |
#include <asm/bootinfo.h> |
| 25 |
#include <asm/hazards.h> |
| 26 |
#include <asm/mmu_context.h> |
| 27 |
#include <asm/tlb.h> |
| 28 |
#include <asm/tlbdebug.h> |
| 29 |
#include <asm/tlbex.h> |
| 30 |
#include <asm/tlbmisc.h> |
| 31 |
#include <asm/setup.h> |
| 32 |
|
| 33 |
/* |
| 34 |
* LOONGSON-2 has a 4 entry itlb which is a subset of jtlb, LOONGSON-3 has |
| 35 |
* a 4 entry itlb and a 4 entry dtlb which are subsets of jtlb. Unfortunately, |
| 36 |
* itlb/dtlb are not totally transparent to software. |
| 37 |
*/ |
| 38 |
static inline void flush_micro_tlb(void) |
| 39 |
{ |
| 40 |
switch (current_cpu_type()) { |
| 41 |
case CPU_LOONGSON2EF: |
| 42 |
write_c0_diag(LOONGSON_DIAG_ITLB); |
| 43 |
break; |
| 44 |
case CPU_LOONGSON64: |
| 45 |
write_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB); |
| 46 |
break; |
| 47 |
default: |
| 48 |
break; |
| 49 |
} |
| 50 |
} |
| 51 |
|
| 52 |
static inline void flush_micro_tlb_vm(struct vm_area_struct *vma) |
| 53 |
{ |
| 54 |
if (vma->vm_flags & VM_EXEC) |
| 55 |
flush_micro_tlb(); |
| 56 |
} |
| 57 |
|
| 58 |
void local_flush_tlb_all(void) |
| 59 |
{ |
| 60 |
unsigned long flags; |
| 61 |
unsigned long old_ctx; |
| 62 |
int entry, ftlbhighset; |
| 63 |
|
| 64 |
local_irq_save(flags); |
| 65 |
/* Save old context and create impossible VPN2 value */ |
| 66 |
old_ctx = read_c0_entryhi(); |
| 67 |
htw_stop(); |
| 68 |
write_c0_entrylo0(0); |
| 69 |
write_c0_entrylo1(0); |
| 70 |
|
| 71 |
entry = num_wired_entries(); |
| 72 |
|
| 73 |
/* |
| 74 |
* Blast 'em all away. |
| 75 |
* If there are any wired entries, fall back to iterating |
| 76 |
*/ |
| 77 |
if (cpu_has_tlbinv && !entry) { |
| 78 |
if (current_cpu_data.tlbsizevtlb) { |
| 79 |
write_c0_index(0); |
| 80 |
mtc0_tlbw_hazard(); |
| 81 |
tlbinvf(); /* invalidate VTLB */ |
| 82 |
} |
| 83 |
ftlbhighset = current_cpu_data.tlbsizevtlb + |
| 84 |
current_cpu_data.tlbsizeftlbsets; |
| 85 |
for (entry = current_cpu_data.tlbsizevtlb; |
| 86 |
entry < ftlbhighset; |
| 87 |
entry++) { |
| 88 |
write_c0_index(entry); |
| 89 |
mtc0_tlbw_hazard(); |
| 90 |
tlbinvf(); /* invalidate one FTLB set */ |
| 91 |
} |
| 92 |
} else { |
| 93 |
while (entry < current_cpu_data.tlbsize) { |
| 94 |
/* Make sure all entries differ. */ |
| 95 |
write_c0_entryhi(UNIQUE_ENTRYHI(entry)); |
| 96 |
write_c0_index(entry); |
| 97 |
mtc0_tlbw_hazard(); |
| 98 |
tlb_write_indexed(); |
| 99 |
entry++; |
| 100 |
} |
| 101 |
} |
| 102 |
tlbw_use_hazard(); |
| 103 |
write_c0_entryhi(old_ctx); |
| 104 |
htw_start(); |
| 105 |
flush_micro_tlb(); |
| 106 |
local_irq_restore(flags); |
| 107 |
} |
| 108 |
EXPORT_SYMBOL(local_flush_tlb_all); |
| 109 |
|
| 110 |
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, |
| 111 |
unsigned long end) |
| 112 |
{ |
| 113 |
struct mm_struct *mm = vma->vm_mm; |
| 114 |
int cpu = smp_processor_id(); |
| 115 |
|
| 116 |
if (cpu_context(cpu, mm) != 0) { |
| 117 |
unsigned long size, flags; |
| 118 |
|
| 119 |
local_irq_save(flags); |
| 120 |
start = round_down(start, PAGE_SIZE << 1); |
| 121 |
end = round_up(end, PAGE_SIZE << 1); |
| 122 |
size = (end - start) >> (PAGE_SHIFT + 1); |
| 123 |
if (size <= (current_cpu_data.tlbsizeftlbsets ? |
| 124 |
current_cpu_data.tlbsize / 8 : |
| 125 |
current_cpu_data.tlbsize / 2)) { |
| 126 |
unsigned long old_entryhi, old_mmid; |
| 127 |
int newpid = cpu_asid(cpu, mm); |
| 128 |
|
| 129 |
old_entryhi = read_c0_entryhi(); |
| 130 |
if (cpu_has_mmid) { |
| 131 |
old_mmid = read_c0_memorymapid(); |
| 132 |
write_c0_memorymapid(newpid); |
| 133 |
} |
| 134 |
|
| 135 |
htw_stop(); |
| 136 |
while (start < end) { |
| 137 |
int idx; |
| 138 |
|
| 139 |
if (cpu_has_mmid) |
| 140 |
write_c0_entryhi(start); |
| 141 |
else |
| 142 |
write_c0_entryhi(start | newpid); |
| 143 |
start += (PAGE_SIZE << 1); |
| 144 |
mtc0_tlbw_hazard(); |
| 145 |
tlb_probe(); |
| 146 |
tlb_probe_hazard(); |
| 147 |
idx = read_c0_index(); |
| 148 |
write_c0_entrylo0(0); |
| 149 |
write_c0_entrylo1(0); |
| 150 |
if (idx < 0) |
| 151 |
continue; |
| 152 |
/* Make sure all entries differ. */ |
| 153 |
write_c0_entryhi(UNIQUE_ENTRYHI(idx)); |
| 154 |
mtc0_tlbw_hazard(); |
| 155 |
tlb_write_indexed(); |
| 156 |
} |
| 157 |
tlbw_use_hazard(); |
| 158 |
write_c0_entryhi(old_entryhi); |
| 159 |
if (cpu_has_mmid) |
| 160 |
write_c0_memorymapid(old_mmid); |
| 161 |
htw_start(); |
| 162 |
} else { |
| 163 |
drop_mmu_context(mm); |
| 164 |
} |
| 165 |
flush_micro_tlb(); |
| 166 |
local_irq_restore(flags); |
| 167 |
} |
| 168 |
} |
| 169 |
|
| 170 |
void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) |
| 171 |
{ |
| 172 |
unsigned long size, flags; |
| 173 |
|
| 174 |
local_irq_save(flags); |
| 175 |
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT; |
| 176 |
size = (size + 1) >> 1; |
| 177 |
if (size <= (current_cpu_data.tlbsizeftlbsets ? |
| 178 |
current_cpu_data.tlbsize / 8 : |
| 179 |
current_cpu_data.tlbsize / 2)) { |
| 180 |
int pid = read_c0_entryhi(); |
| 181 |
|
| 182 |
start &= (PAGE_MASK << 1); |
| 183 |
end += ((PAGE_SIZE << 1) - 1); |
| 184 |
end &= (PAGE_MASK << 1); |
| 185 |
htw_stop(); |
| 186 |
|
| 187 |
while (start < end) { |
| 188 |
int idx; |
| 189 |
|
| 190 |
write_c0_entryhi(start); |
| 191 |
start += (PAGE_SIZE << 1); |
| 192 |
mtc0_tlbw_hazard(); |
| 193 |
tlb_probe(); |
| 194 |
tlb_probe_hazard(); |
| 195 |
idx = read_c0_index(); |
| 196 |
write_c0_entrylo0(0); |
| 197 |
write_c0_entrylo1(0); |
| 198 |
if (idx < 0) |
| 199 |
continue; |
| 200 |
/* Make sure all entries differ. */ |
| 201 |
write_c0_entryhi(UNIQUE_ENTRYHI(idx)); |
| 202 |
mtc0_tlbw_hazard(); |
| 203 |
tlb_write_indexed(); |
| 204 |
} |
| 205 |
tlbw_use_hazard(); |
| 206 |
write_c0_entryhi(pid); |
| 207 |
htw_start(); |
| 208 |
} else { |
| 209 |
local_flush_tlb_all(); |
| 210 |
} |
| 211 |
flush_micro_tlb(); |
| 212 |
local_irq_restore(flags); |
| 213 |
} |
| 214 |
|
| 215 |
void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) |
| 216 |
{ |
| 217 |
int cpu = smp_processor_id(); |
| 218 |
|
| 219 |
if (cpu_context(cpu, vma->vm_mm) != 0) { |
| 220 |
unsigned long old_mmid; |
| 221 |
unsigned long flags, old_entryhi; |
| 222 |
int idx; |
| 223 |
|
| 224 |
page &= (PAGE_MASK << 1); |
| 225 |
local_irq_save(flags); |
| 226 |
old_entryhi = read_c0_entryhi(); |
| 227 |
htw_stop(); |
| 228 |
if (cpu_has_mmid) { |
| 229 |
old_mmid = read_c0_memorymapid(); |
| 230 |
write_c0_entryhi(page); |
| 231 |
write_c0_memorymapid(cpu_asid(cpu, vma->vm_mm)); |
| 232 |
} else { |
| 233 |
write_c0_entryhi(page | cpu_asid(cpu, vma->vm_mm)); |
| 234 |
} |
| 235 |
mtc0_tlbw_hazard(); |
| 236 |
tlb_probe(); |
| 237 |
tlb_probe_hazard(); |
| 238 |
idx = read_c0_index(); |
| 239 |
write_c0_entrylo0(0); |
| 240 |
write_c0_entrylo1(0); |
| 241 |
if (idx < 0) |
| 242 |
goto finish; |
| 243 |
/* Make sure all entries differ. */ |
| 244 |
write_c0_entryhi(UNIQUE_ENTRYHI(idx)); |
| 245 |
mtc0_tlbw_hazard(); |
| 246 |
tlb_write_indexed(); |
| 247 |
tlbw_use_hazard(); |
| 248 |
|
| 249 |
finish: |
| 250 |
write_c0_entryhi(old_entryhi); |
| 251 |
if (cpu_has_mmid) |
| 252 |
write_c0_memorymapid(old_mmid); |
| 253 |
htw_start(); |
| 254 |
flush_micro_tlb_vm(vma); |
| 255 |
local_irq_restore(flags); |
| 256 |
} |
| 257 |
} |
| 258 |
|
| 259 |
/* |
| 260 |
* This one is only used for pages with the global bit set so we don't care |
| 261 |
* much about the ASID. |
| 262 |
*/ |
| 263 |
void local_flush_tlb_one(unsigned long page) |
| 264 |
{ |
| 265 |
unsigned long flags; |
| 266 |
int oldpid, idx; |
| 267 |
|
| 268 |
local_irq_save(flags); |
| 269 |
oldpid = read_c0_entryhi(); |
| 270 |
htw_stop(); |
| 271 |
page &= (PAGE_MASK << 1); |
| 272 |
write_c0_entryhi(page); |
| 273 |
mtc0_tlbw_hazard(); |
| 274 |
tlb_probe(); |
| 275 |
tlb_probe_hazard(); |
| 276 |
idx = read_c0_index(); |
| 277 |
write_c0_entrylo0(0); |
| 278 |
write_c0_entrylo1(0); |
| 279 |
if (idx >= 0) { |
| 280 |
/* Make sure all entries differ. */ |
| 281 |
write_c0_entryhi(UNIQUE_ENTRYHI(idx)); |
| 282 |
mtc0_tlbw_hazard(); |
| 283 |
tlb_write_indexed(); |
| 284 |
tlbw_use_hazard(); |
| 285 |
} |
| 286 |
write_c0_entryhi(oldpid); |
| 287 |
htw_start(); |
| 288 |
flush_micro_tlb(); |
| 289 |
local_irq_restore(flags); |
| 290 |
} |
| 291 |
|
| 292 |
/* |
| 293 |
* We will need multiple versions of update_mmu_cache(), one that just |
| 294 |
* updates the TLB with the new pte(s), and another which also checks |
| 295 |
* for the R4k "end of page" hardware bug and does the needy. |
| 296 |
*/ |
| 297 |
void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte) |
| 298 |
{ |
| 299 |
unsigned long flags; |
| 300 |
pgd_t *pgdp; |
| 301 |
p4d_t *p4dp; |
| 302 |
pud_t *pudp; |
| 303 |
pmd_t *pmdp; |
| 304 |
pte_t *ptep, *ptemap = NULL; |
| 305 |
int idx, pid; |
| 306 |
|
| 307 |
/* |
| 308 |
* Handle debugger faulting in for debuggee. |
| 309 |
*/ |
| 310 |
if (current->active_mm != vma->vm_mm) |
| 311 |
return; |
| 312 |
|
| 313 |
local_irq_save(flags); |
| 314 |
|
| 315 |
htw_stop(); |
| 316 |
address &= (PAGE_MASK << 1); |
| 317 |
if (cpu_has_mmid) { |
| 318 |
write_c0_entryhi(address); |
| 319 |
} else { |
| 320 |
pid = read_c0_entryhi() & cpu_asid_mask(¤t_cpu_data); |
| 321 |
write_c0_entryhi(address | pid); |
| 322 |
} |
| 323 |
pgdp = pgd_offset(vma->vm_mm, address); |
| 324 |
mtc0_tlbw_hazard(); |
| 325 |
tlb_probe(); |
| 326 |
tlb_probe_hazard(); |
| 327 |
p4dp = p4d_offset(pgdp, address); |
| 328 |
pudp = pud_offset(p4dp, address); |
| 329 |
pmdp = pmd_offset(pudp, address); |
| 330 |
idx = read_c0_index(); |
| 331 |
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT |
| 332 |
/* this could be a huge page */ |
| 333 |
if (pmd_leaf(*pmdp)) { |
| 334 |
unsigned long lo; |
| 335 |
write_c0_pagemask(PM_HUGE_MASK); |
| 336 |
ptep = (pte_t *)pmdp; |
| 337 |
lo = pte_to_entrylo(pte_val(*ptep)); |
| 338 |
write_c0_entrylo0(lo); |
| 339 |
write_c0_entrylo1(lo + (HPAGE_SIZE >> 7)); |
| 340 |
|
| 341 |
mtc0_tlbw_hazard(); |
| 342 |
if (idx < 0) |
| 343 |
tlb_write_random(); |
| 344 |
else |
| 345 |
tlb_write_indexed(); |
| 346 |
tlbw_use_hazard(); |
| 347 |
write_c0_pagemask(PM_DEFAULT_MASK); |
| 348 |
} else |
| 349 |
#endif |
| 350 |
{ |
| 351 |
ptemap = ptep = pte_offset_map(pmdp, address); |
| 352 |
/* |
| 353 |
* update_mmu_cache() is called between pte_offset_map_lock() |
| 354 |
* and pte_unmap_unlock(), so we can assume that ptep is not |
| 355 |
* NULL here: and what should be done below if it were NULL? |
| 356 |
*/ |
| 357 |
|
| 358 |
#if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32) |
| 359 |
#ifdef CONFIG_XPA |
| 360 |
write_c0_entrylo0(pte_to_entrylo(ptep->pte_high)); |
| 361 |
if (cpu_has_xpa) |
| 362 |
writex_c0_entrylo0(ptep->pte_low & _PFNX_MASK); |
| 363 |
ptep++; |
| 364 |
write_c0_entrylo1(pte_to_entrylo(ptep->pte_high)); |
| 365 |
if (cpu_has_xpa) |
| 366 |
writex_c0_entrylo1(ptep->pte_low & _PFNX_MASK); |
| 367 |
#else |
| 368 |
write_c0_entrylo0(ptep->pte_high); |
| 369 |
ptep++; |
| 370 |
write_c0_entrylo1(ptep->pte_high); |
| 371 |
#endif |
| 372 |
#else |
| 373 |
write_c0_entrylo0(pte_to_entrylo(pte_val(*ptep++))); |
| 374 |
write_c0_entrylo1(pte_to_entrylo(pte_val(*ptep))); |
| 375 |
#endif |
| 376 |
mtc0_tlbw_hazard(); |
| 377 |
if (idx < 0) |
| 378 |
tlb_write_random(); |
| 379 |
else |
| 380 |
tlb_write_indexed(); |
| 381 |
} |
| 382 |
tlbw_use_hazard(); |
| 383 |
htw_start(); |
| 384 |
flush_micro_tlb_vm(vma); |
| 385 |
|
| 386 |
if (ptemap) |
| 387 |
pte_unmap(ptemap); |
| 388 |
local_irq_restore(flags); |
| 389 |
} |
| 390 |
|
| 391 |
void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1, |
| 392 |
unsigned long entryhi, unsigned long pagemask) |
| 393 |
{ |
| 394 |
#ifdef CONFIG_XPA |
| 395 |
panic("Broken for XPA kernels"); |
| 396 |
#else |
| 397 |
unsigned int old_mmid; |
| 398 |
unsigned long flags; |
| 399 |
unsigned long wired; |
| 400 |
unsigned long old_pagemask; |
| 401 |
unsigned long old_ctx; |
| 402 |
|
| 403 |
local_irq_save(flags); |
| 404 |
if (cpu_has_mmid) { |
| 405 |
old_mmid = read_c0_memorymapid(); |
| 406 |
write_c0_memorymapid(MMID_KERNEL_WIRED); |
| 407 |
} |
| 408 |
/* Save old context and create impossible VPN2 value */ |
| 409 |
old_ctx = read_c0_entryhi(); |
| 410 |
htw_stop(); |
| 411 |
old_pagemask = read_c0_pagemask(); |
| 412 |
wired = num_wired_entries(); |
| 413 |
write_c0_wired(wired + 1); |
| 414 |
write_c0_index(wired); |
| 415 |
tlbw_use_hazard(); /* What is the hazard here? */ |
| 416 |
write_c0_pagemask(pagemask); |
| 417 |
write_c0_entryhi(entryhi); |
| 418 |
write_c0_entrylo0(entrylo0); |
| 419 |
write_c0_entrylo1(entrylo1); |
| 420 |
mtc0_tlbw_hazard(); |
| 421 |
tlb_write_indexed(); |
| 422 |
tlbw_use_hazard(); |
| 423 |
|
| 424 |
write_c0_entryhi(old_ctx); |
| 425 |
if (cpu_has_mmid) |
| 426 |
write_c0_memorymapid(old_mmid); |
| 427 |
tlbw_use_hazard(); /* What is the hazard here? */ |
| 428 |
htw_start(); |
| 429 |
write_c0_pagemask(old_pagemask); |
| 430 |
local_flush_tlb_all(); |
| 431 |
local_irq_restore(flags); |
| 432 |
#endif |
| 433 |
} |
| 434 |
|
| 435 |
#ifdef CONFIG_TRANSPARENT_HUGEPAGE |
| 436 |
|
| 437 |
int has_transparent_hugepage(void) |
| 438 |
{ |
| 439 |
static unsigned int mask = -1; |
| 440 |
|
| 441 |
if (mask == -1) { /* first call comes during __init */ |
| 442 |
unsigned long flags; |
| 443 |
|
| 444 |
local_irq_save(flags); |
| 445 |
write_c0_pagemask(PM_HUGE_MASK); |
| 446 |
back_to_back_c0_hazard(); |
| 447 |
mask = read_c0_pagemask(); |
| 448 |
write_c0_pagemask(PM_DEFAULT_MASK); |
| 449 |
local_irq_restore(flags); |
| 450 |
} |
| 451 |
return mask == PM_HUGE_MASK; |
| 452 |
} |
| 453 |
EXPORT_SYMBOL(has_transparent_hugepage); |
| 454 |
|
| 455 |
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
| 456 |
|
| 457 |
/* |
| 458 |
* Used for loading TLB entries before trap_init() has started, when we |
| 459 |
* don't actually want to add a wired entry which remains throughout the |
| 460 |
* lifetime of the system |
| 461 |
*/ |
| 462 |
|
| 463 |
int temp_tlb_entry; |
| 464 |
|
| 465 |
#ifndef CONFIG_64BIT |
| 466 |
__init int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, |
| 467 |
unsigned long entryhi, unsigned long pagemask) |
| 468 |
{ |
| 469 |
int ret = 0; |
| 470 |
unsigned long flags; |
| 471 |
unsigned long wired; |
| 472 |
unsigned long old_pagemask; |
| 473 |
unsigned long old_ctx; |
| 474 |
|
| 475 |
local_irq_save(flags); |
| 476 |
/* Save old context and create impossible VPN2 value */ |
| 477 |
htw_stop(); |
| 478 |
old_ctx = read_c0_entryhi(); |
| 479 |
old_pagemask = read_c0_pagemask(); |
| 480 |
wired = num_wired_entries(); |
| 481 |
if (--temp_tlb_entry < wired) { |
| 482 |
printk(KERN_WARNING |
| 483 |
"No TLB space left for add_temporary_entry\n"); |
| 484 |
ret = -ENOSPC; |
| 485 |
goto out; |
| 486 |
} |
| 487 |
|
| 488 |
write_c0_index(temp_tlb_entry); |
| 489 |
write_c0_pagemask(pagemask); |
| 490 |
write_c0_entryhi(entryhi); |
| 491 |
write_c0_entrylo0(entrylo0); |
| 492 |
write_c0_entrylo1(entrylo1); |
| 493 |
mtc0_tlbw_hazard(); |
| 494 |
tlb_write_indexed(); |
| 495 |
tlbw_use_hazard(); |
| 496 |
|
| 497 |
write_c0_entryhi(old_ctx); |
| 498 |
write_c0_pagemask(old_pagemask); |
| 499 |
htw_start(); |
| 500 |
out: |
| 501 |
local_irq_restore(flags); |
| 502 |
return ret; |
| 503 |
} |
| 504 |
#endif |
| 505 |
|
| 506 |
static int ntlb; |
| 507 |
static int __init set_ntlb(char *str) |
| 508 |
{ |
| 509 |
get_option(&str, &ntlb); |
| 510 |
return 1; |
| 511 |
} |
| 512 |
|
| 513 |
__setup("ntlb=", set_ntlb); |
| 514 |
|
| 515 |
|
| 516 |
/* The start bit position of VPN2 and Mask in EntryHi/PageMask registers. */ |
| 517 |
#define VPN2_SHIFT 13 |
| 518 |
|
| 519 |
/* Read full EntryHi even with CONFIG_32BIT. */ |
| 520 |
static inline unsigned long long read_c0_entryhi_native(void) |
| 521 |
{ |
| 522 |
return cpu_has_64bits ? read_c0_entryhi_64() : read_c0_entryhi(); |
| 523 |
} |
| 524 |
|
| 525 |
/* Write full EntryHi even with CONFIG_32BIT. */ |
| 526 |
static inline void write_c0_entryhi_native(unsigned long long v) |
| 527 |
{ |
| 528 |
if (cpu_has_64bits) |
| 529 |
write_c0_entryhi_64(v); |
| 530 |
else |
| 531 |
write_c0_entryhi(v); |
| 532 |
} |
| 533 |
|
| 534 |
/* TLB entry state for uniquification. */ |
| 535 |
struct tlbent { |
| 536 |
unsigned long long wired:1; |
| 537 |
unsigned long long global:1; |
| 538 |
unsigned long long asid:10; |
| 539 |
unsigned long long vpn:51; |
| 540 |
unsigned long long pagesz:5; |
| 541 |
unsigned long long index:14; |
| 542 |
}; |
| 543 |
|
| 544 |
/* |
| 545 |
* Comparison function for TLB entry sorting. Place wired entries first, |
| 546 |
* then global entries, then order by the increasing VPN/ASID and the |
| 547 |
* decreasing page size. This lets us avoid clashes with wired entries |
| 548 |
* easily and get entries for larger pages out of the way first. |
| 549 |
* |
| 550 |
* We could group bits so as to reduce the number of comparisons, but this |
| 551 |
* is seldom executed and not performance-critical, so prefer legibility. |
| 552 |
*/ |
| 553 |
static int r4k_entry_cmp(const void *a, const void *b) |
| 554 |
{ |
| 555 |
struct tlbent ea = *(struct tlbent *)a, eb = *(struct tlbent *)b; |
| 556 |
|
| 557 |
if (ea.wired > eb.wired) |
| 558 |
return -1; |
| 559 |
else if (ea.wired < eb.wired) |
| 560 |
return 1; |
| 561 |
else if (ea.global > eb.global) |
| 562 |
return -1; |
| 563 |
else if (ea.global < eb.global) |
| 564 |
return 1; |
| 565 |
else if (ea.vpn < eb.vpn) |
| 566 |
return -1; |
| 567 |
else if (ea.vpn > eb.vpn) |
| 568 |
return 1; |
| 569 |
else if (ea.asid < eb.asid) |
| 570 |
return -1; |
| 571 |
else if (ea.asid > eb.asid) |
| 572 |
return 1; |
| 573 |
else if (ea.pagesz > eb.pagesz) |
| 574 |
return -1; |
| 575 |
else if (ea.pagesz < eb.pagesz) |
| 576 |
return 1; |
| 577 |
else |
| 578 |
return 0; |
| 579 |
} |
| 580 |
|
| 581 |
/* |
| 582 |
* Fetch all the TLB entries. Mask individual VPN values retrieved with |
| 583 |
* the corresponding page mask and ignoring any 1KiB extension as we'll |
| 584 |
* be using 4KiB pages for uniquification. |
| 585 |
*/ |
| 586 |
static void __ref r4k_tlb_uniquify_read(struct tlbent *tlb_vpns, int tlbsize) |
| 587 |
{ |
| 588 |
int start = num_wired_entries(); |
| 589 |
unsigned long long vpn_mask; |
| 590 |
bool global; |
| 591 |
int i; |
| 592 |
|
| 593 |
vpn_mask = GENMASK(current_cpu_data.vmbits - 1, VPN2_SHIFT); |
| 594 |
vpn_mask |= cpu_has_64bits ? 3ULL << 62 : 1 << 31; |
| 595 |
|
| 596 |
for (i = 0; i < tlbsize; i++) { |
| 597 |
unsigned long long entryhi, vpn, mask, asid; |
| 598 |
unsigned int pagesz; |
| 599 |
|
| 600 |
write_c0_index(i); |
| 601 |
mtc0_tlbr_hazard(); |
| 602 |
tlb_read(); |
| 603 |
tlb_read_hazard(); |
| 604 |
|
| 605 |
global = !!(read_c0_entrylo0() & ENTRYLO_G); |
| 606 |
entryhi = read_c0_entryhi_native(); |
| 607 |
mask = read_c0_pagemask(); |
| 608 |
|
| 609 |
asid = entryhi & cpu_asid_mask(¤t_cpu_data); |
| 610 |
vpn = (entryhi & vpn_mask & ~mask) >> VPN2_SHIFT; |
| 611 |
pagesz = ilog2((mask >> VPN2_SHIFT) + 1); |
| 612 |
|
| 613 |
tlb_vpns[i].global = global; |
| 614 |
tlb_vpns[i].asid = global ? 0 : asid; |
| 615 |
tlb_vpns[i].vpn = vpn; |
| 616 |
tlb_vpns[i].pagesz = pagesz; |
| 617 |
tlb_vpns[i].wired = i < start; |
| 618 |
tlb_vpns[i].index = i; |
| 619 |
} |
| 620 |
} |
| 621 |
|
| 622 |
/* |
| 623 |
* Write unique values to all but the wired TLB entries each, using |
| 624 |
* the 4KiB page size. This size might not be supported with R6, but |
| 625 |
* EHINV is mandatory for R6, so we won't ever be called in that case. |
| 626 |
* |
| 627 |
* A sorted table is supplied with any wired entries at the beginning, |
| 628 |
* followed by any global entries, and then finally regular entries. |
| 629 |
* We start at the VPN and ASID values of zero and only assign user |
| 630 |
* addresses, therefore guaranteeing no clash with addresses produced |
| 631 |
* by UNIQUE_ENTRYHI. We avoid any VPN values used by wired or global |
| 632 |
* entries, by increasing the VPN value beyond the span of such entry. |
| 633 |
* |
| 634 |
* When a VPN/ASID clash is found with a regular entry we increment the |
| 635 |
* ASID instead until no VPN/ASID clash has been found or the ASID space |
| 636 |
* has been exhausted, in which case we increase the VPN value beyond |
| 637 |
* the span of the largest clashing entry. |
| 638 |
* |
| 639 |
* We do not need to be concerned about FTLB or MMID configurations as |
| 640 |
* those are required to implement the EHINV feature. |
| 641 |
*/ |
| 642 |
static void __ref r4k_tlb_uniquify_write(struct tlbent *tlb_vpns, int tlbsize) |
| 643 |
{ |
| 644 |
unsigned long long asid, vpn, vpn_size, pagesz; |
| 645 |
int widx, gidx, idx, sidx, lidx, i; |
| 646 |
|
| 647 |
vpn_size = 1ULL << (current_cpu_data.vmbits - VPN2_SHIFT); |
| 648 |
pagesz = ilog2((PM_4K >> VPN2_SHIFT) + 1); |
| 649 |
|
| 650 |
write_c0_pagemask(PM_4K); |
| 651 |
write_c0_entrylo0(0); |
| 652 |
write_c0_entrylo1(0); |
| 653 |
|
| 654 |
asid = 0; |
| 655 |
vpn = 0; |
| 656 |
widx = 0; |
| 657 |
gidx = 0; |
| 658 |
for (sidx = 0; sidx < tlbsize && tlb_vpns[sidx].wired; sidx++) |
| 659 |
; |
| 660 |
for (lidx = sidx; lidx < tlbsize && tlb_vpns[lidx].global; lidx++) |
| 661 |
; |
| 662 |
idx = gidx = sidx + 1; |
| 663 |
for (i = sidx; i < tlbsize; i++) { |
| 664 |
unsigned long long entryhi, vpn_pagesz = 0; |
| 665 |
|
| 666 |
while (1) { |
| 667 |
if (WARN_ON(vpn >= vpn_size)) { |
| 668 |
dump_tlb_all(); |
| 669 |
/* Pray local_flush_tlb_all() will cope. */ |
| 670 |
return; |
| 671 |
} |
| 672 |
|
| 673 |
/* VPN must be below the next wired entry. */ |
| 674 |
if (widx < sidx && vpn >= tlb_vpns[widx].vpn) { |
| 675 |
vpn = max(vpn, |
| 676 |
(tlb_vpns[widx].vpn + |
| 677 |
(1ULL << tlb_vpns[widx].pagesz))); |
| 678 |
asid = 0; |
| 679 |
widx++; |
| 680 |
continue; |
| 681 |
} |
| 682 |
/* VPN must be below the next global entry. */ |
| 683 |
if (gidx < lidx && vpn >= tlb_vpns[gidx].vpn) { |
| 684 |
vpn = max(vpn, |
| 685 |
(tlb_vpns[gidx].vpn + |
| 686 |
(1ULL << tlb_vpns[gidx].pagesz))); |
| 687 |
asid = 0; |
| 688 |
gidx++; |
| 689 |
continue; |
| 690 |
} |
| 691 |
/* Try to find a free ASID so as to conserve VPNs. */ |
| 692 |
if (idx < tlbsize && vpn == tlb_vpns[idx].vpn && |
| 693 |
asid == tlb_vpns[idx].asid) { |
| 694 |
unsigned long long idx_pagesz; |
| 695 |
|
| 696 |
idx_pagesz = tlb_vpns[idx].pagesz; |
| 697 |
vpn_pagesz = max(vpn_pagesz, idx_pagesz); |
| 698 |
do |
| 699 |
idx++; |
| 700 |
while (idx < tlbsize && |
| 701 |
vpn == tlb_vpns[idx].vpn && |
| 702 |
asid == tlb_vpns[idx].asid); |
| 703 |
asid++; |
| 704 |
if (asid > cpu_asid_mask(¤t_cpu_data)) { |
| 705 |
vpn += vpn_pagesz; |
| 706 |
asid = 0; |
| 707 |
vpn_pagesz = 0; |
| 708 |
} |
| 709 |
continue; |
| 710 |
} |
| 711 |
/* VPN mustn't be above the next regular entry. */ |
| 712 |
if (idx < tlbsize && vpn > tlb_vpns[idx].vpn) { |
| 713 |
vpn = max(vpn, |
| 714 |
(tlb_vpns[idx].vpn + |
| 715 |
(1ULL << tlb_vpns[idx].pagesz))); |
| 716 |
asid = 0; |
| 717 |
idx++; |
| 718 |
continue; |
| 719 |
} |
| 720 |
break; |
| 721 |
} |
| 722 |
|
| 723 |
entryhi = (vpn << VPN2_SHIFT) | asid; |
| 724 |
write_c0_entryhi_native(entryhi); |
| 725 |
write_c0_index(tlb_vpns[i].index); |
| 726 |
mtc0_tlbw_hazard(); |
| 727 |
tlb_write_indexed(); |
| 728 |
|
| 729 |
tlb_vpns[i].asid = asid; |
| 730 |
tlb_vpns[i].vpn = vpn; |
| 731 |
tlb_vpns[i].pagesz = pagesz; |
| 732 |
|
| 733 |
asid++; |
| 734 |
if (asid > cpu_asid_mask(¤t_cpu_data)) { |
| 735 |
vpn += 1ULL << pagesz; |
| 736 |
asid = 0; |
| 737 |
} |
| 738 |
} |
| 739 |
} |
| 740 |
|
| 741 |
/* |
| 742 |
* Initialise all TLB entries with unique values that do not clash with |
| 743 |
* what we have been handed over and what we'll be using ourselves. |
| 744 |
*/ |
| 745 |
static void __ref r4k_tlb_uniquify(void) |
| 746 |
{ |
| 747 |
int tlbsize = current_cpu_data.tlbsize; |
| 748 |
bool use_slab = slab_is_available(); |
| 749 |
phys_addr_t tlb_vpn_size; |
| 750 |
struct tlbent *tlb_vpns; |
| 751 |
|
| 752 |
tlb_vpn_size = tlbsize * sizeof(*tlb_vpns); |
| 753 |
tlb_vpns = (use_slab ? |
| 754 |
kmalloc(tlb_vpn_size, GFP_ATOMIC) : |
| 755 |
memblock_alloc_raw(tlb_vpn_size, sizeof(*tlb_vpns))); |
| 756 |
if (WARN_ON(!tlb_vpns)) |
| 757 |
return; /* Pray local_flush_tlb_all() is good enough. */ |
| 758 |
|
| 759 |
htw_stop(); |
| 760 |
|
| 761 |
r4k_tlb_uniquify_read(tlb_vpns, tlbsize); |
| 762 |
|
| 763 |
sort(tlb_vpns, tlbsize, sizeof(*tlb_vpns), r4k_entry_cmp, NULL); |
| 764 |
|
| 765 |
r4k_tlb_uniquify_write(tlb_vpns, tlbsize); |
| 766 |
|
| 767 |
write_c0_pagemask(PM_DEFAULT_MASK); |
| 768 |
|
| 769 |
tlbw_use_hazard(); |
| 770 |
htw_start(); |
| 771 |
flush_micro_tlb(); |
| 772 |
if (use_slab) |
| 773 |
kfree(tlb_vpns); |
| 774 |
else |
| 775 |
memblock_free(tlb_vpns, tlb_vpn_size); |
| 776 |
} |
| 777 |
|
| 778 |
/* |
| 779 |
* Configure TLB (for init or after a CPU has been powered off). |
| 780 |
*/ |
| 781 |
static void r4k_tlb_configure(void) |
| 782 |
{ |
| 783 |
/* |
| 784 |
* You should never change this register: |
| 785 |
* - On R4600 1.7 the tlbp never hits for pages smaller than |
| 786 |
* the value in the c0_pagemask register. |
| 787 |
* - The entire mm handling assumes the c0_pagemask register to |
| 788 |
* be set to fixed-size pages. |
| 789 |
*/ |
| 790 |
write_c0_pagemask(PM_DEFAULT_MASK); |
| 791 |
back_to_back_c0_hazard(); |
| 792 |
if (read_c0_pagemask() != PM_DEFAULT_MASK) |
| 793 |
panic("MMU doesn't support PAGE_SIZE=0x%lx", PAGE_SIZE); |
| 794 |
|
| 795 |
write_c0_wired(0); |
| 796 |
if (current_cpu_type() == CPU_R10000 || |
| 797 |
current_cpu_type() == CPU_R12000 || |
| 798 |
current_cpu_type() == CPU_R14000 || |
| 799 |
current_cpu_type() == CPU_R16000) |
| 800 |
write_c0_framemask(0); |
| 801 |
|
| 802 |
if (cpu_has_rixi) { |
| 803 |
/* |
| 804 |
* Enable the no read, no exec bits, and enable large physical |
| 805 |
* address. |
| 806 |
*/ |
| 807 |
#ifdef CONFIG_64BIT |
| 808 |
set_c0_pagegrain(PG_RIE | PG_XIE | PG_ELPA); |
| 809 |
#else |
| 810 |
set_c0_pagegrain(PG_RIE | PG_XIE); |
| 811 |
#endif |
| 812 |
} |
| 813 |
|
| 814 |
temp_tlb_entry = current_cpu_data.tlbsize - 1; |
| 815 |
|
| 816 |
/* From this point on the ARC firmware is dead. */ |
| 817 |
if (!cpu_has_tlbinv) |
| 818 |
r4k_tlb_uniquify(); |
| 819 |
local_flush_tlb_all(); |
| 820 |
|
| 821 |
/* Did I tell you that ARC SUCKS? */ |
| 822 |
} |
| 823 |
|
| 824 |
void tlb_init(void) |
| 825 |
{ |
| 826 |
r4k_tlb_configure(); |
| 827 |
|
| 828 |
if (ntlb) { |
| 829 |
if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) { |
| 830 |
int wired = current_cpu_data.tlbsize - ntlb; |
| 831 |
write_c0_wired(wired); |
| 832 |
write_c0_index(wired-1); |
| 833 |
printk("Restricting TLB to %d entries\n", ntlb); |
| 834 |
} else |
| 835 |
printk("Ignoring invalid argument ntlb=%d\n", ntlb); |
| 836 |
} |
| 837 |
|
| 838 |
build_tlb_refill_handler(); |
| 839 |
} |
| 840 |
|
| 841 |
static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd, |
| 842 |
void *v) |
| 843 |
{ |
| 844 |
switch (cmd) { |
| 845 |
case CPU_PM_ENTER_FAILED: |
| 846 |
case CPU_PM_EXIT: |
| 847 |
r4k_tlb_configure(); |
| 848 |
break; |
| 849 |
} |
| 850 |
|
| 851 |
return NOTIFY_OK; |
| 852 |
} |
| 853 |
|
| 854 |
static struct notifier_block r4k_tlb_pm_notifier_block = { |
| 855 |
.notifier_call = r4k_tlb_pm_notifier, |
| 856 |
}; |
| 857 |
|
| 858 |
static int __init r4k_tlb_init_pm(void) |
| 859 |
{ |
| 860 |
return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block); |
| 861 |
} |
| 862 |
arch_initcall(r4k_tlb_init_pm); |