| 1 |
/* SPDX-License-Identifier: GPL-2.0-only */ |
| 2 |
#ifndef __MIPS_ASM_SYNC_H__ |
| 3 |
#define __MIPS_ASM_SYNC_H__ |
| 4 |
|
| 5 |
/* |
| 6 |
* sync types are defined by the MIPS64 Instruction Set documentation in Volume |
| 7 |
* II-A of the MIPS Architecture Reference Manual, which can be found here: |
| 8 |
* |
| 9 |
* https://www.mips.com/?do-download=the-mips64-instruction-set-v6-06 |
| 10 |
* |
| 11 |
* Two types of barrier are provided: |
| 12 |
* |
| 13 |
* 1) Completion barriers, which ensure that a memory operation has actually |
| 14 |
* completed & often involve stalling the CPU pipeline to do so. |
| 15 |
* |
| 16 |
* 2) Ordering barriers, which only ensure that affected memory operations |
| 17 |
* won't be reordered in the CPU pipeline in a manner that violates the |
| 18 |
* restrictions imposed by the barrier. |
| 19 |
* |
| 20 |
* Ordering barriers can be more efficient than completion barriers, since: |
| 21 |
* |
| 22 |
* a) Ordering barriers only require memory access instructions which precede |
| 23 |
* them in program order (older instructions) to reach a point in the |
| 24 |
* load/store datapath beyond which reordering is not possible before |
| 25 |
* allowing memory access instructions which follow them (younger |
| 26 |
* instructions) to be performed. That is, older instructions don't |
| 27 |
* actually need to complete - they just need to get far enough that all |
| 28 |
* other coherent CPUs will observe their completion before they observe |
| 29 |
* the effects of younger instructions. |
| 30 |
* |
| 31 |
* b) Multiple variants of ordering barrier are provided which allow the |
| 32 |
* effects to be restricted to different combinations of older or younger |
| 33 |
* loads or stores. By way of example, if we only care that stores older |
| 34 |
* than a barrier are observed prior to stores that are younger than a |
| 35 |
* barrier & don't care about the ordering of loads then the 'wmb' |
| 36 |
* ordering barrier can be used. Limiting the barrier's effects to stores |
| 37 |
* allows loads to continue unaffected & potentially allows the CPU to |
| 38 |
* make progress faster than if younger loads had to wait for older stores |
| 39 |
* to complete. |
| 40 |
*/ |
| 41 |
|
| 42 |
/* |
| 43 |
* No sync instruction at all; used to allow code to nullify the effect of the |
| 44 |
* __SYNC() macro without needing lots of #ifdefery. |
| 45 |
*/ |
| 46 |
#define __SYNC_none -1 |
| 47 |
|
| 48 |
/* |
| 49 |
* A full completion barrier; all memory accesses appearing prior to this sync |
| 50 |
* instruction in program order must complete before any memory accesses |
| 51 |
* appearing after this sync instruction in program order. |
| 52 |
*/ |
| 53 |
#define __SYNC_full 0x00 |
| 54 |
|
| 55 |
/* |
| 56 |
* For now we use a full completion barrier to implement all sync types, until |
| 57 |
* we're satisfied that lightweight ordering barriers defined by MIPSr6 are |
| 58 |
* sufficient to uphold our desired memory model. |
| 59 |
*/ |
| 60 |
#define __SYNC_aq __SYNC_full |
| 61 |
#define __SYNC_rl __SYNC_full |
| 62 |
#define __SYNC_mb __SYNC_full |
| 63 |
|
| 64 |
/* |
| 65 |
* ...except on Cavium Octeon CPUs, which have been using the 'wmb' ordering |
| 66 |
* barrier since 2010 & omit 'rmb' barriers because the CPUs don't perform |
| 67 |
* speculative reads. |
| 68 |
*/ |
| 69 |
#ifdef CONFIG_CPU_CAVIUM_OCTEON |
| 70 |
# define __SYNC_rmb __SYNC_none |
| 71 |
# define __SYNC_wmb 0x04 |
| 72 |
#else |
| 73 |
# define __SYNC_rmb __SYNC_full |
| 74 |
# define __SYNC_wmb __SYNC_full |
| 75 |
#endif |
| 76 |
|
| 77 |
/* |
| 78 |
* A GINV sync is a little different; it doesn't relate directly to loads or |
| 79 |
* stores, but instead causes synchronization of an icache or TLB global |
| 80 |
* invalidation operation triggered by the ginvi or ginvt instructions |
| 81 |
* respectively. In cases where we need to know that a ginvi or ginvt operation |
| 82 |
* has been performed by all coherent CPUs, we must issue a sync instruction of |
| 83 |
* this type. Once this instruction graduates all coherent CPUs will have |
| 84 |
* observed the invalidation. |
| 85 |
*/ |
| 86 |
#define __SYNC_ginv 0x14 |
| 87 |
|
| 88 |
/* Trivial; indicate that we always need this sync instruction. */ |
| 89 |
#define __SYNC_always (1 << 0) |
| 90 |
|
| 91 |
/* |
| 92 |
* Indicate that we need this sync instruction only on systems with weakly |
| 93 |
* ordered memory access. In general this is most MIPS systems, but there are |
| 94 |
* exceptions which provide strongly ordered memory. |
| 95 |
*/ |
| 96 |
#ifdef CONFIG_WEAK_ORDERING |
| 97 |
# define __SYNC_weak_ordering (1 << 1) |
| 98 |
#else |
| 99 |
# define __SYNC_weak_ordering 0 |
| 100 |
#endif |
| 101 |
|
| 102 |
/* |
| 103 |
* Indicate that we need this sync instruction only on systems where LL/SC |
| 104 |
* don't implicitly provide a memory barrier. In general this is most MIPS |
| 105 |
* systems. |
| 106 |
*/ |
| 107 |
#ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC |
| 108 |
# define __SYNC_weak_llsc (1 << 2) |
| 109 |
#else |
| 110 |
# define __SYNC_weak_llsc 0 |
| 111 |
#endif |
| 112 |
|
| 113 |
/* |
| 114 |
* Some Loongson 3 CPUs have a bug wherein execution of a memory access (load, |
| 115 |
* store or prefetch) in between an LL & SC can cause the SC instruction to |
| 116 |
* erroneously succeed, breaking atomicity. Whilst it's unusual to write code |
| 117 |
* containing such sequences, this bug bites harder than we might otherwise |
| 118 |
* expect due to reordering & speculation: |
| 119 |
* |
| 120 |
* 1) A memory access appearing prior to the LL in program order may actually |
| 121 |
* be executed after the LL - this is the reordering case. |
| 122 |
* |
| 123 |
* In order to avoid this we need to place a memory barrier (ie. a SYNC |
| 124 |
* instruction) prior to every LL instruction, in between it and any earlier |
| 125 |
* memory access instructions. |
| 126 |
* |
| 127 |
* This reordering case is fixed by 3A R2 CPUs, ie. 3A2000 models and later. |
| 128 |
* |
| 129 |
* 2) If a conditional branch exists between an LL & SC with a target outside |
| 130 |
* of the LL-SC loop, for example an exit upon value mismatch in cmpxchg() |
| 131 |
* or similar, then misprediction of the branch may allow speculative |
| 132 |
* execution of memory accesses from outside of the LL-SC loop. |
| 133 |
* |
| 134 |
* In order to avoid this we need a memory barrier (ie. a SYNC instruction) |
| 135 |
* at each affected branch target. |
| 136 |
* |
| 137 |
* This case affects all current Loongson 3 CPUs. |
| 138 |
* |
| 139 |
* The above described cases cause an error in the cache coherence protocol; |
| 140 |
* such that the Invalidate of a competing LL-SC goes 'missing' and SC |
| 141 |
* erroneously observes its core still has Exclusive state and lets the SC |
| 142 |
* proceed. |
| 143 |
* |
| 144 |
* Therefore the error only occurs on SMP systems. |
| 145 |
*/ |
| 146 |
#ifdef CONFIG_CPU_LOONGSON3_WORKAROUNDS |
| 147 |
# define __SYNC_loongson3_war (1 << 31) |
| 148 |
#else |
| 149 |
# define __SYNC_loongson3_war 0 |
| 150 |
#endif |
| 151 |
|
| 152 |
/* |
| 153 |
* Some Cavium Octeon CPUs suffer from a bug that causes a single wmb ordering |
| 154 |
* barrier to be ineffective, requiring the use of 2 in sequence to provide an |
| 155 |
* effective barrier as noted by commit 6b07d38aaa52 ("MIPS: Octeon: Use |
| 156 |
* optimized memory barrier primitives."). Here we specify that the affected |
| 157 |
* sync instructions should be emitted twice. |
| 158 |
* Note that this expression is evaluated by the assembler (not the compiler), |
| 159 |
* and that the assembler evaluates '==' as 0 or -1, not 0 or 1. |
| 160 |
*/ |
| 161 |
#ifdef CONFIG_CPU_CAVIUM_OCTEON |
| 162 |
# define __SYNC_rpt(type) (1 - (type == __SYNC_wmb)) |
| 163 |
#else |
| 164 |
# define __SYNC_rpt(type) 1 |
| 165 |
#endif |
| 166 |
|
| 167 |
/* |
| 168 |
* The main event. Here we actually emit a sync instruction of a given type, if |
| 169 |
* reason is non-zero. |
| 170 |
* |
| 171 |
* In future we have the option of emitting entries in a fixups-style table |
| 172 |
* here that would allow us to opportunistically remove some sync instructions |
| 173 |
* when we detect at runtime that we're running on a CPU that doesn't need |
| 174 |
* them. |
| 175 |
*/ |
| 176 |
#ifdef CONFIG_CPU_HAS_SYNC |
| 177 |
# define ____SYNC(_type, _reason, _else) \ |
| 178 |
.if (( _type ) != -1) && ( _reason ); \ |
| 179 |
.set push; \ |
| 180 |
.set MIPS_ISA_LEVEL_RAW; \ |
| 181 |
.rept __SYNC_rpt(_type); \ |
| 182 |
sync _type; \ |
| 183 |
.endr; \ |
| 184 |
.set pop; \ |
| 185 |
.else; \ |
| 186 |
_else; \ |
| 187 |
.endif |
| 188 |
#else |
| 189 |
# define ____SYNC(_type, _reason, _else) |
| 190 |
#endif |
| 191 |
|
| 192 |
/* |
| 193 |
* Preprocessor magic to expand macros used as arguments before we insert them |
| 194 |
* into assembly code. |
| 195 |
*/ |
| 196 |
#ifdef __ASSEMBLER__ |
| 197 |
# define ___SYNC(type, reason, else) \ |
| 198 |
____SYNC(type, reason, else) |
| 199 |
#else |
| 200 |
# define ___SYNC(type, reason, else) \ |
| 201 |
__stringify(____SYNC(type, reason, else)) |
| 202 |
#endif |
| 203 |
|
| 204 |
#define __SYNC(type, reason) \ |
| 205 |
___SYNC(__SYNC_##type, __SYNC_##reason, ) |
| 206 |
#define __SYNC_ELSE(type, reason, else) \ |
| 207 |
___SYNC(__SYNC_##type, __SYNC_##reason, else) |
| 208 |
|
| 209 |
#endif /* __MIPS_ASM_SYNC_H__ */ |