| 1 |
/* SPDX-License-Identifier: GPL-2.0 */ |
| 2 |
/* |
| 3 |
* kernel/lockdep_internals.h |
| 4 |
* |
| 5 |
* Runtime locking correctness validator |
| 6 |
* |
| 7 |
* lockdep subsystem internal functions and variables. |
| 8 |
*/ |
| 9 |
|
| 10 |
/* |
| 11 |
* Lock-class usage-state bits: |
| 12 |
*/ |
| 13 |
enum lock_usage_bit { |
| 14 |
#define LOCKDEP_STATE(__STATE) \ |
| 15 |
LOCK_USED_IN_##__STATE, \ |
| 16 |
LOCK_USED_IN_##__STATE##_READ, \ |
| 17 |
LOCK_ENABLED_##__STATE, \ |
| 18 |
LOCK_ENABLED_##__STATE##_READ, |
| 19 |
#include "lockdep_states.h" |
| 20 |
#undef LOCKDEP_STATE |
| 21 |
LOCK_USED, |
| 22 |
LOCK_USED_READ, |
| 23 |
LOCK_USAGE_STATES, |
| 24 |
}; |
| 25 |
|
| 26 |
/* states after LOCK_USED_READ are not traced and printed */ |
| 27 |
static_assert(LOCK_TRACE_STATES == LOCK_USAGE_STATES); |
| 28 |
|
| 29 |
#define LOCK_USAGE_READ_MASK 1 |
| 30 |
#define LOCK_USAGE_DIR_MASK 2 |
| 31 |
#define LOCK_USAGE_STATE_MASK (~(LOCK_USAGE_READ_MASK | LOCK_USAGE_DIR_MASK)) |
| 32 |
|
| 33 |
/* |
| 34 |
* Usage-state bitmasks: |
| 35 |
*/ |
| 36 |
#define __LOCKF(__STATE) LOCKF_##__STATE = (1 << LOCK_##__STATE), |
| 37 |
|
| 38 |
enum { |
| 39 |
#define LOCKDEP_STATE(__STATE) \ |
| 40 |
__LOCKF(USED_IN_##__STATE) \ |
| 41 |
__LOCKF(USED_IN_##__STATE##_READ) \ |
| 42 |
__LOCKF(ENABLED_##__STATE) \ |
| 43 |
__LOCKF(ENABLED_##__STATE##_READ) |
| 44 |
#include "lockdep_states.h" |
| 45 |
#undef LOCKDEP_STATE |
| 46 |
__LOCKF(USED) |
| 47 |
__LOCKF(USED_READ) |
| 48 |
}; |
| 49 |
|
| 50 |
enum { |
| 51 |
#define LOCKDEP_STATE(__STATE) LOCKF_ENABLED_##__STATE | |
| 52 |
LOCKF_ENABLED_IRQ = |
| 53 |
#include "lockdep_states.h" |
| 54 |
0, |
| 55 |
#undef LOCKDEP_STATE |
| 56 |
|
| 57 |
#define LOCKDEP_STATE(__STATE) LOCKF_USED_IN_##__STATE | |
| 58 |
LOCKF_USED_IN_IRQ = |
| 59 |
#include "lockdep_states.h" |
| 60 |
0, |
| 61 |
#undef LOCKDEP_STATE |
| 62 |
|
| 63 |
#define LOCKDEP_STATE(__STATE) LOCKF_ENABLED_##__STATE##_READ | |
| 64 |
LOCKF_ENABLED_IRQ_READ = |
| 65 |
#include "lockdep_states.h" |
| 66 |
0, |
| 67 |
#undef LOCKDEP_STATE |
| 68 |
|
| 69 |
#define LOCKDEP_STATE(__STATE) LOCKF_USED_IN_##__STATE##_READ | |
| 70 |
LOCKF_USED_IN_IRQ_READ = |
| 71 |
#include "lockdep_states.h" |
| 72 |
0, |
| 73 |
#undef LOCKDEP_STATE |
| 74 |
}; |
| 75 |
|
| 76 |
#define LOCKF_ENABLED_IRQ_ALL (LOCKF_ENABLED_IRQ | LOCKF_ENABLED_IRQ_READ) |
| 77 |
#define LOCKF_USED_IN_IRQ_ALL (LOCKF_USED_IN_IRQ | LOCKF_USED_IN_IRQ_READ) |
| 78 |
|
| 79 |
#define LOCKF_IRQ (LOCKF_ENABLED_IRQ | LOCKF_USED_IN_IRQ) |
| 80 |
#define LOCKF_IRQ_READ (LOCKF_ENABLED_IRQ_READ | LOCKF_USED_IN_IRQ_READ) |
| 81 |
|
| 82 |
/* |
| 83 |
* CONFIG_LOCKDEP_SMALL is defined for sparc. Sparc requires .text, |
| 84 |
* .data and .bss to fit in required 32MB limit for the kernel. With |
| 85 |
* CONFIG_LOCKDEP we could go over this limit and cause system boot-up problems. |
| 86 |
* So, reduce the static allocations for lockdeps related structures so that |
| 87 |
* everything fits in current required size limit. |
| 88 |
*/ |
| 89 |
#ifdef CONFIG_LOCKDEP_SMALL |
| 90 |
/* |
| 91 |
* MAX_LOCKDEP_ENTRIES is the maximum number of lock dependencies |
| 92 |
* we track. |
| 93 |
* |
| 94 |
* We use the per-lock dependency maps in two ways: we grow it by adding |
| 95 |
* every to-be-taken lock to all currently held lock's own dependency |
| 96 |
* table (if it's not there yet), and we check it for lock order |
| 97 |
* conflicts and deadlocks. |
| 98 |
*/ |
| 99 |
#define MAX_LOCKDEP_ENTRIES 16384UL |
| 100 |
#define MAX_LOCKDEP_CHAINS_BITS 15 |
| 101 |
#define MAX_STACK_TRACE_ENTRIES 262144UL |
| 102 |
#define STACK_TRACE_HASH_SIZE 8192 |
| 103 |
#else |
| 104 |
#define MAX_LOCKDEP_ENTRIES (1UL << CONFIG_LOCKDEP_BITS) |
| 105 |
|
| 106 |
#define MAX_LOCKDEP_CHAINS_BITS CONFIG_LOCKDEP_CHAINS_BITS |
| 107 |
|
| 108 |
/* |
| 109 |
* Stack-trace: tightly packed array of stack backtrace |
| 110 |
* addresses. Protected by the hash_lock. |
| 111 |
*/ |
| 112 |
#define MAX_STACK_TRACE_ENTRIES (1UL << CONFIG_LOCKDEP_STACK_TRACE_BITS) |
| 113 |
#define STACK_TRACE_HASH_SIZE (1 << CONFIG_LOCKDEP_STACK_TRACE_HASH_BITS) |
| 114 |
#endif |
| 115 |
|
| 116 |
/* |
| 117 |
* Bit definitions for lock_chain.irq_context |
| 118 |
*/ |
| 119 |
#define LOCK_CHAIN_SOFTIRQ_CONTEXT (1 << 0) |
| 120 |
#define LOCK_CHAIN_HARDIRQ_CONTEXT (1 << 1) |
| 121 |
|
| 122 |
#define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS) |
| 123 |
|
| 124 |
#define AVG_LOCKDEP_CHAIN_DEPTH 5 |
| 125 |
#define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS * AVG_LOCKDEP_CHAIN_DEPTH) |
| 126 |
|
| 127 |
extern struct lock_chain lock_chains[]; |
| 128 |
|
| 129 |
#define LOCK_USAGE_CHARS (2*XXX_LOCK_USAGE_STATES + 1) |
| 130 |
|
| 131 |
extern void get_usage_chars(struct lock_class *class, |
| 132 |
char usage[LOCK_USAGE_CHARS]); |
| 133 |
|
| 134 |
extern const char *__get_key_name(const struct lockdep_subclass_key *key, |
| 135 |
char *str); |
| 136 |
|
| 137 |
struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i); |
| 138 |
|
| 139 |
extern unsigned long nr_lock_classes; |
| 140 |
extern unsigned long nr_zapped_classes; |
| 141 |
extern unsigned long nr_zapped_lock_chains; |
| 142 |
extern unsigned long nr_list_entries; |
| 143 |
extern unsigned long nr_dynamic_keys; |
| 144 |
long lockdep_next_lockchain(long i); |
| 145 |
unsigned long lock_chain_count(void); |
| 146 |
extern unsigned long nr_stack_trace_entries; |
| 147 |
|
| 148 |
extern unsigned int nr_hardirq_chains; |
| 149 |
extern unsigned int nr_softirq_chains; |
| 150 |
extern unsigned int nr_process_chains; |
| 151 |
extern unsigned int nr_free_chain_hlocks; |
| 152 |
extern unsigned int nr_lost_chain_hlocks; |
| 153 |
extern unsigned int nr_large_chain_blocks; |
| 154 |
|
| 155 |
extern unsigned int max_lockdep_depth; |
| 156 |
extern unsigned int max_bfs_queue_depth; |
| 157 |
extern unsigned long max_lock_class_idx; |
| 158 |
|
| 159 |
extern struct lock_class lock_classes[MAX_LOCKDEP_KEYS]; |
| 160 |
extern unsigned long lock_classes_in_use[]; |
| 161 |
|
| 162 |
#ifdef CONFIG_PROVE_LOCKING |
| 163 |
extern unsigned long lockdep_count_forward_deps(struct lock_class *); |
| 164 |
extern unsigned long lockdep_count_backward_deps(struct lock_class *); |
| 165 |
#ifdef CONFIG_TRACE_IRQFLAGS |
| 166 |
u64 lockdep_stack_trace_count(void); |
| 167 |
u64 lockdep_stack_hash_count(void); |
| 168 |
#endif |
| 169 |
#else |
| 170 |
static inline unsigned long |
| 171 |
lockdep_count_forward_deps(struct lock_class *class) |
| 172 |
{ |
| 173 |
return 0; |
| 174 |
} |
| 175 |
static inline unsigned long |
| 176 |
lockdep_count_backward_deps(struct lock_class *class) |
| 177 |
{ |
| 178 |
return 0; |
| 179 |
} |
| 180 |
#endif |
| 181 |
|
| 182 |
#ifdef CONFIG_DEBUG_LOCKDEP |
| 183 |
|
| 184 |
#include <asm/local.h> |
| 185 |
/* |
| 186 |
* Various lockdep statistics. |
| 187 |
* We want them per cpu as they are often accessed in fast path |
| 188 |
* and we want to avoid too much cache bouncing. |
| 189 |
*/ |
| 190 |
struct lockdep_stats { |
| 191 |
unsigned long chain_lookup_hits; |
| 192 |
unsigned int chain_lookup_misses; |
| 193 |
unsigned long hardirqs_on_events; |
| 194 |
unsigned long hardirqs_off_events; |
| 195 |
unsigned long redundant_hardirqs_on; |
| 196 |
unsigned long redundant_hardirqs_off; |
| 197 |
unsigned long softirqs_on_events; |
| 198 |
unsigned long softirqs_off_events; |
| 199 |
unsigned long redundant_softirqs_on; |
| 200 |
unsigned long redundant_softirqs_off; |
| 201 |
int nr_unused_locks; |
| 202 |
unsigned int nr_redundant_checks; |
| 203 |
unsigned int nr_redundant; |
| 204 |
unsigned int nr_cyclic_checks; |
| 205 |
unsigned int nr_find_usage_forwards_checks; |
| 206 |
unsigned int nr_find_usage_backwards_checks; |
| 207 |
|
| 208 |
/* |
| 209 |
* Per lock class locking operation stat counts |
| 210 |
*/ |
| 211 |
unsigned long lock_class_ops[MAX_LOCKDEP_KEYS]; |
| 212 |
}; |
| 213 |
|
| 214 |
DECLARE_PER_CPU(struct lockdep_stats, lockdep_stats); |
| 215 |
|
| 216 |
#define __debug_atomic_inc(ptr) \ |
| 217 |
this_cpu_inc(lockdep_stats.ptr); |
| 218 |
|
| 219 |
#define debug_atomic_inc(ptr) { \ |
| 220 |
WARN_ON_ONCE(!irqs_disabled()); \ |
| 221 |
__this_cpu_inc(lockdep_stats.ptr); \ |
| 222 |
} |
| 223 |
|
| 224 |
#define debug_atomic_dec(ptr) { \ |
| 225 |
WARN_ON_ONCE(!irqs_disabled()); \ |
| 226 |
__this_cpu_dec(lockdep_stats.ptr); \ |
| 227 |
} |
| 228 |
|
| 229 |
#define debug_atomic_read(ptr) ({ \ |
| 230 |
struct lockdep_stats *__cpu_lockdep_stats; \ |
| 231 |
unsigned long long __total = 0; \ |
| 232 |
int __cpu; \ |
| 233 |
for_each_possible_cpu(__cpu) { \ |
| 234 |
__cpu_lockdep_stats = &per_cpu(lockdep_stats, __cpu); \ |
| 235 |
__total += __cpu_lockdep_stats->ptr; \ |
| 236 |
} \ |
| 237 |
__total; \ |
| 238 |
}) |
| 239 |
|
| 240 |
static inline void debug_class_ops_inc(struct lock_class *class) |
| 241 |
{ |
| 242 |
int idx; |
| 243 |
|
| 244 |
idx = class - lock_classes; |
| 245 |
__debug_atomic_inc(lock_class_ops[idx]); |
| 246 |
} |
| 247 |
|
| 248 |
static inline unsigned long debug_class_ops_read(struct lock_class *class) |
| 249 |
{ |
| 250 |
int idx, cpu; |
| 251 |
unsigned long ops = 0; |
| 252 |
|
| 253 |
idx = class - lock_classes; |
| 254 |
for_each_possible_cpu(cpu) |
| 255 |
ops += per_cpu(lockdep_stats.lock_class_ops[idx], cpu); |
| 256 |
return ops; |
| 257 |
} |
| 258 |
|
| 259 |
#else |
| 260 |
# define __debug_atomic_inc(ptr) do { } while (0) |
| 261 |
# define debug_atomic_inc(ptr) do { } while (0) |
| 262 |
# define debug_atomic_dec(ptr) do { } while (0) |
| 263 |
# define debug_atomic_read(ptr) 0 |
| 264 |
# define debug_class_ops_inc(ptr) do { } while (0) |
| 265 |
#endif |