Debian Codegraph @

runtime/linux/alloc.c

raw
1 /* -*- linux-c -*-
2 * Memory allocation functions
3 * Copyright (C) 2005-2018 Red Hat Inc.
4 *
5 * This file is part of systemtap, and is free software. You can
6 * redistribute it and/or modify it under the terms of the GNU General
7 * Public License (GPL); either version 2, or (at your option) any
8 * later version.
9 */
10
11 #ifndef _STAPLINUX_ALLOC_C_
12 #define _STAPLINUX_ALLOC_C_
13
14 #include <linux/percpu.h>
15 #include "stp_helper_lock.h"
16
17 static long _stp_allocated_net_memory = 0;
18 /* Default, and should be "safe" from anywhere. */
19 /* NB: GFP_NOWAIT is defined since kernel 2.6.37 */
20 #define STP_ALLOC_FLAGS (GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN)
21 /* May only be used in context that can sleep. __GFP_NORETRY is to
22 suppress the oom-killer from kicking in. */
23 #define STP_ALLOC_SLEEP_FLAGS (GFP_KERNEL | __GFP_NORETRY)
24
25 #ifdef __GFP_DIRECT_RECLAIM
26
27 #define STP_ALLOC_FLAGS_MIGHT_SLEEP(flags) \
28 ((flags) & (__GFP_DIRECT_RECLAIM | __GFP_IO | __GFP_FS))
29
30 #elif defined(__GFP_WAIT)
31
32 #define STP_ALLOC_FLAGS_MIGHT_SLEEP(flags) \
33 ((flags) & (__GFP_WAIT | __GFP_IO | __GFP_FS))
34
35 #elif defined(__GFP_RECLAIM)
36
37 #define STP_ALLOC_FLAGS_MIGHT_SLEEP(flags) \
38 ((flags) & (__GFP_RECLAIM | __GFP_IO | __GFP_FS))
39
40 #endif
41
42 /* #define DEBUG_MEMALLOC_MIGHT_SLEEP */
43 /*
44 * If DEBUG_MEMALLOC_MIGHT_SLEEP is defined (stap -DDEBUG_MEMALLOC_MIGHT_SLEEP ...)
45 * then all memory allocation operations are assumed to sleep. This
46 * can be used to check whether any memory allocations occur in atomic
47 * context, which is discouraged on realtime kernels. */
48
49 /* #define DEBUG_MEM */
50 /*
51 * If DEBUG_MEM is defined (stap -DDEBUG_MEM ...) then full memory
52 * tracking is used. Each allocation is recorded and matched with
53 * a free. Also a fence is set around the allocated memory so overflows
54 * and underflows can be detected. Errors are written to the system log
55 * with printk.
56 *
57 * NOTE: if youy system is slow or your script makes a very large number
58 * of allocations, you may get a warning in the system log:
59 * BUG: soft lockup - CPU#1 stuck for 11s! [staprun:28269]
60 * This is an expected side-effect of the overhead of tracking, especially
61 * with a simple linked list of allocations. Optimization
62 * would be nice, but DEBUG_MEM is only for testing.
63 */
64
65 static long _stp_allocated_memory = 0;
66
67 #ifdef DEBUG_MEM
68
69 #define STRINGIZE(x) STRINGIZE2(x)
70 #define STRINGIZE2(x) #x
71 #define LINE_STRING STRINGIZE(__LINE__)
72 #define MTAG _stp_curr_debug_mem_tag = __FILE__ ":" LINE_STRING;
73
74 #else
75
76 #define MTAG
77
78 #endif
79
80 #ifdef DEBUG_MEM
81 static STP_DEFINE_SPINLOCK(_stp_mem_lock);
82
83 static const char *_stp_curr_debug_mem_tag = NULL;
84
85 #define STP_MEM_MAGIC 0xc11cf77f
86 #define MEM_FENCE_SIZE 32
87
88 enum _stp_memtype { STP_MEM_KMALLOC, STP_MEM_VMALLOC, STP_MEM_PERCPU };
89
90 typedef struct {
91 char *alloc;
92 char *free;
93 } _stp_malloc_type;
94
95 static const _stp_malloc_type _stp_malloc_types[] = {
96 {"kmalloc", "kfree"},
97 {"vmalloc", "vfree"},
98 {"alloc_percpu", "free_percpu"}
99 };
100
101 struct _stp_mem_entry {
102 struct list_head list;
103 int32_t magic;
104 enum _stp_memtype type;
105 size_t len;
106 void *addr;
107 const char *tag;
108 };
109
110 #define MEM_DEBUG_SIZE (2*MEM_FENCE_SIZE+sizeof(struct _stp_mem_entry))
111
112 static LIST_HEAD(_stp_mem_list);
113
114 static void _stp_check_mem_fence (char *addr, int size)
115 {
116 char *ptr;
117 int i;
118
119 ptr = addr - MEM_FENCE_SIZE;
120 while (ptr < addr) {
121 if (*ptr != 0x55) {
122 printk("SYSTEMTAP ERROR: Memory fence corrupted before allocated memory\n");
123 printk("at addr %p. (Allocation starts at %p)", ptr, addr);
124 return;
125 }
126 ptr++;
127 }
128 ptr = addr + size;
129 while (ptr < addr + size + MEM_FENCE_SIZE) {
130 if (*ptr != 0x55) {
131 printk("SYSTEMTAP ERROR: Memory fence corrupted after allocated memory\n");
132 printk("at addr %p. (Allocation ends at %p)", ptr, addr + size - 1);
133 return;
134 }
135 ptr++;
136 }
137 }
138
139 static void *_stp_mem_debug_setup(void *addr, size_t size, enum _stp_memtype type)
140 {
141 struct list_head *p;
142 struct _stp_mem_entry *m;
143 unsigned long flags;
144 memset(addr, 0x55, MEM_FENCE_SIZE);
145 addr += MEM_FENCE_SIZE;
146 memset(addr + size, 0x55, MEM_FENCE_SIZE);
147 p = (struct list_head *)(addr + size + MEM_FENCE_SIZE);
148 m = (struct _stp_mem_entry *)p;
149 m->magic = STP_MEM_MAGIC;
150 m->type = type;
151 m->len = size;
152 m->addr = addr;
153 m->tag = _stp_curr_debug_mem_tag;
154 stp_spin_lock_irqsave(&_stp_mem_lock, flags);
155 list_add(p, &_stp_mem_list);
156 stp_spin_unlock_irqrestore(&_stp_mem_lock, flags);
157 return addr;
158 }
159
160 /* Percpu allocations don't have the fence. Implementing it is problematic. */
161 static void _stp_mem_debug_percpu(struct _stp_mem_entry *m, void *addr, size_t size)
162 {
163 struct list_head *p = (struct list_head *)m;
164 unsigned long flags;
165 m->magic = STP_MEM_MAGIC;
166 m->type = STP_MEM_PERCPU;
167 m->len = size;
168 m->addr = addr;
169 stp_spin_lock_irqsave(&_stp_mem_lock, flags);
170 list_add(p, &_stp_mem_list);
171 stp_spin_unlock_irqrestore(&_stp_mem_lock, flags);
172 }
173
174 static void _stp_mem_debug_free(void *addr, enum _stp_memtype type)
175 {
176 int found = 0;
177 struct list_head *p, *tmp;
178 struct _stp_mem_entry *m = NULL;
179 unsigned long flags;
180
181 if (!addr) {
182 /* Passing NULL to these *free() functions is safe */
183 switch (type) {
184 case STP_MEM_KMALLOC:
185 case STP_MEM_PERCPU:
186 case STP_MEM_VMALLOC:
187 return;
188 }
189 }
190
191 stp_spin_lock_irqsave(&_stp_mem_lock, flags);
192 list_for_each_safe(p, tmp, &_stp_mem_list) {
193 m = list_entry(p, struct _stp_mem_entry, list);
194 if (m->addr == addr) {
195 list_del(p);
196 found = 1;
197 break;
198 }
199 }
200 stp_spin_unlock_irqrestore(&_stp_mem_lock, flags);
201 if (!found) {
202 printk("SYSTEMTAP ERROR: Free of unallocated memory %p type=%s\n",
203 addr, _stp_malloc_types[type].free);
204 return;
205 }
206 if (m->magic != STP_MEM_MAGIC) {
207 printk("SYSTEMTAP ERROR: Memory at %p corrupted!!\n", addr);
208 return;
209 }
210 if (m->type != type) {
211 printk("SYSTEMTAP ERROR: Memory allocated with %s and freed with %s\n",
212 _stp_malloc_types[m->type].alloc,
213 _stp_malloc_types[type].free);
214 }
215
216 switch (m->type) {
217 case STP_MEM_KMALLOC:
218 _stp_check_mem_fence(addr, m->len);
219 kfree(addr - MEM_FENCE_SIZE);
220 break;
221 case STP_MEM_PERCPU:
222 free_percpu(addr);
223 kfree(p);
224 break;
225 case STP_MEM_VMALLOC:
226 _stp_check_mem_fence(addr, m->len);
227 vfree(addr - MEM_FENCE_SIZE);
228 break;
229 default:
230 printk("SYSTEMTAP ERROR: Attempted to free memory at addr %p len=%d with unknown allocation type.\n", addr, (int)m->len);
231 }
232
233 return;
234 }
235
236 static void _stp_mem_debug_validate(void *addr)
237 {
238 int found = 0;
239 struct list_head *p, *tmp;
240 struct _stp_mem_entry *m = NULL;
241 unsigned long flags;
242
243 stp_spin_lock_irqsave(&_stp_mem_lock, flags);
244 list_for_each_safe(p, tmp, &_stp_mem_list) {
245 m = list_entry(p, struct _stp_mem_entry, list);
246 if (m->addr == addr) {
247 found = 1;
248 break;
249 }
250 }
251 stp_spin_unlock_irqrestore(&_stp_mem_lock, flags);
252 if (!found) {
253 printk("SYSTEMTAP ERROR: Couldn't validate memory %p\n",
254 addr);
255 return;
256 }
257 if (m->magic != STP_MEM_MAGIC) {
258 printk("SYSTEMTAP ERROR: Memory at %p corrupted!!\n", addr);
259 return;
260 }
261
262 switch (m->type) {
263 case STP_MEM_KMALLOC:
264 _stp_check_mem_fence(addr, m->len);
265 break;
266 case STP_MEM_PERCPU:
267 /* do nothing */
268 break;
269 case STP_MEM_VMALLOC:
270 _stp_check_mem_fence(addr, m->len);
271 break;
272 default:
273 printk("SYSTEMTAP ERROR: Attempted to validate memory at addr %p len=%d with unknown allocation type.\n", addr, (int)m->len);
274 }
275
276 return;
277 }
278 #endif
279
280 /* #define STP_MAXMEMORY 8192 */
281 /*
282 * If STP_MAXMEMORY is defined to a value (stap -DSTP_MAXMEMORY=8192
283 * ...) then every memory allocation is checked to make sure the
284 * systemtap module doesn't use more than STP_MAXMEMORY of memory.
285 * STP_MAXMEMORY is specified in kilobytes, so, for example, '8192'
286 * means that the systemtap module won't use more than 8 megabytes of
287 * memory.
288 *
289 * Note 1: This size does include the size of the module itself, plus
290 * any additional allocations.
291 *
292 * Note 2: Since we can't be ensured that the module transport is set
293 * up when a memory allocation problem happens, this code can't
294 * directly report an error back to a user (so instead it uses
295 * 'printk'). If the modules transport has been set up, the code that
296 * calls the memory allocation functions
297 * (_stp_kmalloc/_stp_kzalloc/etc.) should report an error directly to
298 * the user.
299 *
300 * Note 3: This only tracks direct allocations by the systemtap
301 * runtime. This does not track indirect allocations (such as done by
302 * kprobes/uprobes/etc. internals).
303 */
304
305 #ifdef STP_MAXMEMORY
306 #ifdef STAPCONF_MODULE_LAYOUT
307 #define _STP_MODULE_CORE_SIZE (THIS_MODULE->core_layout.size)
308 #elif defined(STAPCONF_MODULE_MEMORY)
309 #define _STP_MODULE_CORE_SIZE (THIS_MODULE->mem[MOD_TEXT].size)
310 #elif defined(STAPCONF_GRSECURITY)
311 #define _STP_MODULE_CORE_SIZE (THIS_MODULE->core_size_rw)
312 #else
313 #define _STP_MODULE_CORE_SIZE (THIS_MODULE->core_size)
314 #endif
315 #endif
316
317 static void *_stp_kmalloc_gfp(size_t size, gfp_t gfp_mask)
318 {
319 void *ret;
320 #ifdef DEBUG_MEMALLOC_MIGHT_SLEEP
321 if (STP_ALLOC_FLAGS_MIGHT_SLEEP(gfp_mask)) {
322 might_sleep();
323 }
324 #endif
325 #ifdef STP_MAXMEMORY
326 if ((_STP_MODULE_CORE_SIZE + _stp_allocated_memory + size)
327 > (STP_MAXMEMORY * 1024)) {
328 return NULL;
329 }
330 #endif
331 #ifdef DEBUG_MEM
332 ret = kmalloc(size + MEM_DEBUG_SIZE, gfp_mask);
333 if (likely(ret)) {
334 _stp_allocated_memory += size;
335 ret = _stp_mem_debug_setup(ret, size, STP_MEM_KMALLOC);
336 }
337 #else
338 ret = kmalloc(size, gfp_mask);
339 if (likely(ret)) {
340 _stp_allocated_memory += size;
341 }
342 #endif
343 return ret;
344 }
345
346 static void *_stp_kmalloc(size_t size)
347 {
348 return _stp_kmalloc_gfp(size, STP_ALLOC_FLAGS);
349 }
350
351 static void *_stp_kzalloc_gfp(size_t size, gfp_t gfp_mask)
352 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15)
353 {
354 void *ret;
355 #ifdef STP_MAXMEMORY
356 if ((_STP_MODULE_CORE_SIZE + _stp_allocated_memory + size)
357 > (STP_MAXMEMORY * 1024)) {
358 return NULL;
359 }
360 #endif
361 #ifdef DEBUG_MEM
362 ret = kmalloc(size + MEM_DEBUG_SIZE, gfp_mask);
363 if (likely(ret)) {
364 _stp_allocated_memory += size;
365 ret = _stp_mem_debug_setup(ret, size, STP_MEM_KMALLOC);
366 memset (ret, 0, size);
367 }
368 #else
369 ret = kmalloc(size, gfp_mask);
370 if (likely(ret)) {
371 _stp_allocated_memory += size;
372 memset (ret, 0, size);
373 }
374 #endif /* DEBUG_MEM */
375 return ret;
376 }
377 #else /* LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15) */
378 {
379 void *ret;
380 #ifdef STP_MAXMEMORY
381 if ((_STP_MODULE_CORE_SIZE + _stp_allocated_memory + size)
382 > (STP_MAXMEMORY * 1024)) {
383 return NULL;
384 }
385 #endif
386 #ifdef DEBUG_MEM
387 ret = kzalloc(size + MEM_DEBUG_SIZE, gfp_mask);
388 if (likely(ret)) {
389 _stp_allocated_memory += size;
390 ret = _stp_mem_debug_setup(ret, size, STP_MEM_KMALLOC);
391 }
392 #else
393 ret = kzalloc(size, gfp_mask);
394 if (likely(ret)) {
395 _stp_allocated_memory += size;
396 }
397 #endif
398 return ret;
399 }
400 #endif /* LINUX_VERSION_CODE < KERNEL_VERSION(2,6,15) */
401
402 static void *_stp_kzalloc(size_t size)
403 {
404 return _stp_kzalloc_gfp(size, STP_ALLOC_FLAGS);
405 }
406
407 static void *_stp_vzalloc(size_t size)
408 {
409 void *ret;
410 #ifdef STP_MAXMEMORY
411 if ((_STP_MODULE_CORE_SIZE + _stp_allocated_memory + size)
412 > (STP_MAXMEMORY * 1024)) {
413 return NULL;
414 }
415 #endif
416 #ifdef DEBUG_MEM
417 ret = vzalloc(size + MEM_DEBUG_SIZE);
418 if (likely(ret)) {
419 _stp_allocated_memory += size;
420 ret = _stp_mem_debug_setup(ret, size, STP_MEM_VMALLOC);
421 }
422 #else
423 ret = vzalloc(size);
424 if (likely(ret)) {
425 _stp_allocated_memory += size;
426 }
427 #endif
428 return ret;
429 }
430
431 static void *_stp_vzalloc_node(size_t size, int node)
432 {
433 void *ret;
434
435 #ifdef STP_MAXMEMORY
436 if ((_STP_MODULE_CORE_SIZE + _stp_allocated_memory + size)
437 > (STP_MAXMEMORY * 1024)) {
438 return NULL;
439 }
440 #endif
441 #ifdef DEBUG_MEM
442 ret = vzalloc_node(size + MEM_DEBUG_SIZE, node);
443 if (likely(ret)) {
444 _stp_allocated_memory += size;
445 ret = _stp_mem_debug_setup(ret, size, STP_MEM_VMALLOC);
446 }
447 #else
448 ret = vzalloc_node(size, node);
449 if (likely(ret)) {
450 _stp_allocated_memory += size;
451 }
452 #endif
453 return ret;
454 }
455
456 #ifdef PCPU_MIN_UNIT_SIZE
457 #define _STP_MAX_PERCPU_SIZE PCPU_MIN_UNIT_SIZE
458 #else
459 #define _STP_MAX_PERCPU_SIZE 131072
460 #endif
461
462 /* Note, calls __alloc_percpu which may sleep and always uses GFP_KERNEL. */
463 static void __percpu *_stp_alloc_percpu(size_t size)
464 {
465 void __percpu *ret;
466
467 if (size > _STP_MAX_PERCPU_SIZE)
468 return NULL;
469
470 #ifdef STP_MAXMEMORY
471 if ((_STP_MODULE_CORE_SIZE + _stp_allocated_memory
472 + (size * num_possible_cpus()))
473 > (STP_MAXMEMORY * 1024)) {
474 return NULL;
475 }
476 #endif
477
478 #ifdef STAPCONF_ALLOC_PERCPU_ALIGN
479 ret = __alloc_percpu(size, 8);
480 #else
481 ret = __alloc_percpu(size);
482 #endif
483 #ifdef DEBUG_MEM
484 if (likely(ret)) {
485 struct _stp_mem_entry *m = kmalloc(sizeof(struct _stp_mem_entry), GFP_KERNEL);
486 if (unlikely(m == NULL)) {
487 free_percpu(ret);
488 return NULL;
489 }
490 _stp_allocated_memory += size * num_possible_cpus();
491 _stp_mem_debug_percpu(m, ret, size);
492 }
493 #else
494 if (likely(ret)) {
495 _stp_allocated_memory += size * num_possible_cpus();
496 }
497 #endif
498 return ret;
499 }
500
501 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,12)
502 #define _stp_kmalloc_node(size,node) _stp_kmalloc(size)
503 #define _stp_kmalloc_node_gfp(size,node,gfp) _stp_kmalloc_gfp(size,gfp)
504 #define _stp_kzalloc_node(size,node) _stp_kzalloc(size)
505 #define _stp_kzalloc_node_gfp(size,node,gfp) _stp_kzalloc_gfp(size,gfp)
506 #else
507 static void *_stp_kmalloc_node_gfp(size_t size, int node, gfp_t gfp_mask)
508 {
509 void *ret;
510 #ifdef DEBUG_MEMALLOC_MIGHT_SLEEP
511 if (STP_ALLOC_FLAGS_MIGHT_SLEEP(gfp_mask)) {
512 might_sleep();
513 }
514 #endif
515 #ifdef STP_MAXMEMORY
516 if ((_STP_MODULE_CORE_SIZE + _stp_allocated_memory + size)
517 > (STP_MAXMEMORY * 1024)) {
518 return NULL;
519 }
520 #endif
521 #ifdef DEBUG_MEM
522 ret = kmalloc_node(size + MEM_DEBUG_SIZE, gfp_mask, node);
523 if (likely(ret)) {
524 _stp_allocated_memory += size;
525 ret = _stp_mem_debug_setup(ret, size, STP_MEM_KMALLOC);
526 }
527 #else
528 ret = kmalloc_node(size, gfp_mask, node);
529 if (likely(ret)) {
530 _stp_allocated_memory += size;
531 }
532 #endif
533 return ret;
534 }
535 static void *_stp_kzalloc_node_gfp(size_t size, int node, gfp_t gfp_mask)
536 {
537 /* This used to be simply:
538 * return _stp_kmalloc_node_gfp(size, node, gfp_mask | __GFP_ZERO);
539 * but rhel5-era kernels BUG out on that flag. (PR14957)
540 *
541 * We could make a big #if-alternation for kernels which have support
542 * for kzalloc_node (kernel commit 979b0fea), as in _stp_kzalloc_gfp,
543 * but IMO that's needlessly complex.
544 *
545 * So for now, just malloc and zero it manually.
546 */
547 void *ret = _stp_kmalloc_node_gfp(size, node, gfp_mask);
548 #ifdef DEBUG_MEMALLOC_MIGHT_SLEEP
549 if (STP_ALLOC_FLAGS_MIGHT_SLEEP(gfp_mask)) {
550 might_sleep();
551 }
552 #endif
553 if (likely(ret)) {
554 memset (ret, 0, size);
555 }
556 return ret;
557 }
558 static void *_stp_kmalloc_node(size_t size, int node)
559 {
560 return _stp_kmalloc_node_gfp(size, node, STP_ALLOC_FLAGS);
561 }
562 static void *_stp_kzalloc_node(size_t size, int node)
563 {
564 return _stp_kzalloc_node_gfp(size, node, STP_ALLOC_FLAGS);
565 }
566 #endif /* LINUX_VERSION_CODE */
567
568 static void _stp_kfree(void *addr)
569 {
570 #ifdef DEBUG_MEM
571 _stp_mem_debug_free(addr, STP_MEM_KMALLOC);
572 #else
573 kfree(addr);
574 #endif
575 }
576
577 static void _stp_vfree(void *addr)
578 {
579 #ifdef DEBUG_MEMALLOC_MIGHT_SLEEP
580 might_sleep();
581 #endif
582 #ifdef DEBUG_MEM
583 _stp_mem_debug_free(addr, STP_MEM_VMALLOC);
584 #else
585 vfree(addr);
586 #endif
587 }
588
589 static void _stp_free_percpu(void __percpu *addr)
590 {
591 #ifdef DEBUG_MEM
592 _stp_mem_debug_free(addr, STP_MEM_PERCPU);
593 #else
594 free_percpu(addr);
595 #endif
596 }
597
598 static void _stp_mem_debug_done(void)
599 {
600 #ifdef DEBUG_MEM
601 struct list_head *p, *tmp;
602 struct _stp_mem_entry *m;
603 unsigned long flags;
604
605 stp_spin_lock_irqsave(&_stp_mem_lock, flags);
606 list_for_each_safe(p, tmp, &_stp_mem_list) {
607 m = list_entry(p, struct _stp_mem_entry, list);
608 list_del(p);
609
610 printk("SYSTEMTAP ERROR: Memory %p len=%d tag=%s allocation type: %s. Not freed.\n",
611 m->addr, (int)m->len, m->tag ? m->tag : "",
612 _stp_malloc_types[m->type].alloc);
613
614 if (m->magic != STP_MEM_MAGIC) {
615 printk("SYSTEMTAP ERROR: Memory at %p len=%d corrupted!!\n", m->addr, (int)m->len);
616 /* Don't free. Too dangerous */
617 goto done;
618 }
619
620 switch (m->type) {
621 case STP_MEM_KMALLOC:
622 _stp_check_mem_fence(m->addr, m->len);
623 kfree(m->addr - MEM_FENCE_SIZE);
624 break;
625 case STP_MEM_PERCPU:
626 free_percpu(m->addr);
627 kfree(p);
628 break;
629 case STP_MEM_VMALLOC:
630 _stp_check_mem_fence(m->addr, m->len);
631 vfree(m->addr - MEM_FENCE_SIZE);
632 break;
633 default:
634 printk("SYSTEMTAP ERROR: Attempted to free memory at addr %p len=%d with unknown allocation type.\n", m->addr, (int)m->len);
635 }
636 }
637 done:
638 stp_spin_unlock_irqrestore(&_stp_mem_lock, flags);
639
640 return;
641
642 #endif
643 }
644 #endif /* _STAPLINUX_ALLOC_C_ */