| 1 |
/* -*- linux-c -*- |
| 2 |
* Map Runtime Functions |
| 3 |
* Copyright (C) 2012-2016 Red Hat Inc. |
| 4 |
* |
| 5 |
* This file is part of systemtap, and is free software. You can |
| 6 |
* redistribute it and/or modify it under the terms of the GNU General |
| 7 |
* Public License (GPL); either version 2, or (at your option) any |
| 8 |
* later version. |
| 9 |
*/ |
| 10 |
|
| 11 |
#ifndef _LINUX_MAP_RUNTIME_H_ |
| 12 |
#define _LINUX_MAP_RUNTIME_H_ |
| 13 |
|
| 14 |
/* get/put_cpu wrappers. Unnecessary if caller is already atomic. */ |
| 15 |
#define MAP_GET_CPU() smp_processor_id() |
| 16 |
#define MAP_PUT_CPU() do {} while (0) |
| 17 |
|
| 18 |
|
| 19 |
struct pmap { |
| 20 |
int bit_shift; /* scale factor for integer arithmetic */ |
| 21 |
int stat_ops; /* related statistical operators */ |
| 22 |
MAP agg; /* aggregation map */ |
| 23 |
MAP __percpu *map; /* per-cpu maps */ |
| 24 |
}; |
| 25 |
|
| 26 |
static inline MAP _stp_pmap_get_agg(PMAP p) |
| 27 |
{ |
| 28 |
return p->agg; |
| 29 |
} |
| 30 |
|
| 31 |
static inline void _stp_pmap_set_agg(PMAP p, MAP agg) |
| 32 |
{ |
| 33 |
p->agg = agg; |
| 34 |
} |
| 35 |
|
| 36 |
static inline MAP _stp_pmap_get_map(PMAP p, unsigned cpu) |
| 37 |
{ |
| 38 |
return *per_cpu_ptr(p->map, cpu); |
| 39 |
} |
| 40 |
|
| 41 |
static inline void _stp_pmap_set_map(PMAP p, MAP m, unsigned cpu) |
| 42 |
{ |
| 43 |
*per_cpu_ptr(p->map, cpu) = m; |
| 44 |
} |
| 45 |
|
| 46 |
|
| 47 |
/** Deletes a map. |
| 48 |
* Deletes a map, freeing all memory in all elements. |
| 49 |
* Normally done only when the module exits. |
| 50 |
* @param map |
| 51 |
*/ |
| 52 |
|
| 53 |
static void _stp_map_del(MAP map) |
| 54 |
{ |
| 55 |
if (map == NULL) |
| 56 |
return; |
| 57 |
|
| 58 |
if (map->node_mem) |
| 59 |
_stp_vfree(map->node_mem); |
| 60 |
|
| 61 |
_stp_vfree(map); |
| 62 |
} |
| 63 |
|
| 64 |
static void _stp_pmap_del(PMAP pmap) |
| 65 |
{ |
| 66 |
int i; |
| 67 |
|
| 68 |
if (pmap == NULL) |
| 69 |
return; |
| 70 |
|
| 71 |
/* NB We cannot use the for_each_online_cpu() here since online |
| 72 |
* CPUs may get changed on-the-fly through the CPU hotplug feature |
| 73 |
* of the kernel. We only allocated the context structs on original |
| 74 |
* online CPUs when _stp_pmap_new() was called. |
| 75 |
*/ |
| 76 |
for_each_possible_cpu(i) { |
| 77 |
MAP m = _stp_pmap_get_map (pmap, i); |
| 78 |
if (likely(m)) |
| 79 |
_stp_map_del(m); |
| 80 |
} |
| 81 |
|
| 82 |
_stp_free_percpu (pmap->map); |
| 83 |
|
| 84 |
/* free agg map elements */ |
| 85 |
_stp_map_del(_stp_pmap_get_agg(pmap)); |
| 86 |
_stp_vfree(pmap); |
| 87 |
} |
| 88 |
|
| 89 |
|
| 90 |
static void* |
| 91 |
_stp_map_vzalloc(size_t size, int cpu) |
| 92 |
{ |
| 93 |
/* Called from module_init, so user context, may sleep alloc. */ |
| 94 |
if (cpu < 0) |
| 95 |
return _stp_vzalloc(size); |
| 96 |
return _stp_vzalloc_node(size, cpu_to_node(cpu)); |
| 97 |
} |
| 98 |
|
| 99 |
|
| 100 |
static int |
| 101 |
_stp_map_init(MAP m, unsigned max_entries, unsigned hash_table_mask, |
| 102 |
int wrap, int node_size, int cpu) |
| 103 |
{ |
| 104 |
unsigned i; |
| 105 |
|
| 106 |
INIT_MLIST_HEAD(&m->pool); |
| 107 |
INIT_MLIST_HEAD(&m->head); |
| 108 |
|
| 109 |
m->hash_table_mask = hash_table_mask; |
| 110 |
m->maxnum = max_entries; |
| 111 |
m->wrap = wrap; |
| 112 |
for (i = 0; i <= hash_table_mask; i++) |
| 113 |
INIT_MHLIST_HEAD(&m->hashes[i]); |
| 114 |
|
| 115 |
/* Since we're using _stp_map_vzalloc(), we can afford to |
| 116 |
* allocate the nodes in one big chunk. */ |
| 117 |
m->node_mem = _stp_map_vzalloc(node_size * max_entries, cpu); |
| 118 |
if (m->node_mem == NULL) |
| 119 |
return -1; |
| 120 |
|
| 121 |
for (i = 0; i < max_entries; i++) { |
| 122 |
struct map_node *node = m->node_mem + i * node_size; |
| 123 |
mlist_add(&node->lnode, &m->pool); |
| 124 |
INIT_MHLIST_NODE(&node->hnode); |
| 125 |
} |
| 126 |
|
| 127 |
return 0; |
| 128 |
} |
| 129 |
|
| 130 |
|
| 131 |
/** Create a new map. |
| 132 |
* Maps must be created at module initialization time. |
| 133 |
* @param max_entries The maximum number of entries allowed. Currently that |
| 134 |
* number will be preallocated.If more entries are required, the oldest ones |
| 135 |
* will be deleted. This makes it effectively a circular buffer. |
| 136 |
* @return A MAP on success or NULL on failure. |
| 137 |
* @ingroup map_create |
| 138 |
*/ |
| 139 |
|
| 140 |
static MAP |
| 141 |
_stp_map_new(unsigned max_entries, int wrap, int node_size, int cpu) |
| 142 |
{ |
| 143 |
MAP m; |
| 144 |
unsigned hash_table_mask = HASHTABLESIZE(max_entries)-1; /* usable as bitmask */ |
| 145 |
m = _stp_map_vzalloc(sizeof(struct map_root) + |
| 146 |
sizeof(struct mhlist_head) * (hash_table_mask+1), |
| 147 |
cpu); |
| 148 |
if (m == NULL) |
| 149 |
return NULL; |
| 150 |
|
| 151 |
if (_stp_map_init(m, max_entries, hash_table_mask, wrap, node_size, cpu)) { |
| 152 |
_stp_map_del(m); |
| 153 |
return NULL; |
| 154 |
} |
| 155 |
return m; |
| 156 |
} |
| 157 |
|
| 158 |
static PMAP |
| 159 |
_stp_pmap_new(unsigned max_entries, int wrap, int node_size) |
| 160 |
{ |
| 161 |
int i; |
| 162 |
MAP m; |
| 163 |
|
| 164 |
PMAP pmap = _stp_map_vzalloc(sizeof(struct pmap), -1); |
| 165 |
if (unlikely(pmap == NULL)) |
| 166 |
return NULL; |
| 167 |
|
| 168 |
pmap->map = _stp_alloc_percpu (sizeof(MAP)); |
| 169 |
if (unlikely(pmap->map == NULL)) { |
| 170 |
_stp_vfree(pmap); |
| 171 |
return NULL; |
| 172 |
} |
| 173 |
|
| 174 |
/* Allocate the per-cpu maps. */ |
| 175 |
|
| 176 |
/* We don't use for_each_possible_cpu() here since the number of possible |
| 177 |
* CPUs may be very large even though there are many fewere online CPUs. |
| 178 |
* For example, VMWare guests usually have 128 possible CPUs while only |
| 179 |
* have a few online CPUs. Once the context structs were |
| 180 |
* allocated for online CPUs at this point, we will discard any context |
| 181 |
* fetching operations on any future online CPUs dynamically added |
| 182 |
* through the kernel's CPU hotplug feature. Memory allocations of the |
| 183 |
* context structs can only happen right here. |
| 184 |
*/ |
| 185 |
for_each_online_cpu(i) { |
| 186 |
m = _stp_map_new(max_entries, wrap, node_size, i); |
| 187 |
if (unlikely(m == NULL)) |
| 188 |
goto err1; |
| 189 |
_stp_pmap_set_map(pmap, m, i); |
| 190 |
} |
| 191 |
|
| 192 |
/* Allocate the aggregate map. */ |
| 193 |
m = _stp_map_new(max_entries, wrap, node_size, -1); |
| 194 |
if (m == NULL) |
| 195 |
goto err1; |
| 196 |
_stp_pmap_set_agg(pmap, m); |
| 197 |
|
| 198 |
return pmap; |
| 199 |
|
| 200 |
err1: |
| 201 |
for_each_possible_cpu(i) { |
| 202 |
m = _stp_pmap_get_map (pmap, i); |
| 203 |
if (likely(m)) |
| 204 |
_stp_map_del(m); |
| 205 |
} |
| 206 |
_stp_free_percpu (pmap->map); |
| 207 |
_stp_vfree(pmap); |
| 208 |
return NULL; |
| 209 |
} |
| 210 |
|
| 211 |
#endif /* _LINUX_MAP_RUNTIME_H_ */ |