Debian Codegraph @

runtime/linux/map_runtime.h

raw
1 /* -*- linux-c -*-
2 * Map Runtime Functions
3 * Copyright (C) 2012-2016 Red Hat Inc.
4 *
5 * This file is part of systemtap, and is free software. You can
6 * redistribute it and/or modify it under the terms of the GNU General
7 * Public License (GPL); either version 2, or (at your option) any
8 * later version.
9 */
10
11 #ifndef _LINUX_MAP_RUNTIME_H_
12 #define _LINUX_MAP_RUNTIME_H_
13
14 /* get/put_cpu wrappers. Unnecessary if caller is already atomic. */
15 #define MAP_GET_CPU() smp_processor_id()
16 #define MAP_PUT_CPU() do {} while (0)
17
18
19 struct pmap {
20 int bit_shift; /* scale factor for integer arithmetic */
21 int stat_ops; /* related statistical operators */
22 MAP agg; /* aggregation map */
23 MAP __percpu *map; /* per-cpu maps */
24 };
25
26 static inline MAP _stp_pmap_get_agg(PMAP p)
27 {
28 return p->agg;
29 }
30
31 static inline void _stp_pmap_set_agg(PMAP p, MAP agg)
32 {
33 p->agg = agg;
34 }
35
36 static inline MAP _stp_pmap_get_map(PMAP p, unsigned cpu)
37 {
38 return *per_cpu_ptr(p->map, cpu);
39 }
40
41 static inline void _stp_pmap_set_map(PMAP p, MAP m, unsigned cpu)
42 {
43 *per_cpu_ptr(p->map, cpu) = m;
44 }
45
46
47 /** Deletes a map.
48 * Deletes a map, freeing all memory in all elements.
49 * Normally done only when the module exits.
50 * @param map
51 */
52
53 static void _stp_map_del(MAP map)
54 {
55 if (map == NULL)
56 return;
57
58 if (map->node_mem)
59 _stp_vfree(map->node_mem);
60
61 _stp_vfree(map);
62 }
63
64 static void _stp_pmap_del(PMAP pmap)
65 {
66 int i;
67
68 if (pmap == NULL)
69 return;
70
71 /* NB We cannot use the for_each_online_cpu() here since online
72 * CPUs may get changed on-the-fly through the CPU hotplug feature
73 * of the kernel. We only allocated the context structs on original
74 * online CPUs when _stp_pmap_new() was called.
75 */
76 for_each_possible_cpu(i) {
77 MAP m = _stp_pmap_get_map (pmap, i);
78 if (likely(m))
79 _stp_map_del(m);
80 }
81
82 _stp_free_percpu (pmap->map);
83
84 /* free agg map elements */
85 _stp_map_del(_stp_pmap_get_agg(pmap));
86 _stp_vfree(pmap);
87 }
88
89
90 static void*
91 _stp_map_vzalloc(size_t size, int cpu)
92 {
93 /* Called from module_init, so user context, may sleep alloc. */
94 if (cpu < 0)
95 return _stp_vzalloc(size);
96 return _stp_vzalloc_node(size, cpu_to_node(cpu));
97 }
98
99
100 static int
101 _stp_map_init(MAP m, unsigned max_entries, unsigned hash_table_mask,
102 int wrap, int node_size, int cpu)
103 {
104 unsigned i;
105
106 INIT_MLIST_HEAD(&m->pool);
107 INIT_MLIST_HEAD(&m->head);
108
109 m->hash_table_mask = hash_table_mask;
110 m->maxnum = max_entries;
111 m->wrap = wrap;
112 for (i = 0; i <= hash_table_mask; i++)
113 INIT_MHLIST_HEAD(&m->hashes[i]);
114
115 /* Since we're using _stp_map_vzalloc(), we can afford to
116 * allocate the nodes in one big chunk. */
117 m->node_mem = _stp_map_vzalloc(node_size * max_entries, cpu);
118 if (m->node_mem == NULL)
119 return -1;
120
121 for (i = 0; i < max_entries; i++) {
122 struct map_node *node = m->node_mem + i * node_size;
123 mlist_add(&node->lnode, &m->pool);
124 INIT_MHLIST_NODE(&node->hnode);
125 }
126
127 return 0;
128 }
129
130
131 /** Create a new map.
132 * Maps must be created at module initialization time.
133 * @param max_entries The maximum number of entries allowed. Currently that
134 * number will be preallocated.If more entries are required, the oldest ones
135 * will be deleted. This makes it effectively a circular buffer.
136 * @return A MAP on success or NULL on failure.
137 * @ingroup map_create
138 */
139
140 static MAP
141 _stp_map_new(unsigned max_entries, int wrap, int node_size, int cpu)
142 {
143 MAP m;
144 unsigned hash_table_mask = HASHTABLESIZE(max_entries)-1; /* usable as bitmask */
145 m = _stp_map_vzalloc(sizeof(struct map_root) +
146 sizeof(struct mhlist_head) * (hash_table_mask+1),
147 cpu);
148 if (m == NULL)
149 return NULL;
150
151 if (_stp_map_init(m, max_entries, hash_table_mask, wrap, node_size, cpu)) {
152 _stp_map_del(m);
153 return NULL;
154 }
155 return m;
156 }
157
158 static PMAP
159 _stp_pmap_new(unsigned max_entries, int wrap, int node_size)
160 {
161 int i;
162 MAP m;
163
164 PMAP pmap = _stp_map_vzalloc(sizeof(struct pmap), -1);
165 if (unlikely(pmap == NULL))
166 return NULL;
167
168 pmap->map = _stp_alloc_percpu (sizeof(MAP));
169 if (unlikely(pmap->map == NULL)) {
170 _stp_vfree(pmap);
171 return NULL;
172 }
173
174 /* Allocate the per-cpu maps. */
175
176 /* We don't use for_each_possible_cpu() here since the number of possible
177 * CPUs may be very large even though there are many fewere online CPUs.
178 * For example, VMWare guests usually have 128 possible CPUs while only
179 * have a few online CPUs. Once the context structs were
180 * allocated for online CPUs at this point, we will discard any context
181 * fetching operations on any future online CPUs dynamically added
182 * through the kernel's CPU hotplug feature. Memory allocations of the
183 * context structs can only happen right here.
184 */
185 for_each_online_cpu(i) {
186 m = _stp_map_new(max_entries, wrap, node_size, i);
187 if (unlikely(m == NULL))
188 goto err1;
189 _stp_pmap_set_map(pmap, m, i);
190 }
191
192 /* Allocate the aggregate map. */
193 m = _stp_map_new(max_entries, wrap, node_size, -1);
194 if (m == NULL)
195 goto err1;
196 _stp_pmap_set_agg(pmap, m);
197
198 return pmap;
199
200 err1:
201 for_each_possible_cpu(i) {
202 m = _stp_pmap_get_map (pmap, i);
203 if (likely(m))
204 _stp_map_del(m);
205 }
206 _stp_free_percpu (pmap->map);
207 _stp_vfree(pmap);
208 return NULL;
209 }
210
211 #endif /* _LINUX_MAP_RUNTIME_H_ */