Debian Codegraph @

runtime/linux/print.c

raw
1 /* -*- linux-c -*-
2 * Print Functions
3 * Copyright (C) 2007-2018 Red Hat Inc.
4 *
5 * This file is part of systemtap, and is free software. You can
6 * redistribute it and/or modify it under the terms of the GNU General
7 * Public License (GPL); either version 2, or (at your option) any
8 * later version.
9 */
10
11 #ifndef _STAPLINUX_PRINT_C_
12 #define _STAPLINUX_PRINT_C_
13
14
15 #include "stp_string.h"
16 #include "print.h"
17 #include "transport/transport.c"
18 #include "vsprintf.c"
19
20 /** @file print.c
21 * Printing Functions.
22 */
23
24 /** @addtogroup print Print Functions
25 * The print buffer is for collecting output to send to the user daemon.
26 * This is a per-cpu static buffer. The buffer is sent when
27 * _stp_print_flush() is called.
28 *
29 * The reason to do this is to allow multiple small prints to be combined then
30 * timestamped and sent together to staprun. This is more efficient than sending
31 * numerous small packets.
32 *
33 * This function is called automatically when the print buffer is full.
34 * It MUST also be called at the end of every probe that prints something.
35 * @{
36 */
37
38 struct _stp_log {
39 unsigned int len; /* Bytes used in the buffer */
40 char *buf; /* NB we don't use arrays here to avoid allocating memory
41 on offline CPUs (but still possible ones) */
42 atomic_t reentrancy_lock;
43 bool no_flush;
44 bool is_full;
45 };
46 #include "print_flush.c"
47
48 static struct _stp_log __percpu *_stp_log_pcpu;
49
50 /*
51 * An atomic counter is used to synchronize every possible print buffer usage
52 * with the _stp_print_cleanup() function. The cleanup function sets the counter
53 * to INT_MAX after waiting for everything using the print buffer to finish. We
54 * cannot use a lock primitive to implement this because lock_acquire() contains
55 * tracepoints and print statements are used both inside and outside of probes.
56 * If the lock were only used inside probes, the runtime context would protect
57 * us from recursing into the lock_acquire() tracepoints and deadlocking. We
58 * instead use _stp_print_ctr as if it were a read-write lock.
59 */
60 static atomic_t _stp_print_ctr = ATOMIC_INIT(0);
61
62 /*
63 * This disables IRQs to make per-CPU print buffer accesses atomic. There is a
64 * reentrancy protection mechanism specifically for NMIs, since they can violate
65 * our atomic guarantee. Reentrancy is otherwise allowed within code sections
66 * that have the runtime context held (via _stp_runtime_entryfn_get_context()).
67 */
68 static bool _stp_print_trylock_irqsave(unsigned long *flags)
69 {
70 bool context_held = false;
71 struct _stp_log *log;
72
73 log = per_cpu_ptr(_stp_log_pcpu, raw_smp_processor_id());
74
75 /* We have to check log->buf == NULL here since the online
76 * CPUs may change at any time through the CPU hotplug
77 * feature of the kernel. So the current CPU's log->buf could be NULL
78 * if it is a new online CPU. And we cannot allocate new memory from within
79 * this context.
80 */
81 if (unlikely(log->buf == NULL))
82 return false;
83
84 local_irq_save(*flags);
85 if (!atomic_add_unless(&_stp_print_ctr, 1, INT_MAX))
86 goto irq_restore;
87
88 /*
89 * Check the per-CPU reentrancy lock for contention, unless the runtime
90 * context is already held, in which case we already have reentrancy
91 * protection. Otherwise, if the reentrancy lock is contented, that
92 * means we're either inside an NMI that fired while the current CPU was
93 * accessing the log buffer, or something is trying to nest calls to
94 * _stp_print_trylock_irqsave(). Our only choice is to reject the log
95 * access attempt in this case because log buffer corruption and panics
96 * could ensue if we're inside an NMI.
97 */
98 if (_stp_runtime_context_trylock()) {
99 struct context *c = _stp_runtime_get_context();
100 context_held = c && atomic_read(&c->busy);
101 _stp_runtime_context_unlock();
102 }
103
104 /* Fall back onto the reentrancy lock if the context isn't held */
105 if (!context_held) {
106 if (atomic_cmpxchg(&log->reentrancy_lock, 0, 1))
107 goto print_unlock;
108 }
109
110 return true;
111
112 print_unlock:
113 atomic_dec(&_stp_print_ctr);
114 irq_restore:
115 local_irq_restore(*flags);
116 return false;
117 }
118
119 static void _stp_print_unlock_irqrestore(unsigned long *flags)
120 {
121 bool context_held = false;
122 struct _stp_log *log;
123
124 if (_stp_runtime_context_trylock()) {
125 struct context *c = _stp_runtime_get_context();
126 context_held = c && atomic_read(&c->busy);
127 _stp_runtime_context_unlock();
128 }
129
130 if (!context_held) {
131 log = per_cpu_ptr(_stp_log_pcpu, raw_smp_processor_id());
132 atomic_set(&log->reentrancy_lock, 0);
133 }
134
135 atomic_dec(&_stp_print_ctr);
136 local_irq_restore(*flags);
137 }
138
139 /* create percpu print and io buffers */
140 static int _stp_print_init (void)
141 {
142 unsigned int cpu;
143
144 /* _stp_alloc_percpu() always zero-fills the allocated percpu memory
145 * since the kernel's __alloc_percpu() always does that.
146 */
147 _stp_log_pcpu = _stp_alloc_percpu(sizeof(*_stp_log_pcpu));
148 if (!_stp_log_pcpu)
149 return -ENOMEM;
150
151 /* We don't use for_aech_possible_cpu() here since the number of possible
152 * CPUs may be very large even though there are many fewere online CPUs.
153 * For example, VMWare guests usually have 128 possible CPUs while only
154 * have a few online CPUs. Once the bufs were allocated for
155 * online CPUs at this point, we will discard any printing operations on
156 * any future online CPUs dynamically added through the kernel's CPU
157 * hotplug feature. Memory allocations of the bufs can only happen right
158 * here.
159 */
160 for_each_online_cpu(cpu) {
161 struct _stp_log *log = per_cpu_ptr(_stp_log_pcpu, cpu);
162
163 log->reentrancy_lock = (atomic_t)ATOMIC_INIT(0);
164 log->buf = _stp_vzalloc_node(STP_BUFFER_SIZE, cpu_to_node(cpu));
165 if (unlikely(log->buf == NULL)) {
166 _stp_error ("print log buf (size %lu per cpu) allocation failed",
167 (unsigned long) STP_BUFFER_SIZE);
168 return -ENOMEM;
169 }
170 }
171 return 0;
172 }
173
174 static void _stp_print_cleanup (void)
175 {
176 unsigned int cpu;
177
178 /* Wait for the loggers to finish modifying the print buffers */
179 while (atomic_cmpxchg(&_stp_print_ctr, 0, INT_MAX))
180 cpu_relax();
181
182 /* NB We cannot use the for_each_online_cpu() here since online
183 * CPUs may get changed on-the-fly through the CPU hotplug feature
184 * of the kernel. We only allocated bufs on original online CPUs
185 * when _stp_print_init() was called. */
186 for_each_possible_cpu(cpu) {
187 struct _stp_log *log = per_cpu_ptr(_stp_log_pcpu, cpu);
188
189 /*
190 * Flush anything that could be left in the print buffer. It is
191 * safe to do this without any kind of synchronization mechanism
192 * because nothing is using this print buffer anymore.
193 */
194 if (likely(log->buf)) {
195 /* We have to check log->buf != NULL here since the
196 * online CPUs may change at any time through the CPU
197 * hotplug feature of the kernel.
198 */
199 __stp_print_flush(log);
200 _stp_vfree(log->buf);
201 log->buf = NULL;
202 }
203 }
204
205 _stp_free_percpu(_stp_log_pcpu);
206 }
207
208 static inline void _stp_print_flush(void)
209 {
210 struct _stp_log *log;
211 unsigned long flags;
212
213 if (!_stp_print_trylock_irqsave(&flags))
214 return;
215
216 log = per_cpu_ptr(_stp_log_pcpu, raw_smp_processor_id());
217 __stp_print_flush(log);
218 _stp_print_unlock_irqrestore(&flags);
219 }
220
221 #ifndef STP_MAXBINARYARGS
222 #define STP_MAXBINARYARGS 127
223 #endif
224
225
226 /** Reserves space in the output buffer for direct I/O. Must be called with
227 * _stp_print_trylock_irqsave() held.
228 */
229 static void * _stp_reserve_bytes (int numbytes)
230 {
231 struct _stp_log *log;
232 char *ret;
233
234 if (unlikely(numbytes == 0 || numbytes > STP_BUFFER_SIZE))
235 return NULL;
236
237 log = per_cpu_ptr(_stp_log_pcpu, raw_smp_processor_id());
238 /* _stp_print_trylock_irqsave already checks log->buf != NULL */
239
240 if (unlikely(numbytes > (STP_BUFFER_SIZE - log->len)))
241 __stp_print_flush(log);
242
243 if (log->is_full)
244 return NULL;
245
246 ret = &log->buf[log->len];
247 log->len += numbytes;
248 return ret;
249 }
250
251
252 static void _stp_unreserve_bytes (int numbytes)
253 {
254 struct _stp_log *log;
255
256 log = per_cpu_ptr(_stp_log_pcpu, raw_smp_processor_id());
257
258 /* We have to check log->buf == NULL here since the online CPUs may
259 * change at any time through the CPU hotplug feature of the kernel.
260 * So the current CPU's log->buf could be NULL if it is a new online
261 * CPU.
262 */
263 if (unlikely(log->buf == NULL))
264 return;
265
266 if (numbytes <= log->len)
267 log->len -= numbytes;
268 }
269
270 /** Write 64-bit args directly into the output stream.
271 * This function takes a variable number of 64-bit arguments
272 * and writes them directly into the output stream. Marginally faster
273 * than doing the same in _stp_vsnprintf().
274 * @sa _stp_vsnprintf()
275 */
276 static void _stp_print_binary (int num, ...)
277 {
278 unsigned long flags;
279 va_list vargs;
280 int i;
281 int64_t *args;
282
283 if (unlikely(num > STP_MAXBINARYARGS))
284 num = STP_MAXBINARYARGS;
285
286 if (!_stp_print_trylock_irqsave(&flags))
287 return;
288
289 args = _stp_reserve_bytes(num * sizeof(int64_t));
290 if (args) {
291 va_start(vargs, num);
292 for (i = 0; i < num; i++)
293 args[i] = va_arg(vargs, int64_t);
294 va_end(vargs);
295 }
296 _stp_print_unlock_irqrestore(&flags);
297 }
298
299 /** Print into the print buffer.
300 * Like C printf.
301 *
302 * @sa _stp_print_flush()
303 */
304 static void _stp_printf (const char *fmt, ...)
305 {
306 va_list args;
307
308 va_start(args, fmt);
309 _stp_vsnprintf(NULL, 0, fmt, args);
310 va_end(args);
311 }
312
313 /** Write a string into the print buffer.
314 * @param str A C string (char *)
315 */
316
317 static void _stp_print (const char *str)
318 {
319 struct _stp_log *log;
320 unsigned long flags;
321
322 if (!_stp_print_trylock_irqsave(&flags))
323 return;
324
325 log = per_cpu_ptr(_stp_log_pcpu, raw_smp_processor_id());
326 while (!log->is_full) {
327 while (log->len < STP_BUFFER_SIZE && *str)
328 log->buf[log->len++] = *str++;
329 if (likely(!*str))
330 break;
331 __stp_print_flush(log);
332 }
333 _stp_print_unlock_irqrestore(&flags);
334 }
335
336 static void _stp_print_char (const char c)
337 {
338 struct _stp_log *log;
339 unsigned long flags;
340
341 if (!_stp_print_trylock_irqsave(&flags))
342 return;
343
344 log = per_cpu_ptr(_stp_log_pcpu, raw_smp_processor_id());
345 if (unlikely(log->len == STP_BUFFER_SIZE))
346 __stp_print_flush(log);
347
348 if (likely(!log->no_flush || !log->is_full))
349 log->buf[log->len++] = c;
350
351 _stp_print_unlock_irqrestore(&flags);
352 }
353
354 static void _stp_print_kernel_info(char *sname, char *vstr, int ctx, int num_probes)
355 {
356 printk(KERN_DEBUG
357 "%s (%s): systemtap: %s, base: %lx"
358 ", memory: %ludata/%lutext/%uctx/%lunet/%lualloc kb"
359 ", probes: %d"
360 #if ! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPDEV)
361 ", unpriv-uid: %d"
362 #endif
363 "\n",
364 THIS_MODULE->name,
365 sname, /* name of source file */
366 vstr, /* stap version */
367 #ifdef STAPCONF_MODULE_LAYOUT
368 (unsigned long) THIS_MODULE->core_layout.base,
369 (unsigned long) (THIS_MODULE->core_layout.size - THIS_MODULE->core_layout.text_size)/1024,
370 (unsigned long) (THIS_MODULE->core_layout.text_size)/1024,
371 #else
372 #ifdef STAPCONF_MODULE_MEMORY
373 (unsigned long) THIS_MODULE->mem[MOD_TEXT].base,
374 (unsigned long) (THIS_MODULE->mem[MOD_DATA].size)/1024,
375 (unsigned long) (THIS_MODULE->mem[MOD_TEXT].size)/1024,
376 #else
377 #ifndef STAPCONF_GRSECURITY
378 (unsigned long) THIS_MODULE->module_core,
379 (unsigned long) (THIS_MODULE->core_size - THIS_MODULE->core_text_size)/1024,
380 (unsigned long) (THIS_MODULE->core_text_size)/1024,
381 #else
382 (unsigned long) THIS_MODULE->module_core_rx,
383 (unsigned long) (THIS_MODULE->core_size_rw - THIS_MODULE->core_size_rx)/1024,
384 (unsigned long) (THIS_MODULE->core_size_rx)/1024,
385 #endif
386 #endif
387 #endif
388 ctx/1024,
389 _stp_allocated_net_memory/1024,
390 (_stp_allocated_memory - _stp_allocated_net_memory - ctx)/1024,
391 /* (un-double-counting net/ctx because they're also stp_alloc'd) */
392 num_probes
393 #if ! STP_PRIVILEGE_CONTAINS (STP_PRIVILEGE, STP_PR_STAPDEV)
394 , _stp_uid
395 #endif
396 );
397 }
398
399 /** @} */
400 #endif /* _STAPLINUX_PRINT_C_ */