Debian Codegraph @

runtime/transport/relay_v2.c

raw
1 /* -*- linux-c -*-
2 *
3 * This transport version uses relayfs on top of a debugfs or procfs
4 * file. This code started as a proposed relayfs interface called
5 * 'utt'. It has been modified and simplified for systemtap.
6 *
7 * Changes Copyright (C) 2009-2020 Red Hat Inc.
8 *
9 * Original utt code by:
10 * Copyright (C) 2006 Jens Axboe <[email protected]>
11 * Moved to utt.c by Tom Zanussi, 2006
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License version 2 as
15 * published by the Free Software Foundation.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program. If not, see <http://www.gnu.org/licenses/>.
24 *
25 */
26 #include <linux/kernel.h>
27 #include <linux/module.h>
28 #include <linux/percpu.h>
29 #include <linux/init.h>
30 #include <linux/debugfs.h>
31 #include <linux/mm.h>
32 #include "../linux/timer_compatibility.h"
33 #include "../uidgid_compatibility.h"
34 #include "relay_compat.h"
35
36 #ifndef STP_RELAY_TIMER_INTERVAL
37 /* Wakeup timer interval in jiffies (default 1 ms) */
38 #define STP_RELAY_TIMER_INTERVAL ((HZ + 999) / 1000)
39 #endif
40
41 /* Note: if struct _stp_relay_data_type changes, staplog.c might need
42 * to be changed. */
43 struct _stp_relay_data_type {
44 struct rchan *rchan;
45 atomic_t /* enum _stp_transport_state */ transport_state;
46 atomic_t wakeup;
47 struct timer_list timer;
48 int overwrite_flag;
49 };
50 struct _stp_relay_data_type _stp_relay_data;
51
52 /* relay_file_operations is const, so .owner is obviously not set there.
53 * Below struct, filled in _stp_transport_data_fs_init(), fixes it. */
54 static struct file_operations relay_file_operations_w_owner;
55
56 /*
57 * __stp_relay_switch_subbuf - switch to a new sub-buffer
58 *
59 * Most of this function is deadcopy of relay_switch_subbuf.
60 */
61 static size_t __stp_relay_switch_subbuf(struct rchan_buf *buf, size_t length)
62 {
63 char *old, *new;
64 size_t old_subbuf, new_subbuf;
65 #if !defined(STAPCONF_PR33324_PREV_PADDING)
66 size_t prev_padding;
67 #endif
68
69 if (unlikely(buf == NULL))
70 return 0;
71
72 if (unlikely(length > buf->chan->subbuf_size))
73 length = buf->chan->subbuf_size;
74
75 if (buf->offset != buf->chan->subbuf_size + 1) {
76 #if defined(STAPCONF_PR33324_PREV_PADDING)
77 buf->prev_padding = buf->chan->subbuf_size - buf->offset;
78 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
79 buf->padding[old_subbuf] = buf->prev_padding;
80 #else
81 prev_padding = buf->chan->subbuf_size - buf->offset;
82 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
83 buf->padding[old_subbuf] = prev_padding;
84 #endif
85 buf->subbufs_produced++;
86 buf->dentry->d_inode->i_size += buf->chan->subbuf_size -
87 buf->padding[old_subbuf];
88 smp_mb();
89 }
90
91 old = buf->data;
92 new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
93 new = (char*)buf->start + new_subbuf * buf->chan->subbuf_size;
94 buf->offset = 0;
95 #if defined(STAPCONF_PR33324_PREV_PADDING)
96 if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {
97 #else
98 if (!buf->chan->cb->subbuf_start(buf, new, old)) {
99 #endif
100 buf->offset = buf->chan->subbuf_size + 1;
101 return 0;
102 }
103 buf->data = new;
104 buf->padding[new_subbuf] = 0;
105
106 return length;
107 }
108
109 static bool __stp_relay_buf_empty(struct rchan_buf *buf)
110 {
111 return !buf->offset && buf->subbufs_produced == buf->subbufs_consumed;
112 }
113
114 static int __stp_relay_file_open(struct inode *inode, struct file *filp)
115 {
116 struct rchan_buf *buf = inode->i_private;
117
118 /* kernel's relay_file_open() does not check NULL buf (at least as of
119 * Sep 9, 2022) and there is a race window in our
120 * __stp_procfs_relay_create_buf_file_callback() function,
121 * between proc_create() and the `in->i_private = buf` assignment.
122 * without this check, we can reproduce a NULL ptr deref kernel panic
123 * while loading a system with a lot of short-lived stap sessions.
124 */
125 if (unlikely(buf == NULL)) {
126 dbug_trans(0, "relay_file_open: found NULL rchan buf\n");
127 return -ENOENT;
128 }
129
130 return relay_file_operations.open(inode, filp);
131 }
132
133 static unsigned int __stp_relay_file_poll(struct file *filp, poll_table *wait)
134 {
135 struct rchan_buf *buf = filp->private_data;
136
137 if (buf->finalized)
138 return POLLERR;
139
140 if (filp->f_mode & FMODE_READ) {
141 poll_wait(filp, &buf->read_wait, wait);
142 if (!__stp_relay_buf_empty(buf))
143 return POLLIN | POLLRDNORM;
144 }
145
146 return 0;
147 }
148
149 static ssize_t __stp_relay_file_read(struct file *filp, char __user *buffer,
150 size_t count, loff_t *ppos)
151 {
152 struct rchan_buf *buf = filp->private_data;
153 unsigned long flags;
154
155 /* This trylock will only fail after the print driver is destroyed */
156 if (_stp_print_trylock_irqsave(&flags)) {
157 /* Switch out the current buffer if it has any data */
158 if (buf->offset)
159 __stp_relay_switch_subbuf(buf, 0);
160 _stp_print_unlock_irqrestore(&flags);
161 }
162
163 /*
164 * Optimization: bail out early if there's nothing to read. This is
165 * faster than going into relay's read() function without having any
166 * data, as it will acquire a mutex lock for the inode before bailing.
167 */
168 if (buf->subbufs_produced == buf->subbufs_consumed)
169 return -ENODATA;
170
171 return relay_file_operations.read(filp, buffer, count, ppos);
172 }
173
174 static void __stp_relay_wakeup_readers(struct rchan_buf *buf)
175 {
176 if (buf && waitqueue_active(&buf->read_wait) &&
177 !__stp_relay_buf_empty(buf))
178 wake_up_interruptible(&buf->read_wait);
179 }
180
181 static void __stp_relay_wakeup_timer(stp_timer_callback_parameter_t unused)
182 {
183 int i;
184
185 if (atomic_cmpxchg(&_stp_relay_data.wakeup, 1, 0)) {
186 struct rchan_buf *buf;
187
188 /* NB it makes no sense to wake up readers on offline CPUs */
189 for_each_online_cpu(i) {
190 buf = _stp_get_rchan_subbuf(_stp_relay_data.rchan->buf,
191 i);
192
193 /* relay_open() only initializes bufs on the online CPUs
194 * at the time of invocation. The online CPUs may
195 * change at any time, so the current online CPU might
196 * be offline when relay_open() was called. so we must
197 * check if the buf for the current CPU is invalid
198 * otherwise we may dereference a NULL pointer.
199 */
200 if (likely(buf))
201 __stp_relay_wakeup_readers(buf);
202 }
203 }
204
205 if (atomic_read(&_stp_relay_data.transport_state) == STP_TRANSPORT_RUNNING)
206 mod_timer(&_stp_relay_data.timer, jiffies + STP_RELAY_TIMER_INTERVAL);
207 else
208 dbug_trans(0, "relay_v2 wakeup timer expiry\n");
209 }
210
211 static void __stp_relay_timer_init(void)
212 {
213 atomic_set(&_stp_relay_data.wakeup, 0);
214 timer_setup(&_stp_relay_data.timer, __stp_relay_wakeup_timer, 0);
215 _stp_relay_data.timer.expires = jiffies + STP_RELAY_TIMER_INTERVAL;
216 add_timer(&_stp_relay_data.timer);
217 smp_mb();
218 }
219
220 static enum _stp_transport_state _stp_transport_get_state(void)
221 {
222 return atomic_read (&_stp_relay_data.transport_state);
223 }
224
225 static void _stp_transport_data_fs_overwrite(int overwrite)
226 {
227 _stp_relay_data.overwrite_flag = overwrite;
228 }
229
230
231 /*
232 * Keep track of how many times we encountered a full subbuffer, to aid
233 * the user space app in telling how many lost events there were.
234 */
235 #if defined(STAPCONF_PR33324_PREV_PADDING)
236 static int __stp_relay_subbuf_start_callback(struct rchan_buf *buf,
237 void *subbuf, void *prev_subbuf,
238 size_t prev_padding)
239 #else
240 static int __stp_relay_subbuf_start_callback(struct rchan_buf *buf,
241 void *subbuf, void *prev_subbuf)
242 #endif
243 {
244 if (_stp_relay_data.overwrite_flag || !relay_buf_full(buf))
245 return 1;
246
247 #ifdef _STP_USE_DROPPED_FILE
248 atomic_inc(&_stp_relay_data.dropped);
249 #endif
250 return 0;
251 }
252
253
254 // PR26665: demultiplex debugfs vs procfs host
255
256 static int __stp_relay_remove_buf_file_callback(struct dentry *dentry)
257 {
258 if (debugfs_p)
259 return __stp_debugfs_relay_remove_buf_file_callback(dentry);
260 if (procfs_p)
261 return __stp_procfs_relay_remove_buf_file_callback(dentry);
262 return 0;
263 }
264
265
266 static struct dentry *
267 __stp_relay_create_buf_file_callback(const char *filename,
268 struct dentry *parent,
269 #ifdef STAPCONF_RELAY_UMODE_T
270 umode_t mode,
271 #else
272 int mode,
273 #endif
274 struct rchan_buf *buf,
275 int *is_global)
276 {
277 if (debugfs_p)
278 return __stp_debugfs_relay_create_buf_file_callback(filename, parent, mode, buf, is_global);
279 if (procfs_p)
280 return __stp_procfs_relay_create_buf_file_callback(filename, parent, mode, buf, is_global);
281 return NULL;
282 }
283
284
285 static struct rchan_callbacks __stp_relay_callbacks = {
286 .subbuf_start = __stp_relay_subbuf_start_callback,
287 .create_buf_file = __stp_relay_create_buf_file_callback,
288 .remove_buf_file = __stp_relay_remove_buf_file_callback,
289 };
290
291
292 static void _stp_transport_data_fs_start(void)
293 {
294 if (atomic_read (&_stp_relay_data.transport_state) == STP_TRANSPORT_INITIALIZED) {
295 atomic_set (&_stp_relay_data.transport_state, STP_TRANSPORT_RUNNING);
296 /* We're initialized. Now start the timer. */
297 __stp_relay_timer_init();
298 }
299 }
300
301 static void _stp_transport_data_fs_stop(void)
302 {
303
304 if (atomic_read (&_stp_relay_data.transport_state) == STP_TRANSPORT_RUNNING) {
305 atomic_set (&_stp_relay_data.transport_state, STP_TRANSPORT_STOPPED);
306 STP_TIMER_DELETE_SYNC(&_stp_relay_data.timer);
307 dbug_trans(0, "flushing...\n");
308 if (_stp_relay_data.rchan)
309 relay_flush(_stp_relay_data.rchan);
310 }
311 }
312
313 static void _stp_transport_data_fs_close(void)
314 {
315 _stp_transport_data_fs_stop();
316 if (_stp_relay_data.rchan) {
317 relay_close(_stp_relay_data.rchan);
318 _stp_relay_data.rchan = NULL;
319 }
320 }
321
322 static int _stp_transport_data_fs_init(void)
323 {
324 int rc;
325 u64 npages;
326 struct sysinfo si;
327
328 atomic_set(&_stp_relay_data.transport_state, STP_TRANSPORT_STOPPED);
329 _stp_relay_data.overwrite_flag = 0;
330 _stp_relay_data.rchan = NULL;
331
332 /* Create "trace" file. */
333 npages = _stp_subbuf_size * _stp_nsubbufs;
334 npages *= num_online_cpus();
335 npages >>= PAGE_SHIFT;
336 si_meminfo(&si);
337 #define MB(i) (unsigned long)((i) >> (20 - PAGE_SHIFT))
338 if (npages > (si.freeram + si.bufferram)) {
339 errk("Not enough free+buffered memory(%luMB) for log buffer(%luMB)\n",
340 MB(si.freeram + si.bufferram),
341 MB(npages));
342 rc = -ENOMEM;
343 goto err;
344 }
345 else if (npages > si.freeram) {
346 /* exceeds freeram, but below freeram+bufferram */
347 printk(KERN_WARNING
348 "log buffer size exceeds free memory(%luMB)\n",
349 MB(si.freeram));
350 }
351 relay_file_operations_w_owner = relay_file_operations;
352 relay_file_operations_w_owner.owner = THIS_MODULE;
353 relay_file_operations_w_owner.poll = __stp_relay_file_poll;
354 relay_file_operations_w_owner.read = __stp_relay_file_read;
355 #if (RELAYFS_CHANNEL_VERSION >= 7)
356 _stp_relay_data.rchan = relay_open("trace", _stp_get_module_dir(),
357 _stp_subbuf_size, _stp_nsubbufs,
358 &__stp_relay_callbacks, NULL);
359 #else /* (RELAYFS_CHANNEL_VERSION < 7) */
360 _stp_relay_data.rchan = relay_open("trace", _stp_get_module_dir(),
361 _stp_subbuf_size, _stp_nsubbufs,
362 &__stp_relay_callbacks);
363 #endif /* (RELAYFS_CHANNEL_VERSION < 7) */
364 if (!_stp_relay_data.rchan) {
365 rc = -ENOENT;
366 errk("%s: relay_open() failed: %d\n", THIS_MODULE->name, rc);
367 goto err;
368 }
369 /* Increment _stp_allocated_memory and _stp_allocated_net_memory to account for buffers
370 allocated by relay_open. */
371 {
372 u64 relay_mem;
373
374 /* relay_open() invokes relay_open_buf() to allocate this part
375 * of memory for online CPUs only.
376 */
377 relay_mem = _stp_subbuf_size * _stp_nsubbufs;
378 relay_mem *= num_online_cpus();
379
380 relay_mem += sizeof(struct rchan);
381
382 if (sizeof(_stp_relay_data.rchan->buf) == sizeof(struct rchan_buf *)) {
383 /* newer kernels (newer than 3.10, for example) */
384 /* this part was allocated by alloc_percpu() inside
385 * relay_open(). */
386 relay_mem += sizeof(struct rchan_buf *) * num_possible_cpus();
387
388 } else {
389 /* for older kernels like 3.10, the buf pointers are
390 * already part of struct rchan as its array member
391 * 'struct rchan_buf *buf[NR_CPUS]'. so we do nothing
392 * here.
393 */
394 }
395
396 _stp_allocated_net_memory += relay_mem;
397 _stp_allocated_memory += relay_mem;
398 }
399
400 dbug_trans(1, "returning 0...\n");
401 atomic_set (&_stp_relay_data.transport_state, STP_TRANSPORT_INITIALIZED);
402
403 return 0;
404
405 err:
406 _stp_transport_data_fs_close();
407 return rc;
408 }
409
410
411 /**
412 * _stp_data_write_reserve - try to reserve size_request bytes
413 * @size_request: number of bytes to attempt to reserve
414 * @entry: entry is returned here
415 *
416 * Returns number of bytes reserved, 0 if full. On return, entry
417 * will point to allocated opaque pointer. Use
418 * _stp_data_entry_data() to get pointer to copy data into.
419 *
420 * (For this code's purposes, entry is filled in with the actual
421 * data pointer, but the caller doesn't know that.)
422 */
423 static size_t
424 _stp_data_write_reserve(size_t size_request, void **entry)
425 {
426 struct rchan_buf *buf;
427
428 if (entry == NULL)
429 return -EINVAL;
430
431 buf = _stp_get_rchan_subbuf(_stp_relay_data.rchan->buf,
432 smp_processor_id());
433
434 /* relay_open() only initializes bufs on the online CPUs at the time
435 * of invocation. The online CPUs may change at any time, so the
436 * current online CPU might be offline when relay_open() was called.
437 * So we must check if the buf for the current CPU is invalid otherwise
438 * we may dereference a NULL pointer.
439 */
440 if (unlikely(buf == NULL))
441 return -EINVAL;
442
443 if (buf->offset >= buf->chan->subbuf_size) {
444 size_request = __stp_relay_switch_subbuf(buf, size_request);
445 if (!size_request)
446 return 0;
447 } else if (buf->offset + size_request > buf->chan->subbuf_size) {
448 size_request = buf->chan->subbuf_size - buf->offset;
449 }
450 *entry = (char*)buf->data + buf->offset;
451 buf->offset += size_request;
452
453 return size_request;
454 }
455
456 static unsigned char *_stp_data_entry_data(void *entry)
457 {
458 /* Nothing to do here. */
459 return entry;
460 }
461
462 static int _stp_data_write_commit(void *entry)
463 {
464 atomic_set(&_stp_relay_data.wakeup, 1);
465 return 0;
466 }