Debian Codegraph @

runtime/transport/control.c

raw
1 /* -*- linux-c -*-
2 *
3 * control channel
4 * Copyright (C) 2007-2018 Red Hat Inc.
5 *
6 * This file is part of systemtap, and is free software. You can
7 * redistribute it and/or modify it under the terms of the GNU General
8 * Public License (GPL); either version 2, or (at your option) any
9 * later version.
10 */
11
12 #include "control.h"
13 #include "../mempool.c"
14 #include "symbols.c"
15 #include <linux/delay.h>
16 #include <linux/poll.h>
17 #include "../uidgid_compatibility.h"
18
19 static _stp_mempool_t *_stp_pool_q;
20 static struct list_head _stp_ctl_ready_q;
21 static STP_DEFINE_SPINLOCK(_stp_ctl_ready_lock);
22 static STP_DEFINE_SPINLOCK(_stp_ctl_special_msg_lock);
23
24 static void _stp_cleanup_and_exit(int send_exit);
25 static void _stp_handle_tzinfo (struct _stp_msg_tzinfo* tzi);
26 static void _stp_handle_privilege_credentials (struct _stp_msg_privilege_credentials* pc);
27 static void _stp_handle_remote_id (struct _stp_msg_remote_id* rem);
28 static void _stp_handle_namespaces_pid (struct _stp_msg_ns_pid *nspid);
29 static void _stp_handle_mnt_ns_fds (struct _stp_msg_mnt_ns_fds *nsfds);
30
31
32 static ssize_t _stp_ctl_write_cmd(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
33 {
34 static DEFINE_MUTEX(cmd_mutex);
35 u32 type;
36 int rc = 0;
37
38 #ifdef STAPCONF_TASK_UID
39 uid_t euid = current->euid;
40 #else
41 #if defined(CONFIG_USER_NS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(3,14,0))
42 uid_t euid = from_kuid_munged(current_user_ns(), current_euid());
43 #else
44 uid_t euid = current_euid();
45 #endif
46 #endif
47
48 _stp_pid = current->pid;
49
50 if (count < sizeof(u32))
51 return 0;
52
53 if (get_user(type, (u32 __user *)buf))
54 return -EFAULT;
55
56 count -= sizeof(u32);
57 buf += sizeof(u32);
58
59 #if defined(DEBUG_TRANS) && (DEBUG_TRANS >= 2)
60 if (type < STP_MAX_CMD)
61 dbug_trans2("Got %s. euid=%ld, len=%d\n", _stp_command_name[min(type, (u32)STP_MAX_CMD)] ?: "?",
62 (long)euid, (int)count);
63 #endif
64
65 // PR17232: preclude reentrancy during handling of messages.
66 // This also permits use of static variables in the switch/case.
67 might_sleep();
68 mutex_lock (& cmd_mutex);
69 // NB: past this point, no 'return;' - use 'goto out;'
70
71 switch (type) {
72 case STP_START:
73 {
74 static struct _stp_msg_start st;
75 if (count < sizeof(st)) {
76 rc = 0; // ?
77 goto out;
78 }
79 if (copy_from_user(&st, buf, sizeof(st))) {
80 rc = -EFAULT;
81 goto out;
82 }
83 _stp_handle_start(&st);
84 }
85 break;
86
87 case STP_EXIT:
88 _stp_cleanup_and_exit(1);
89 break;
90
91 case STP_BULK:
92 // NB: this signals the runtime to save separate
93 // per-cpu files; our kernel->userspace files are now
94 // always bulkmode (trace$N files).
95 #ifdef STP_BULKMODE
96 // no action needed
97 break;
98 #else
99 rc = -EINVAL;
100 goto out;
101 #endif
102
103 case STP_RELOCATION:
104 if (euid != 0) {
105 rc = -EPERM;
106 goto out;
107 }
108 /* This message is too large to copy here.
109 Further error checking is within the
110 function, but XXX no rc is passed back. */
111 _stp_do_relocation (buf, count);
112 break;
113
114 case STP_TZINFO:
115 {
116 /* NB PR13445: We use a static struct here to contain
117 the kernel-side copy of the user-space message.
118 This should be suspicious (due to concurrency
119 concerns), but actually it's OK. The main reason
120 is that _stp_ctl_open_cmd() enforces only a single
121 open() at a time on the .ctl file, and staprun
122 (euid=0) isn't multithreaded, and doesn't pass this
123 filehandle anywhere. */
124 static struct _stp_msg_tzinfo tzi;
125 if (euid != 0) {
126 rc = -EPERM;
127 goto out;
128 }
129 if (count < sizeof(tzi)) {
130 rc = 0;
131 goto out;
132 }
133 if (copy_from_user(&tzi, buf, sizeof(tzi))) {
134 rc = -EFAULT;
135 goto out;
136 }
137 _stp_handle_tzinfo(&tzi);
138 }
139 break;
140
141 case STP_PRIVILEGE_CREDENTIALS:
142 {
143 /* NB PR13445: as above. */
144 static struct _stp_msg_privilege_credentials pc;
145 if (euid != 0) {
146 rc = -EPERM;
147 goto out;
148 }
149 if (count < sizeof(pc)) {
150 rc = 0;
151 goto out;
152 }
153 if (copy_from_user(&pc, buf, sizeof(pc))) {
154 rc = -EFAULT;
155 goto out;
156 }
157 _stp_handle_privilege_credentials(&pc);
158 }
159 break;
160
161 case STP_REMOTE_ID:
162 {
163 /* NB PR13445: as above. */
164 static struct _stp_msg_remote_id rem;
165 if (euid != 0) {
166 rc = -EPERM;
167 goto out;
168 }
169 if (count < sizeof(rem)) {
170 rc = 0;
171 goto out;
172 }
173 if (copy_from_user(&rem, buf, sizeof(rem))) {
174 rc = -EFAULT;
175 goto out;
176 }
177 _stp_handle_remote_id(&rem);
178 }
179 break;
180
181 case STP_READY:
182 break;
183 case STP_NAMESPACES_PID:
184 {
185 static struct _stp_msg_ns_pid nspid;
186 if (count < sizeof(nspid)) {
187 rc = 0;
188 goto out;
189 }
190 if (copy_from_user(&nspid, buf, sizeof(nspid))) {
191 rc = -EFAULT;
192 goto out;
193 }
194
195 _stp_handle_namespaces_pid(&nspid);
196 }
197 break;
198
199 case STP_MNT_NS_FDS:
200 {
201 static struct _stp_msg_mnt_ns_fds nsfds;
202 if (count < sizeof(nsfds)) {
203 rc = 0;
204 goto out;
205 }
206 if (copy_from_user(&nsfds, buf, sizeof(nsfds))) {
207 rc = -EFAULT;
208 goto out;
209 }
210
211 _stp_handle_mnt_ns_fds(&nsfds);
212 }
213 break;
214
215 default:
216 #ifdef DEBUG_TRANS
217 dbug_trans2("invalid command type %d\n", type);
218 #endif
219 rc = -EINVAL;
220 goto out;
221 }
222
223 // fall through
224 rc = count + sizeof(u32); /* Pretend that we absorbed the entire message. */
225
226 out:
227 mutex_unlock (& cmd_mutex);
228
229 #if defined(DEBUG_TRANS) && (DEBUG_TRANS >= 2)
230 if (type < STP_MAX_CMD)
231 dbug_trans2("Completed %s (rc=%d)\n",
232 _stp_command_name[min(type, (u32)STP_MAX_CMD)] ?: "?",
233 rc);
234 #endif
235 return rc;
236 }
237
238 static DECLARE_WAIT_QUEUE_HEAD(_stp_ctl_wq);
239
240 #ifdef DEBUG_TRANS
241 static void _stp_ctl_write_dbug(int type, void *data, int len)
242 {
243 char buf[64];
244 switch (type) {
245 case STP_START:
246 dbug_trans2("sending STP_START\n");
247 break;
248 case STP_EXIT:
249 dbug_trans2("sending STP_EXIT\n");
250 break;
251 case STP_OOB_DATA:
252 snprintf(buf, sizeof(buf), "%s", (char *)data);
253 dbug_trans2("sending %d bytes of STP_OOB_DATA: %s\n", len,
254 buf);
255 break;
256 case STP_SYSTEM:
257 snprintf(buf, sizeof(buf), "%s", (char *)data);
258 dbug_trans2("sending STP_SYSTEM: %s\n", buf);
259 break;
260 case STP_TRANSPORT:
261 dbug_trans2("sending STP_TRANSPORT\n");
262 break;
263 case STP_CONNECT:
264 dbug_trans2("sending STP_CONNECT\n");
265 break;
266 case STP_DISCONNECT:
267 dbug_trans2("sending STP_DISCONNECT\n");
268 break;
269 case STP_BULK:
270 dbug_trans2("sending STP_BULK\n");
271 break;
272 case STP_READY:
273 case STP_RELOCATION:
274 case STP_BUF_INFO:
275 case STP_SUBBUFS_CONSUMED:
276 dbug_trans2("sending old message\n");
277 break;
278 case STP_REALTIME_DATA:
279 dbug_trans2("sending %d bytes of STP_REALTIME_DATA\n", len);
280 break;
281 case STP_REQUEST_EXIT:
282 dbug_trans2("sending STP_REQUEST_EXIT\n");
283 break;
284 default:
285 dbug_trans2("ERROR: unknown message type: %d\n", type);
286 break;
287 }
288 }
289 #endif
290
291 /* Marker to show a "special" message buffer isn't being used.
292 Will be put in the _stp_buffer type field. The type field Should
293 only be manipulated while holding the _stp_ctl_special_msg_lock. */
294 #define _STP_CTL_MSG_UNUSED STP_MAX_CMD
295
296 /* cmd messages allocated ahead of time. There can be only one. */
297 static struct _stp_buffer *_stp_ctl_start_msg;
298 static struct _stp_buffer *_stp_ctl_exit_msg;
299 static struct _stp_buffer *_stp_ctl_transport_msg;
300 static struct _stp_buffer *_stp_ctl_request_exit_msg;
301
302 /* generic overflow messages allocated ahread of time. */
303 static struct _stp_buffer *_stp_ctl_oob_warn;
304 static struct _stp_buffer *_stp_ctl_oob_err;
305 static struct _stp_buffer *_stp_ctl_system_warn;
306 static struct _stp_buffer *_stp_ctl_realtime_err;
307
308 /* Set aside buffers for all "special" message types, plus generic
309 warning and error messages. */
310 static int _stp_ctl_alloc_special_buffers(void)
311 {
312 size_t len;
313 const char *msg;
314
315 /* There can be only one of start, exit, transport and request. */
316 _stp_ctl_start_msg = _stp_mempool_alloc(_stp_pool_q);
317 if (_stp_ctl_start_msg == NULL)
318 return -1;
319 _stp_ctl_start_msg->type = _STP_CTL_MSG_UNUSED;
320
321 _stp_ctl_exit_msg = _stp_mempool_alloc(_stp_pool_q);
322 if (_stp_ctl_exit_msg == NULL)
323 return -1;
324 _stp_ctl_exit_msg->type = _STP_CTL_MSG_UNUSED;
325
326 _stp_ctl_transport_msg = _stp_mempool_alloc(_stp_pool_q);
327 if (_stp_ctl_transport_msg == NULL)
328 return -1;
329 _stp_ctl_transport_msg->type = _STP_CTL_MSG_UNUSED;
330
331 _stp_ctl_request_exit_msg = _stp_mempool_alloc(_stp_pool_q);
332 if (_stp_ctl_request_exit_msg == NULL)
333 return -1;
334 _stp_ctl_request_exit_msg->type = _STP_CTL_MSG_UNUSED;
335
336 /* oob_warn, oob_err, system and realtime are dynamically
337 allocated and a special static warn/err message take their
338 place if we run out of memory before delivery. */
339 _stp_ctl_oob_warn = _stp_mempool_alloc(_stp_pool_q);
340 if (_stp_ctl_oob_warn == NULL)
341 return -1;
342 _stp_ctl_oob_warn->type = _STP_CTL_MSG_UNUSED;
343 /* Note that the following message shouldn't be translated,
344 * since "WARNING:" is part of the module cmd protocol. */
345 msg = "WARNING: too many pending (warning) messages\n";
346 len = strlen(msg) + 1;
347 _stp_ctl_oob_warn->len = len;
348 memcpy(&_stp_ctl_oob_warn->buf, msg, len);
349
350 _stp_ctl_oob_err = _stp_mempool_alloc(_stp_pool_q);
351 if (_stp_ctl_oob_err == NULL)
352 return -1;
353 _stp_ctl_oob_err->type = _STP_CTL_MSG_UNUSED;
354 /* Note that the following message shouldn't be translated,
355 * since "ERROR:" is part of the module cmd protocol. */
356 msg = "ERROR: too many pending (error) messages\n";
357 len = strlen(msg) + 1;
358 _stp_ctl_oob_err->len = len;
359 memcpy(&_stp_ctl_oob_err->buf, msg, len);
360
361 _stp_ctl_system_warn = _stp_mempool_alloc(_stp_pool_q);
362 if (_stp_ctl_system_warn == NULL)
363 return -1;
364 _stp_ctl_system_warn->type = _STP_CTL_MSG_UNUSED;
365 /* Note that the following message shouldn't be translated,
366 * since "WARNING:" is part of the module cmd protocol. */
367 msg = "WARNING: too many pending (system) messages\n";
368 len = strlen(msg) + 1;
369 _stp_ctl_system_warn->len = len;
370 memcpy(&_stp_ctl_system_warn->buf, msg, len);
371
372 _stp_ctl_realtime_err = _stp_mempool_alloc(_stp_pool_q);
373 if (_stp_ctl_realtime_err == NULL)
374 return -1;
375 _stp_ctl_realtime_err->type = _STP_CTL_MSG_UNUSED;
376 /* Note that the following message shouldn't be translated,
377 * since "ERROR:" is part of the module cmd protocol. */
378 msg = "ERROR: too many pending (realtime) messages\n";
379 len = strlen(msg) + 1;
380 _stp_ctl_realtime_err->len = len;
381 memcpy(&_stp_ctl_realtime_err->buf, msg, len);
382
383 return 0;
384 }
385
386
387 /* Free the buffers for all "special" message types, plus generic
388 warning and error messages. */
389 static void _stp_ctl_free_special_buffers(void)
390 {
391 if (_stp_ctl_start_msg != NULL) {
392 _stp_mempool_free(_stp_ctl_start_msg);
393 _stp_ctl_start_msg = NULL;
394 }
395
396 if (_stp_ctl_exit_msg != NULL) {
397 _stp_mempool_free(_stp_ctl_exit_msg);
398 _stp_ctl_exit_msg = NULL;
399 }
400
401 if (_stp_ctl_transport_msg != NULL) {
402 _stp_mempool_free(_stp_ctl_transport_msg);
403 _stp_ctl_transport_msg = NULL;
404 }
405
406 if (_stp_ctl_request_exit_msg != NULL) {
407 _stp_mempool_free(_stp_ctl_request_exit_msg);
408 _stp_ctl_request_exit_msg = NULL;
409 }
410
411 if (_stp_ctl_oob_warn != NULL) {
412 _stp_mempool_free(_stp_ctl_oob_warn);
413 _stp_ctl_oob_warn = NULL;
414 }
415
416 if (_stp_ctl_oob_err != NULL) {
417 _stp_mempool_free(_stp_ctl_oob_err);
418 _stp_ctl_oob_err = NULL;
419 }
420
421 if (_stp_ctl_system_warn != NULL) {
422 _stp_mempool_free(_stp_ctl_system_warn);
423 _stp_ctl_system_warn = NULL;
424 }
425
426 if (_stp_ctl_realtime_err != NULL) {
427 _stp_mempool_free(_stp_ctl_realtime_err);
428 _stp_ctl_realtime_err = NULL;
429 }
430 }
431
432
433 /* Get a buffer based on type, possibly a generic buffer, when all else
434 fails returns NULL and there is nothing we can do. */
435 static struct _stp_buffer *_stp_ctl_get_buffer(int type, const char *data,
436 unsigned len)
437 {
438 unsigned long flags = 0;
439 struct _stp_buffer *bptr = NULL;
440
441 /* Is it a dynamically allocated message type? */
442 if (type == STP_OOB_DATA
443 || type == STP_SYSTEM
444 || type == STP_REALTIME_DATA)
445 bptr = _stp_mempool_alloc(_stp_pool_q);
446
447 if (bptr != NULL) {
448 bptr->type = type;
449 memcpy(bptr->buf, data, len);
450 bptr->len = len;
451 } else {
452 /* "special" type, or no more dynamic buffers.
453 We must be careful to lock to avoid races between
454 marking as used/free. There can be only one. */
455 switch (type) {
456 case STP_START:
457 bptr = _stp_ctl_start_msg;
458 break;
459 case STP_EXIT:
460 bptr = _stp_ctl_exit_msg;
461 break;
462 case STP_NAMESPACES_PID:
463 break;
464 case STP_TRANSPORT:
465 bptr = _stp_ctl_transport_msg;
466 break;
467 case STP_REQUEST_EXIT:
468 bptr = _stp_ctl_request_exit_msg;
469 break;
470 case STP_OOB_DATA:
471 /* Note that "WARNING:" should not be
472 * translated, since it is part of the module
473 * cmd protocol. */
474 if (data && len >= 9
475 && strncmp(data, "WARNING: ", 9) == 0)
476 bptr = _stp_ctl_oob_warn;
477 /* Note that "ERROR:" should not be
478 * translated, since it is part of the module
479 * cmd protocol. */
480 else if (data && len >= 7
481 && strncmp(data, "ERROR: ", 7) == 0)
482 bptr = _stp_ctl_oob_err;
483 else
484 printk(KERN_WARNING "_stp_ctl_get_buffer unexpected STP_OOB_DATA\n");
485 break;
486 case STP_SYSTEM:
487 bptr = _stp_ctl_system_warn;
488 type = STP_OOB_DATA; /* overflow message */
489 break;
490 case STP_REALTIME_DATA:
491 bptr = _stp_ctl_realtime_err;
492 type = STP_OOB_DATA; /* overflow message */
493 break;
494 default:
495 printk(KERN_WARNING "_stp_ctl_get_buffer unknown type: %d\n", type);
496 bptr = NULL;
497 break;
498 }
499 if (bptr != NULL) {
500 /* OK, it is a special one, but is it free? */
501 stp_nmi_spin_lock_irqsave(&_stp_ctl_special_msg_lock, flags, failed);
502 if (bptr->type == _STP_CTL_MSG_UNUSED)
503 bptr->type = type;
504 else
505 bptr = NULL;
506 stp_nmi_spin_unlock_irqrestore(&_stp_ctl_special_msg_lock, flags);
507 }
508
509 /* Got a special message buffer, with type set, fill it in,
510 unless it is an "overflow" message. */
511 if (bptr != NULL
512 && bptr != _stp_ctl_oob_warn
513 && bptr != _stp_ctl_oob_err
514 && bptr != _stp_ctl_system_warn
515 && bptr != _stp_ctl_realtime_err) {
516 memcpy(bptr->buf, data, len);
517 bptr->len = len;
518 }
519 }
520 return bptr;
521
522 failed:
523 return NULL;
524 }
525
526 /* Returns the given buffer to the pool when dynamically allocated.
527 Marks special buffers as being unused. */
528 static void _stp_ctl_free_buffer(struct _stp_buffer *bptr)
529 {
530 unsigned long flags;
531
532 /* Special buffers need special care and locking. */
533 if (bptr == _stp_ctl_start_msg
534 || bptr == _stp_ctl_exit_msg
535 || bptr == _stp_ctl_transport_msg
536 || bptr == _stp_ctl_request_exit_msg
537 || bptr == _stp_ctl_oob_warn
538 || bptr == _stp_ctl_oob_err
539 || bptr == _stp_ctl_system_warn
540 || bptr == _stp_ctl_realtime_err) {
541 stp_spin_lock_irqsave(&_stp_ctl_special_msg_lock, flags);
542 bptr->type = _STP_CTL_MSG_UNUSED;
543 stp_spin_unlock_irqrestore(&_stp_ctl_special_msg_lock, flags);
544 } else {
545 _stp_mempool_free(bptr);
546 }
547 }
548
549 /* Put a message on the _stp_ctl_ready_q. Safe to call from a probe context.
550 Doesn't call wake_up on _stp_ctl_wq (which would not be safe from all
551 probe context). A timer will come by and pick up the message to notify
552 any readers. Returns the number of bytes queued/send or zero/negative
553 on error. */
554 static int _stp_ctl_send(int type, void *data, unsigned len)
555 {
556 struct context* __restrict__ c = NULL;
557 struct _stp_buffer *bptr;
558 unsigned long flags = 0;
559 unsigned hlen;
560
561 #ifdef DEBUG_TRANS
562 _stp_ctl_write_dbug(type, data, len);
563 #endif
564
565 /* Give the fs a chance to do something special.
566 Like merging two packets in case the previous buffer
567 still has some room (transport version 1 procfs does this. */
568 hlen = _stp_ctl_write_fs(type, data, len);
569 if (hlen > 0)
570 return hlen;
571
572 /* make sure we won't overflow the buffer */
573 if (unlikely(len > STP_CTL_BUFFER_SIZE)) {
574 _stp_warn("runtime control message type=%d len=%d too large\n", type, len);
575 return 0;
576 }
577
578 /* Prevent probe reentrancy while grabbing probe-used locks.
579 Since _stp_ctl_send may be called from arbitrary probe context, we
580 have to make sure that all locks it wants can't possibly be held
581 outside probe context too. This includes:
582 * _stp_ctl_ready_lock
583 * _stp_pool_q->lock
584 * _stp_ctl_special_msg_lock
585 We ensure this by grabbing the context here and everywhere else that
586 uses those locks, so such a probe will appear reentrant and be
587 skipped rather than deadlock. */
588 c = _stp_runtime_entryfn_get_context();
589
590 stp_nmi_spin_lock_irqsave(&_stp_ctl_ready_lock, flags, no_lock);
591
592 /* get a buffer from the free pool */
593 bptr = _stp_ctl_get_buffer(type, data, len);
594 if (unlikely(bptr == NULL)) {
595 /* Nothing else we can do... but let's not spam the kernel
596 with these reports. */
597 /* printk(KERN_ERR "ctl_write_msg type=%d len=%d ENOMEM\n", type, len); */
598 goto no_mem;
599 }
600
601 /* Put it on the pool of ready buffers. It's possible to recursively
602 hit a probe here, like a kprobe in NMI or the lock tracepoints, but
603 they will be squashed since we're holding the context busy. */
604 list_add_tail(&bptr->list, &_stp_ctl_ready_q);
605
606 stp_nmi_spin_unlock_irqrestore(&_stp_ctl_ready_lock, flags);
607
608 _stp_runtime_entryfn_put_context(c);
609
610 /* It would be nice if we could speed up the notification
611 timer at this point, but calling mod_timer() at this
612 point would bring in more locking issues... */
613 return len + sizeof(bptr->type);
614
615 no_lock:
616 _stp_runtime_entryfn_put_context(c);
617 return -EBUSY;
618
619 no_mem:
620 stp_nmi_spin_unlock_irqrestore(&_stp_ctl_ready_lock, flags);
621 _stp_runtime_entryfn_put_context(c);
622 return -ENOMEM;
623 }
624
625 /* Logs a warning or error through the control channel. This function mimics
626 _stp_ctl_send() but directly uses an _stp_buffer to construct the warning or
627 error message. This is *only* for warnings and errors. The logtype string
628 should be either "WARNING: " or "ERROR: ", and logtype_len shouldn't include
629 a trailing NUL termination byte. The message type is always assumed to be
630 STP_OOB_DATA since this is only for warnings and errors. */
631 static void _stp_ctl_log_werr(const char *logtype, size_t logtype_len,
632 const char *fmt, va_list args)
633 {
634 struct context *__restrict__ c;
635 struct _stp_buffer *bptr;
636 unsigned long flags = 0;
637
638 c = _stp_runtime_entryfn_get_context();
639
640 stp_nmi_spin_lock_irqsave(&_stp_ctl_ready_lock, flags, put_context);
641
642 bptr = _stp_ctl_get_buffer(STP_OOB_DATA, logtype, logtype_len);
643 if (!bptr)
644 goto unlock;
645
646 /*
647 * This is a generic failure message for when there's no space left. We
648 * aren't allowed to change it, so just go straight to sending it off.
649 */
650 if (bptr == _stp_ctl_oob_warn || bptr == _stp_ctl_oob_err)
651 goto send_msg;
652
653 /*
654 * The logtype string was already copied in by _stp_ctl_get_buffer(),
655 * now copy the rest of the message. The trailing NUL termination byte
656 * automatically added by vscnprintf() is unneeded, so it's ignored.
657 */
658 bptr->len += vscnprintf(bptr->buf + logtype_len,
659 STP_CTL_BUFFER_SIZE - logtype_len, fmt, args);
660
661 /*
662 * Make sure the last character is a newline. There will always be
663 * enough space to do this because vscnprintf() reserves a byte for the
664 * trailing NUL character which we don't care about.
665 */
666 if (bptr->buf[bptr->len - 1] != '\n')
667 bptr->buf[bptr->len++] = '\n';
668
669 send_msg:
670 list_add_tail(&bptr->list, &_stp_ctl_ready_q);
671
672 unlock:
673 stp_nmi_spin_unlock_irqrestore(&_stp_ctl_ready_lock, flags);
674
675 put_context:
676 _stp_runtime_entryfn_put_context(c);
677 }
678
679 /* Calls _stp_ctl_send and then calls wake_up on _stp_ctl_wq
680 to immediately notify listeners. DO NOT CALL THIS FROM A (KERNEL)
681 PROBE CONTEXT. This is only safe to call from the transport layer
682 itself when in user context. All code that could be triggered from
683 a probe context should call _stp_ctl_send(). */
684 static int _stp_ctl_send_notify(int type, void *data, unsigned len)
685 {
686 int ret;
687 dbug_trans(1, "_stp_ctl_send_notify: type=%d len=%d\n", type, len);
688 ret = _stp_ctl_send(type, data, len);
689
690 /* A message was queued, so wake up all _stp_ctl_wq listeners
691 so stapio can pick it up asap. */
692 if (ret > 0)
693 wake_up_interruptible(&_stp_ctl_wq);
694
695 return ret;
696 }
697
698 /** Called when someone tries to read from our .cmd file.
699 Will take _stp_ctl_ready_lock and pick off the next _stp_buffer
700 from the _stp_ctl_ready_q, will wait_event on _stp_ctl_wq. */
701 static ssize_t _stp_ctl_read_cmd(struct file *file, char __user *buf,
702 size_t count, loff_t *ppos)
703 {
704 struct context* __restrict__ c = NULL;
705 struct _stp_buffer *bptr;
706 int len;
707 unsigned long flags;
708
709 /* Prevent probe reentrancy while grabbing probe-used locks. */
710 c = _stp_runtime_entryfn_get_context();
711
712 /* wait for nonempty ready queue */
713 stp_spin_lock_irqsave(&_stp_ctl_ready_lock, flags);
714 while (list_empty(&_stp_ctl_ready_q)) {
715 stp_spin_unlock_irqrestore(&_stp_ctl_ready_lock, flags);
716 _stp_runtime_entryfn_put_context(c);
717 if (file->f_flags & O_NONBLOCK)
718 return -EAGAIN;
719 if (wait_event_interruptible(_stp_ctl_wq, !list_empty(&_stp_ctl_ready_q)))
720 return -ERESTARTSYS;
721 c = _stp_runtime_entryfn_get_context();
722 stp_spin_lock_irqsave(&_stp_ctl_ready_lock, flags);
723 }
724
725 /* get the next buffer off the ready list */
726 bptr = (struct _stp_buffer *)_stp_ctl_ready_q.next;
727 list_del_init(&bptr->list);
728 stp_spin_unlock_irqrestore(&_stp_ctl_ready_lock, flags);
729
730 /* NB: we can't hold the context across copy_to_user, as it might fault. */
731 _stp_runtime_entryfn_put_context(c);
732
733 /* write it out */
734 len = bptr->len + 4;
735 if (len > count || copy_to_user(buf, &bptr->type, len)) {
736 /* Now what? We took it off the queue then failed to
737 * send it. We can't put it back on the queue because
738 * it will likely be out-of-order. Fortunately, this
739 * should never happen.
740 *
741 * FIXME: need to mark this as a transport failure. */
742 errk("Supplied buffer too small. count:%d len:%d\n", (int)count, len);
743 return -EFAULT;
744 }
745
746 /* put it on the pool of free buffers */
747 c = _stp_runtime_entryfn_get_context();
748 _stp_ctl_free_buffer(bptr);
749 _stp_runtime_entryfn_put_context(c);
750
751 return len;
752 }
753
754 static int _stp_ctl_open_cmd(struct inode *inode, struct file *file)
755 {
756 static struct file_operations _stp_ctl_fops;
757
758 if (atomic_inc_return (&_stp_ctl_attached) > 1) {
759 atomic_dec (&_stp_ctl_attached);
760 return -EBUSY;
761 }
762
763 /*
764 * Replace the file's f_op with our own which has the module owner set.
765 * This is needed because, in do_select(), the only thing that can stop
766 * this module from disappearing while data from our procfs file is in
767 * use is the module reference counter. So we need to set the module
768 * owner pointer and then add a reference to our module, since the
769 * reference addition from the open() has already been skipped by the
770 * time this code is reached. The data which can be used after the
771 * module is freed is `&_stp_ctl_wq`, which is stored and later
772 * dereferenced in do_select(). This pointer is passed to do_select()
773 * from the poll_wait() in _stp_ctl_poll_cmd(), which stores it in
774 * `entry->wait_address`. The reason this use-after-free problem exists
775 * is because procfs doesn't allow for passing in a module owner: all
776 * procfs files use an internal `struct file_operations` located in
777 * fs/proc/inode.c. So we patch in a module owner the hard way. No
778 * locking is needed here due to the `_stp_ctl_attached` guard above.
779 * Note that `_stp_ctl_fops` can only be initialized once; initializing
780 * it more than once could cause a bad race because _stp_ctl_close_cmd()
781 * is called *before* the final `file->f_op` usage, meaning that the
782 * `_stp_ctl_attached` guard won't stop us from mangling `_stp_ctl_fops`
783 * while it's in use for closing an old control channel fd.
784 */
785 if (_stp_ctl_fops.owner != THIS_MODULE) {
786 _stp_ctl_fops = *file->f_op;
787 _stp_ctl_fops.owner = THIS_MODULE;
788 }
789 __module_get(THIS_MODULE);
790 file->f_op = &_stp_ctl_fops;
791
792 _stp_attach();
793 return 0;
794 }
795
796 static int _stp_ctl_close_cmd(struct inode *inode, struct file *file)
797 {
798 if (atomic_dec_return (&_stp_ctl_attached) > 0) {
799 BUG();
800 return -EINVAL;
801 }
802 _stp_detach();
803 return 0;
804 }
805
806 static unsigned _stp_ctl_poll_cmd(struct file *file, poll_table *wait)
807 {
808 /* Pretend we can always write and that there is
809 priority data available. We do this so select
810 will report an exception condition on the file,
811 which is used by stapio to see whether select
812 works. */
813 unsigned res = POLLPRI | POLLOUT | POLLWRNORM;
814 unsigned long flags;
815
816 poll_wait(file, &_stp_ctl_wq, wait);
817
818 /* If there are messages waiting, then there will be
819 data available to read. */
820 stp_spin_lock_irqsave(&_stp_ctl_ready_lock, flags);
821 if (! list_empty(&_stp_ctl_ready_q))
822 res |= POLLIN | POLLRDNORM;
823 stp_spin_unlock_irqrestore(&_stp_ctl_ready_lock, flags);
824
825 return res;
826 }
827
828
829 static struct file_operations _stp_ctl_fops_cmd = {
830 .owner = THIS_MODULE,
831 .read = _stp_ctl_read_cmd,
832 .write = _stp_ctl_write_cmd,
833 .open = _stp_ctl_open_cmd,
834 .release = _stp_ctl_close_cmd,
835 .poll = _stp_ctl_poll_cmd
836 };
837 #ifdef STAPCONF_PROC_OPS
838 static struct proc_ops _stp_ctl_proc_ops_cmd = {
839 .proc_read = _stp_ctl_read_cmd,
840 .proc_write = _stp_ctl_write_cmd,
841 .proc_open = _stp_ctl_open_cmd,
842 .proc_release = _stp_ctl_close_cmd,
843 .proc_poll = _stp_ctl_poll_cmd
844 };
845 #endif
846
847 static int _stp_register_ctl_channel(void)
848 {
849 INIT_LIST_HEAD(&_stp_ctl_ready_q);
850
851 /* allocate buffers */
852 _stp_pool_q = _stp_mempool_init(sizeof(struct _stp_buffer),
853 STP_DEFAULT_BUFFERS);
854 if (unlikely(_stp_pool_q == NULL))
855 goto err0;
856 _stp_allocated_net_memory += sizeof(struct _stp_buffer) * STP_DEFAULT_BUFFERS;
857
858 if (unlikely(_stp_ctl_alloc_special_buffers() != 0))
859 goto err0;
860
861 if (_stp_register_ctl_channel_fs() != 0) // procfs or debugfs decision time
862 goto err0;
863
864 return 0;
865
866 err0:
867 _stp_mempool_destroy(_stp_pool_q);
868 errk("Error creating systemtap control channel.\n");
869 return -1;
870 }
871
872 static void _stp_unregister_ctl_channel(void)
873 {
874 struct _stp_buffer *bptr, *tmp;
875
876 _stp_unregister_ctl_channel_fs();
877
878 /* Return memory to pool and free it. */
879 list_for_each_entry_safe(bptr, tmp, &_stp_ctl_ready_q, list) {
880 list_del(&bptr->list);
881 _stp_ctl_free_buffer(bptr);
882 }
883 _stp_ctl_free_special_buffers();
884 _stp_mempool_destroy(_stp_pool_q);
885 }