Debian Codegraph @

interactive-notebook/isystemtap/stap_jobjects.py

raw
1 import subprocess
2 from queue import Queue
3 from threading import Thread
4 import os
5 import time
6 import json5
7 import re
8 import sys
9 import ast
10 from bqplot import *
11 import bqplot.pyplot as plt
12 import sqlite3
13 from typing import List
14 from .stap_kernel import *
15 from ipyevents import Event
16 from ipywidgets import Text, Button, Tab, GridBox, Layout, Label, HBox, VBox, Output, TagsInput, Combobox, IntProgress, HTML, Valid
17 from IPython.display import JSON, Code, Markdown, Javascript
18 from shlex import quote
19 # FIXME: poll's small changes should be added to the jupyter-ui-poll, and the file should be removed
20 from .poll import ui_events
21
22 JUPYTER_MAGIC = "JUPYTER_MAGIC_"
23 # File is created at build time
24 try:
25 from .constants import STAP_PKGDATADIR, STAP_PREFIX
26 STAP_INSTALL_DIR = f'{STAP_PREFIX}/bin'
27 SUBPROCESS_ENV = {**os.environ, 'PATH':f'{STAP_INSTALL_DIR}:{os.environ["PATH"]}'}
28 except:
29 # An educated guess
30 STAP_PKGDATADIR = '/usr/local/share/systemtap'
31 STAP_INSTALL_DIR = '/usr/bin'
32 SUBPROCESS_ENV = None
33
34 # When running in a container we ssh back to the host to run stap. The dest is the host ip we use (localhost here)
35 try:
36 from .constants import SSH_DEST
37 except:
38 SSH_DEST = None
39
40 def format_subprocess_args(*args):
41 flattened_args = []
42 for a in args:
43 if type(a) is list:
44 flattened_args.extend(a)
45 else:
46 flattened_args.append(a)
47
48 if SSH_DEST:
49 command = f'PATH={STAP_INSTALL_DIR+":$PATH"} ' + " ".join([quote(a) for a in flattened_args])
50 flattened_args = ['ssh', SSH_DEST, command]
51
52 return flattened_args
53
54
55 class JSubprocess(subprocess.Popen):
56 """
57 A subprocess that allows to read its stdout and stderr in real time
58 Based on https://github.com/brendan-rius/jupyter-c-kernel/blob/master/jupyter_c_kernel/kernel.py
59 """
60
61 def __init__(self, cmd):
62 super().__init__(cmd, stdin=subprocess.PIPE,
63 stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0)
64
65 self._stdout_queue = Queue()
66 self._stdout_leftovers = list()
67 self._stdout_thread = Thread(
68 target=JSubprocess._enqueue_output, args=(self.stdout, self._stdout_queue))
69 self._stdout_thread.daemon = True
70 self._stdout_thread.start()
71
72 self._stderr_queue = Queue()
73 self._stderr_leftovers = list()
74 self._stderr_thread = Thread(
75 target=JSubprocess._enqueue_output, args=(self.stderr, self._stderr_queue))
76 self._stderr_thread.daemon = True
77 self._stderr_thread.start()
78
79 def write_to_stdin(self, contents):
80 self.stdin.write(contents.encode())
81 self.stdin.flush()
82
83 @staticmethod
84 def _enqueue_output(stream, queue):
85 """
86 Add chunks of data from a stream to a queue until the stream is empty.
87 """
88 for line in iter(lambda: stream.read(4096), b''):
89 queue.put(line)
90 stream.close()
91
92 def get_lines(self, retain_partial_lines=True):
93 """
94 Return a list of tuples (line, stream).
95 Unless retain_partial_lines is False
96 it is safe to assume a line is complete (ends with \n)
97 """
98 lines = []
99 def read_all_from_queue(queue, leftovers, stream_name):
100 res = b''.join(leftovers)
101 leftovers.clear()
102 size = queue.qsize()
103 while size != 0:
104 res += queue.get_nowait()
105 size -= 1
106 raw_lines = res.split(b'\n')
107 if len(res) > 0 and res[-1] != b'\n' and retain_partial_lines:
108 leftovers.append(raw_lines.pop())
109 lines.extend([(line + b'\n', stream_name)
110 for line in raw_lines if line != b''])
111
112 read_all_from_queue(self._stdout_queue,
113 self._stdout_leftovers, "stdout")
114 read_all_from_queue(self._stderr_queue,
115 self._stderr_leftovers, "stderr")
116 return lines
117
118
119 class JHistogram():
120 def __init__(self, kernel, func, name, start=0, end=0, width=0):
121 self.kernel = kernel
122 self.name = name
123 self.func = func
124 self.start = int(start)
125 self.end = int(end)
126 self.width = int(width)
127 self.data = []
128
129 def record_entry(self, bucket, value):
130 self.data.append((bucket, value))
131
132 def display_hist(self):
133 bins = [p[0] for p in self.data]
134 counts = [p[1] for p in self.data]
135 try:
136 # bqplot bar plot
137 scale_x = OrdinalScale()
138 scale_y = LinearScale()
139 fig = plt.figure(
140 title=f'{"Linear" if self.func == "linear" else "Logarithmic"} Histogram of {self.name}')
141 def_tt = Tooltip(fields=['x', 'y'], labels=['Value', 'Count'])
142 bar = plt.bar(x=bins, y=counts,
143 scales={'x': scale_x, "y": scale_y},
144 colors=['coral'], stroke='white', stroke_width=0.4,
145 tooltip=def_tt)
146 self.kernel.display(fig)
147 except Exception as e:
148 self.kernel.error(e)
149
150
151 class JNamespace:
152 def __init__(self, kernel, name=""):
153 self.name : str = name
154 # If "name" is used as a module name will be updated to "name"i at runtime
155 self.mod_name: str = name
156 self.kernel = kernel
157 self._cells : List[JCell] = list()
158 self._proc : JSubprocess = None
159 self._globals: dict = dict()
160
161 def __del__(self):
162 # If the destructor is called, remove the namespace
163 # from the kernel's map
164 self.kernel.jnamespaces.pop(self.name, None)
165
166 def add_cell(self, cell):
167 self._cells.append(cell)
168
169 def remove_cell(self, cell):
170 if cell in self._cells:
171 self._cells.remove(cell)
172
173 def remove_cell_by_id(self, cell_id):
174 self._cells = [c for c in self._cells if c.cell_id != cell_id]
175
176 def remove_all_cells(self):
177 self._cells.clear()
178
179 def get_cells(self):
180 return self._cells
181
182 def update_globals(self, globals_dict):
183 """Update the internal states of the namespace's globals"""
184 # Keep what is already stored
185 if globals_dict is None:
186 return
187 globs = {}
188 for g, v in globals_dict.items():
189 if isinstance(v, dict):
190 # An array so parse the keys
191 # May have tuple keys for stap assosiative array
192 # For stats keys will just be unparsed strings
193 globs[g] = {(ast.literal_eval(
194 k) if "\"" in k or "(" in k else k): vi for k, vi in v.items()}
195 else:
196 globs[g] = v
197 self._globals = globs
198
199 def get_globals(self):
200 return self._globals
201
202
203 class CellExeMode():
204 EDIT = "edit"
205 RUN = "run"
206 SCRIPT = "script"
207 PYTHON = "python"
208 EXAMPLES = "examples"
209 PROBES = "probes"
210 HELP = "help"
211 UNDEF = "undefined"
212
213
214 class JCell:
215 def __init__(self, cell_id: str, kernel):
216 self.cell_id = cell_id
217 self.kernel = kernel
218 self.namespace: JNamespace = None
219 self.mode = CellExeMode.UNDEF
220 self.script = ""
221 self.options = list()
222 self.args = list()
223 self.hist: JHistogram = None
224
225 def __del__(self):
226 # When a cell's dtor is called, make sure to remove it
227 # (not just a cell with the same id) from the namespace too
228 if self.namespace:
229 self.namespace.remove_cell(self)
230 if self.namespace.get_cells == []:
231 del self.namespace
232
233 def _load_script(self, code: str):
234 script = ""
235 namespace = ""
236 try:
237 for line in code.split("\n"):
238 # We found a possible magic header
239 if len(line) >= 2 and line[:2] == "%%":
240 if self.mode != CellExeMode.UNDEF:
241 self.kernel.error(
242 "Only 1 magic command is allowed per cell")
243 return False
244
245 # Before anything else, see if our internal special argument (--args)
246 # exists, and if so extract and parse it
247 a_start = line.find('--args')
248 a_open, a_close = line.find(
249 '(', a_start), line.rfind(')', a_start)
250 if a_start != -1:
251 if a_open > a_close or a_open == -1 or a_close == -1:
252 self.kernel.error(
253 "Invalid args usage: Should be --args=(x,y,z)")
254 return False
255 else:
256 self.args = line[a_open+1: a_close].split(",")
257 # We've removed the args, so the line is updated accordingly
258 line = line[:a_start] + line[a_close+1:]
259
260 split_magic = line[2:].rstrip().split(" ")
261 command = split_magic.pop(0)
262
263 # Cells with namespaces
264 if command in (CellExeMode.EDIT, CellExeMode.RUN, CellExeMode.PYTHON, CellExeMode.SCRIPT):
265 # We have a magic with a namespace provided
266 # otherwise namespace = "" i.e the global namespace
267 if len(split_magic) >= 1 and split_magic[0][0] != '-':
268 namespace = split_magic.pop(0)
269
270 # Cells with arguments
271 if command != CellExeMode.SCRIPT and self.args != []:
272 self.kernel.error(
273 "Invalid args usage: Should only be used with script cells")
274 return False
275 # Cells with options
276 VALID_OPTIONS = ('-v', '--vp', '-V', '--version', '-P', '-u', '-w', '-W', '-t', '-s', '-I', '-D',
277 '-B', '-a', '--modinfo', '-r', '-d', '--ldd', '--all-modules', '-c', '-x', '-T', '-g',
278 '--prologue-searching', '--suppress-handler-errors', '--compatible', '--check-version',
279 '--disable-cache', '--poison-cache', '--privilege', '--unprivileged', '--download-debuginfo',
280 '--rlimit-', '--sysroot', '--runtime', '--dyninst', '--bpf', '--suppress-time-limits')
281 if command not in (CellExeMode.EDIT, CellExeMode.RUN, CellExeMode.SCRIPT) and split_magic != []:
282 self.kernel.error(
283 "Invalid options usage: Should only be used with run, edit and script cells. Skipping line")
284 return False
285 else:
286 self.options = split_magic
287 # Make sure the flags are all valid
288 if any(all(not flag.startswith(valid_flag) for valid_flag in VALID_OPTIONS) for flag in [o for o in self.options if o[0] == '-']):
289 self.kernel.error(
290 f'Invalid Options: {self.options}. Only {VALID_OPTIONS} can be used')
291 return False
292
293 if command not in (CellExeMode.EDIT, CellExeMode.EXAMPLES, CellExeMode.HELP,
294 CellExeMode.PROBES, CellExeMode.PYTHON, CellExeMode.RUN, CellExeMode.SCRIPT):
295 self.kernel.error(
296 f"Skipping unknown magic command {command}")
297 return False
298 self.mode = command
299 elif len(line) >= 1 and line[0] == "!":
300 p = subprocess.run(format_subprocess_args(line[1:].split(" ")), capture_output=True, text=True, env=SUBPROCESS_ENV)
301 if p.stdout:
302 self.kernel.write(p.stdout)
303 if p.returncode != 0:
304 self.kernel.error(p.stderr)
305 return False
306 else:
307 # We want to find lines which print histograms and bookend them with magic commands that isystemtap
308 # will recognize to replace the output with our own.
309 r = re.compile(
310 r'(.*?(?=print))(print.*\(.*\@hist_)(.*\)*)(.*)')
311 m = r.search(line)
312 if m:
313 prefix = m[1] # Stuff before
314 operator = m[2] # Ex. 'print(@hist_'
315 operands = m[3] # Ex. 'log(foo)' or 'linear(a, 1,2, 3)'
316 suffix = m[4] # Stuff after
317 self.kernel.log.debug(
318 f'Parsed histogram with {prefix, operator, operands, suffix = }')
319 # Don't magic up commented out histograms
320 if any(c in prefix for c in ('//', '#', '/*')):
321 self.kernel.log.debug(
322 'Skipping commented histogram')
323 script += line+"\n"
324 continue
325 # We can't print a histogram for '@hist_log(foo)[0] or @hist_linear(bar[baz], 1, 2 , 3)[2]'
326 # If a ) comes before a [ then we can say that the hist_foo(...) is done and then an index is started so skip
327 if operands.find(")") < operands.rfind("["):
328 self.kernel.log.debug(
329 'Skipping hisogram which is being indexed')
330 script += line+"\n"
331 continue
332 # The inner most '(' and the outmost ')'
333 paren_open, paren_close = operands.rfind(
334 "("), operands.find(')')
335 # We use the commands hist_FUNC:ARG1:... and hist_end
336 args = [operands[:paren_open]] + [s.strip()
337 for s in operands[paren_open + 1:paren_close].split(",")]
338
339 # We want to display the literal for scalar stats BUT the value for the arguments to arrays
340 # Ex. hist_log(foo[name]) should be called foo[VALUE_OF_NAME]
341 p1_args = ""
342 if '[' in args[1]:
343 index_open = args[1].find("[")
344 index_close = args[1].find(']')
345 subscript = args[1][index_open+1: index_close]
346 p1_args = f', {subscript}'
347 # Replace the subscript with the appropriate %s, ...
348 args[1] = args[1][:index_open+1] + ", ".join(
349 ["%s" for _ in subscript.split(",")]) + args[1][index_close:]
350 p1 = "hist_" + ":".join(args)
351 p2 = "hist_end"
352 line = prefix + f' ; printf("\\n{JUPYTER_MAGIC}{p1}\\n"{p1_args}); ' + \
353 operator + operands + \
354 f' ; println("\\n{JUPYTER_MAGIC}{p2}\\n"); ' + suffix
355 self.kernel.log.debug(
356 f'Histogram line replaced with { line }')
357
358 script += line+"\n"
359 except:
360 return False
361
362 if self.mode == CellExeMode.UNDEF and script != "":
363 self.kernel.print(
364 "Defaulting to a script cell")
365 self.mode = CellExeMode.SCRIPT
366
367 if not re.match(r"^[a-z0-9_]*$", namespace):
368 self.kernel.error(
369 f'"{namespace}" is not a valid namespace i.e ^[a-z0-9_]*$')
370 return False
371
372 if namespace == "":
373 namespace = "isystemtap_global"
374
375 # We get a namespace BUT are not yet added to it, that's only done once validity is checked
376 if namespace not in self.kernel.jnamespaces:
377 ns = JNamespace(self.kernel, namespace)
378 self.namespace = ns
379 self.kernel.jnamespaces[namespace] = ns
380 else:
381 self.namespace = self.kernel.jnamespaces[namespace]
382 # FIXME: What if the cell was in a different namespace
383 self.namespace.remove_cell_by_id(self.cell_id)
384 self.script = script
385 self.kernel.log.debug(
386 f'Loaded cell of type {self.mode} into namespace "{self.namespace.name}" with args {self.args} and options {self.options}')
387 self.kernel.log.debug(f'The script is: {self.script}')
388 return True
389
390 def _validate_syntax(self):
391 """
392 Run pass 1; To quickly detect any syntax errors
393 """
394 p = subprocess.run(format_subprocess_args("stap", "-e", self.script,
395 "-p1"), capture_output=True, text=True, env=SUBPROCESS_ENV)
396 if p.returncode == 0:
397 return True
398 else:
399 self.kernel.send_response(self.kernel.iopub_socket, 'stream', {
400 'name': "stderr", 'text': p.stderr})
401 return False
402
403 def _validate_symbols_and_types(self):
404 """
405 Run pass 2 against all of the cells within the namespace, combining them into 1 large virtual_script (In the order they are executed)
406 On success add this cell to the namespace's execution queue otherwise remove any info from this cell, since its invalid
407 """
408
409 # Combine the various partial-scripts
410 virtual_script = "".join(cell.script for cell in self.namespace.get_cells())
411 virtual_script += self.script
412
413 # In order to avoid a premature no probes found error just add a never probe
414 # This will never make it to stap passes 3+, just temporarily avoid the error
415 virtual_script += '\n probe never { printf("Spam, Ham and Eggs") } '
416
417 p = subprocess.run(format_subprocess_args("stap", self.options, "-e", virtual_script,
418 "-p2"), capture_output=True, text=True, env=SUBPROCESS_ENV)
419 self.kernel.log.debug(f'Executing {p.args}')
420 if p.returncode == 0:
421 self.namespace.add_cell(self)
422 return True
423 else:
424 self.kernel.send_response(self.kernel.iopub_socket, 'stream', {
425 'name': "stderr", 'text': p.stderr})
426 return False
427
428 def _run_namespace(self):
429 # Stitch together the cells into 1 virtual script
430 virtual_script = "".join(cell.script for cell in self.namespace.get_cells())
431
432 # We need to ensure the module name is not in use already
433 self.namespace.mod_name = self.namespace.name
434 i = 1
435 with open("/proc/modules", "r") as f:
436 # Technically this introduces a race condition where the modules list is only loaded once
437 # so k users could simultaneously find the same 'free' mod name. But even if restricted to a read each
438 # loop iter the race condition remains, so just leave it since this is faster.
439 modules = f.read()
440 while True:
441 if self.namespace.mod_name not in modules:
442 break
443 self.namespace.mod_name = f'{self.namespace.name}{i}'
444 i += 1
445
446 extra_options = [
447 "-m", self.namespace.mod_name,
448 "--monitor", #FIXME: Is there a way to wait for --monitor mode to start up fully
449 "-D", "MAXSTRINGLEN=4096",
450 "--skip-badvars",
451 "-e", virtual_script
452 ]
453 cmd = format_subprocess_args("stap", self.options, extra_options, self.args)
454 self.kernel.log.info(f'Executing {cmd}')
455 p = JSubprocess(cmd)
456
457 proc_path = f'/proc/systemtap/{self.namespace.mod_name}'
458 def monitor_write(content):
459 try:
460 f = open(proc_path+"/monitor_control", 'w')
461 f.write(content)
462 f.flush()
463 f.close()
464 except OSError:
465 pass
466
467 # Tab 0: Controls
468 class tab_controls:
469 def __init__(self, namespace, options, args):
470 self.uptime = Label(value="Uptime: 00:00:00")
471 self.memory = Label(
472 value="Memory: 0data/0text/0ctx/0net/0alloc KB")
473
474 self.play_pause = Button(
475 button_style='info', tooltip='Pause/resume script by toggling off/on all probes', icon='pause')
476 def handle_play_pause(button):
477 if button.icon == "pause": # Script is running
478 monitor_write('pause')
479 button.icon = "play"
480 else:
481 monitor_write('resume')
482 button.icon = "pause"
483 self.play_pause.on_click(handle_play_pause)
484
485 self.stop = Button(button_style='danger',
486 tooltip='Quit script', icon='stop')
487 self.stop.on_click(lambda _: monitor_write('quit'))
488
489 self.reset = Button(
490 button_style='info', tooltip='Reset all global variables to their initial states', icon='rotate-left')
491 self.reset.on_click(lambda _: monitor_write('clear'))
492
493 self.export = Button(
494 button_style='info', tooltip='Export the script', icon='download')
495 self.export_out = Output()
496 self.export_hbox = HBox([self.export,self.export_out])
497 def handle_export(btn):
498 btn.icon='check-circle'
499 btn.disabled = True
500
501 file_content = f'#!stap {" ".join(options)} {" ".join(args)}\n'
502 file_content += virtual_script
503
504 # The following is based on
505 # https://stackoverflow.com/questions/3749231/download-file-using-javascript-jquery
506 jscode = ' \
507 const blob = new Blob([%s], { type : "plain/text" }); \
508 const url = window.URL.createObjectURL(blob); \
509 const a = document.createElement("a"); \
510 a.style.display = "none"; \
511 a.href = url; \
512 a.download = "%s"; \
513 document.body.appendChild(a); \
514 a.click(); \
515 window.URL.revokeObjectURL(url);' % (json.dumps(file_content), f'{namespace}.stp')
516 self.export_out.append_display_data(Javascript(jscode))
517
518 self.export.on_click(handle_export)
519
520 self.stdin = Text(description='>>>')
521
522 self.controls = VBox(\
523 [self.export_hbox,
524 HBox([self.uptime, self.memory]),
525 HBox([self.play_pause, self.reset, self.stop]),
526 self.stdin])
527
528 def get_widget(self):
529 return self.controls
530
531 def update(self, uptime, memory):
532 if uptime:
533 self.uptime.value = f'Uptime: {uptime}'
534 if memory:
535 self.memory.value = f'Memory: {memory}'
536
537 def disable(self):
538 for sub_widget in [self.play_pause, self.stop, self.reset, self.stdin]:
539 sub_widget.disabled = True
540
541 # Tab 1: Globals
542 class tab_globals:
543 def __init__(self):
544 self.global_viewer = Output()
545 # FIXME: The flickering issue is known, but how do I know a height to set it to?
546 # https://ipywidgets.readthedocs.io/en/stable/examples/Using%20Interact.html#Flickering-and-jumping-output
547
548 def get_widget(self):
549 return self.global_viewer
550
551 def update(self, globals_json):
552 if globals_json is None and len(self.global_viewer.outputs) != 0:
553 # Keep the old data (ex. after cell execution)
554 pass
555 elif globals_json is None and len(self.global_viewer.outputs) == 0:
556 self.global_viewer.append_display_data(Markdown(data="*No data available*"))
557 else:
558 # NB: What should be done here is a tab_globals.clear_output()
559 # The issue is that this isn't the primary Output widget and so
560 # the clear doesn't act correctly. So we do this 'hack' variant
561 self.global_viewer.outputs = ()
562 self.global_viewer.append_display_data(JSON(data=globals_json, expanded=True, root="globals"))
563
564 # Tab 2: Probes
565 class tab_probes:
566 def __init__(self):
567 self._num_columns = 6
568 self.probes = GridBox([], layout=Layout(
569 grid_template_columns='5ch 1fr 1fr 1fr 1fr 1fr'))
570
571 def get_widget(self):
572 return self.probes
573
574 def update(self, probes_json):
575 if probes_json is None:
576 # TODO: Might be nice to have a `no data` sort of thing
577 return
578 p_list = [Label(value=v, style={'font_weight': 'bold'}) for v in (
579 'state', 'name', 'hits', 'min', 'avg', 'max')]
580 for d in probes_json:
581 state = Button(button_style='danger', tooltip='Toggle Probe',
582 icon='check-square' if d['state'] == 'on' else 'square', layout=Layout(width='5ch'))
583 # We need to store which probe this button goes with, since the actual button will change with update,
584 # but the class will remain const
585 state.add_class(f'probe_idx_{d["index"]}')
586 state.on_click(lambda button: monitor_write(button._dom_classes[-1][len("probe_idx_"):]))
587 p_list += [state, Label(value=d['name']), Label(value=str(d['hits'])), Label(
588 value=str(d['min'])), Label(value=str(d['avg'])), Label(value=str(d['max']))]
589 self.probes.children = p_list
590
591 def disable(self):
592 state_buttons = self.probes.children[self._num_columns::self._num_columns]
593 for button in state_buttons:
594 button.disabled = True
595
596 controls_view = tab_controls(self.namespace.name, self.options, self.args)
597 globals_view = tab_globals()
598 probes_view = tab_probes()
599 # The Tab Structure
600 tab_content = {
601 "Controls": controls_view.get_widget(),
602 "Globals": globals_view.get_widget(),
603 "Probes": probes_view.get_widget()
604 }
605 tab = Tab(children=list(tab_content.values()))
606 for i, k in enumerate(tab_content.keys()):
607 tab.set_title(i, k)
608
609 # Handle keyboard events within the widget
610 keyboard_event = Event(source=tab, watched_events=['keydown'])
611 def handle_keydown(event):
612 if not p or p.poll() is not None:
613 return
614 if event['ctrlKey']:
615 match event['key']:
616 case 'c':
617 monitor_write('quit')
618 else:
619 # This means we clicked a special key like 'backspace' or 'arrow'
620 if len(event['key']) > 1:
621 return
622 # Write the new char to stdin
623 try:
624 p.write_to_stdin(event['key'])
625 except BrokenPipeError:
626 pass
627 controls_view.stdin.value += event['key']
628 keyboard_event.on_dom_event(handle_keydown)
629 self.kernel.display(tab)
630
631 def update_widget_states(last_updated_monitor=0, monitor_files_owned = True):
632 """Update the widgets, but only do so once every 0.5s (more frequent isn't needed for graphics)"""
633 jso = dict()
634 try:
635 # We need read/write access to the procfs files. If running directly on the host these have no issues BUT
636 # if running a container as root then the notebook (current) user in the container can't accesss these controls
637 # so the first time we can do a read (monitor_files_owned is false but the file exists), we
638 # chown them
639 if SSH_DEST and not monitor_files_owned and os.path.exists(proc_path):
640 files = ['monitor_control', 'monitor_status']
641 for f in files:
642 subprocess.run(format_subprocess_args('chown', '--silent', f'{os.getuid()}:{os.getgid()}', f'{proc_path}/{f}'),
643 stderr=subprocess.DEVNULL)
644 monitor_files_owned = True
645
646 with open(proc_path+"/monitor_status", 'r') as content:
647 # We use json5 to allow for tailing commas in the json format
648 jso = json5.load(content)
649 except:
650 pass
651
652 # Update the internal copy of the globals, for use with python cells
653 self.namespace.update_globals(jso.get("globals", None))
654
655 now = time.time()
656 if now - last_updated_monitor < 0.5:
657 return last_updated_monitor, monitor_files_owned
658
659 # Update tabs content
660 controls_view.update(jso.get("uptime", None), jso.get("memory", None))
661 globals_view.update(jso.get("globals", None))
662 probes_view.update(jso.get('probe_list', None))
663
664 return now, monitor_files_owned
665
666 def write_to_output(content, output_disabled):
667 """Write content (a list of line, stream pairs) to the stream (stdout or stderr).
668 If output_disabled skip
669 Returns True iff the output is disabled i.e the process is not running
670 """
671 if output_disabled:
672 return True
673 for line, stream in content:
674 s_line = line.decode()
675 # Handle jupyter magic lines
676 if s_line.startswith(JUPYTER_MAGIC):
677 command = s_line[len(JUPYTER_MAGIC):].strip()
678 if command == "hist_end":
679 # Display the histogram and then clear the
680 # internal repr (since while it exists, isystemtap assumes
681 # it needs to be filled)
682 self.hist.display_hist()
683 self.hist = None
684 continue
685 elif command.startswith("hist_"):
686 # The start of a histogram block is found
687 # The format is hist_func:arg1:arg2:...
688 hist_args = command[len("hist_"):].split(":")
689 self.hist = JHistogram(self.kernel, *hist_args)
690 continue
691
692 # We have an active histogram so take that data and make it pretty
693 if self.hist:
694 # s_line is of the form
695 #'value |-------------------------------------------------- count'
696 s_split = s_line.strip().split(" ")
697 # If we're not the heading, record that line
698 if s_split[0] != 'value' and len(s_split) > 3:
699 # FIXME: Looks like when using sprint* we lose the last count
700 value, count = s_split[0], int(s_split[-1])
701 self.hist.record_entry(value, count)
702 continue
703
704 # If we get here, we just have a regular line so output it
705 self.kernel.send_response(self.kernel.iopub_socket, 'stream', {
706 'name': stream, 'text': s_line})
707 return False
708
709 output_disabled = False
710 last_updated_monitor = time.time() # We only need to update so often
711 monitor_files_owned = False
712
713 # Start the main loop
714 with ui_events(self.kernel) as ui_poll:
715 while p.poll() is None:
716 try:
717 # Write any lines to stdout/err if any are ready
718 output_disabled = write_to_output(p.get_lines(), output_disabled)
719 # Get UI events (up to 10 at a time)
720 # NB: Jupyter isn't designed to have UI elements be interactive while a cell is
721 # running, so this polling is very important for the controls widgets
722 ui_poll(10)
723 # Update the widgets states (controls, globals and probes) & update the internal globals (for python usage)
724 last_updated_monitor, monitor_files_owned = update_widget_states(last_updated_monitor, monitor_files_owned)
725 except KeyboardInterrupt:
726 # The kernel sent a sigint (I, I), so catch it and pass on to the child
727 # to allow stap to gracefully terminate
728 monitor_write('quit')
729 # On that last line, don't hold back even if there is a partial result (not \n terminated)
730 output_disabled = write_to_output(p.get_lines(retain_partial_lines=False), output_disabled)
731 # Do our best to cleanup after ourselves
732 try:
733 p = subprocess.run(format_subprocess_args('rm', '-f', f'{self.namespace.name}.ko'))
734 except OSError:
735 pass
736
737 # Disable the buttons now that we're done
738 controls_view.disable()
739 probes_view.disable()
740 # Give a final update to make sure it's as up to date as we can make it
741 update_widget_states()
742 return p.returncode == 0
743
744 def _run_python(self):
745 status = True
746 try:
747 old_stdout, old_stderr = sys.stdout.write, sys.stderr.write
748 sys.stdout.write = lambda *args: self.kernel.send_response(
749 self.kernel.iopub_socket, 'stream', {'name': "stdout", 'text': str(args)})
750 sys.stderr.write = lambda *args: self.kernel.send_response(
751 self.kernel.iopub_socket, 'stream', {'name': "stderr", 'text': str(args)})
752 # Run the script in python with the systemtap globals defined
753 # Also add in display and print
754 exec(self.script, self.namespace.get_globals() | {
755 "display": self.kernel.display, "print": self.kernel.print})
756 except Exception as e:
757 self.kernel.error(e)
758 status = False
759 sys.stdout.write, sys.stderr.write = old_stdout, old_stderr
760
761 return status
762
763 def _run_help(self):
764 self.kernel.print('ISystemtap')
765 self.kernel.print(
766 f'%%{CellExeMode.EDIT} [ NAMESPACE ] -[ OPTIONS ]\n\tA piece of a script in NAMESPACE')
767 self.kernel.print(
768 f'%%{CellExeMode.RUN} [ NAMESPACE ] -[ OPTIONS ]\n\tCombine and run the edit cells in NAMESPACE')
769 self.kernel.print(
770 f'%%{CellExeMode.SCRIPT} [ NAMESPACE ] -[ OPTIONS ] [ --args=(ARGUMENTS) ]\n\tRun an entire script. May be give command line args')
771 self.kernel.print(
772 f'%%{CellExeMode.PYTHON} [ NAMESPACE ]\n\tRun a python script using the globals from NAMESPACE')
773 self.kernel.print(
774 f'%%{CellExeMode.EXAMPLES}\n\tRun a python script using the globals from NAMESPACE')
775 self.kernel.print(
776 f'%%{CellExeMode.PROBES}\n\tList all available probe points matching the given single probe (See stap -L)')
777 self.kernel.print(f'%%{CellExeMode.HELP}\n\tRefer to %%help for more information')
778 self.kernel.print(
779 f'!command [ ARGUMENTS ]\n\tRun command in a shell')
780 self.kernel.print(f'Run !stap --help for more information on available options')
781
782 def _run_examples(self):
783 title = Text()
784
785 # Query the DB for the possibilities
786 con = None
787 DB_PATH = STAP_PKGDATADIR + "/examples/metadatabase.db"
788 try:
789 con = sqlite3.connect(DB_PATH)
790 except:
791 self.kernel.error(f"Unable to connect to examples database: {DB_PATH}")
792 return False
793 res = con.execute('SELECT DISTINCT keywords, name from metavirt')
794 keywords = set() # Possible keywords
795 example_names = list() # Possible examples
796 for k in res.fetchall():
797 keywords |= set(k[0].split(" "))
798 example_names.append(k[1])
799
800 keyword_picker = TagsInput(
801 value=[],
802 allowed_tags=list(keywords),
803 allow_duplicates=False
804 )
805 name = Combobox(
806 placeholder='example_script.stp',
807 options=example_names,
808 ensure_option=True
809 )
810 search_button = Button(button_style='info',
811 tooltip='Search', icon='search')
812 clear_button = Button(button_style='info',
813 tooltip='Clear', icon='refresh')
814
815 # The top control Box
816 search_controls = VBox([
817 Label(value="Systemtap Example Search", style={
818 'font_weight': 'bold', 'font_size': '1.5em'}),
819 HBox([
820 Label(value="Name ", style={'font_weight': 'bold'}),
821 name,
822 Label(value="Title Contains", style={'font_weight': 'bold'}),
823 title
824 ]),
825 HBox([
826 Label(value="Keywords", style={'font_weight': 'bold'}),
827 keyword_picker,
828 ]),
829 HBox([
830 search_button, clear_button
831 ])
832 ])
833
834 # The table, where the results are stored
835 table = VBox(layout=Layout(width='50%'))
836 # Holds the table and the output of the example (as well as the script viewer)
837 side_by_side_viewer = HBox([table])
838
839 SQL_EXTRACTION = 'SELECT title, name, keywords, description, path FROM metavirt WHERE '
840 def make_row(title, name, keywords, description, path):
841 out = Output(layout=Layout(border='solid 2px', width='50%'))
842
843 full_path = f'{STAP_PKGDATADIR}/examples/{path}/{name}'
844
845 # The controls for running an example
846 run = Button(button_style='info', tooltip='Run', icon='play')
847 back = Button(button_style='info',tooltip='Back', icon='times')
848 options = TagsInput(value=[], allow_duplicates=True)
849 args = TagsInput(value=[], allow_duplicates=True)
850 controls = VBox([
851 HBox([run, back], layout=Layout(display='flex',
852 flex_flow='row', align_items='flex-end')),
853 HBox([Label(value="Options", style={
854 'font_weight': 'bold'}), options]),
855 HBox([Label(value="Arguments", style={
856 'font_weight': 'bold'}), args])
857 ])
858 # The example code preview
859 code_block = Code(filename=full_path, language="c")
860 # TODO: In order to highlight the code we need a pygments lexer
861 # class SystemtapLexer(RegexLexer): ... Will end up with the same logic as the codemirror regex highlighting
862 # Left for now due to https://github.com/ipython/ipython/issues/11747
863
864 def handle_back(_):
865 side_by_side_viewer.children = [table]
866 back.on_click(handle_back)
867
868 def handle_run(_):
869 # Remove any old run outputs
870 out.outputs = out.outputs[:2] # Just keep controls, code_block
871 # out.outputs = ()
872 # out.append_display_data(controls)
873 # out.append_display_data(code_block)
874 with out, open(full_path, 'r') as example:
875 c = JCell("examplecell", self.kernel)
876 print("Executing")
877 res = c.execute(
878 f'%%script example_{name[:-4].replace("-", "_")} {" ".join(options.value)} { "" if args.value == [] else "--args=(" + ",".join(args.value) + ")" }\n{example.read()}')
879 self.kernel.display(
880 Valid(value=res['status'] == 'ok', readout="Execution Failure"))
881 run.on_click(handle_run)
882
883
884 preview = Button(button_style='info',
885 tooltip='Preview example script', icon='eye')
886 def handle_preview(_):
887 out.outputs = ()
888 out.append_display_data(controls)
889 out.append_display_data(code_block)
890 side_by_side_viewer.children = [table, out]
891 preview.on_click(handle_preview)
892
893 row = VBox(
894 children=[
895 Label(value=f'{name}: {title}', style={
896 'font_weight': 'bold', 'font_size': '1.1em'}),
897 HTML(description), # HTML wraps
898 Label(value=keywords, style={'font_style': 'italic'}),
899 preview
900 ],
901 layout=Layout(
902 border='solid 2px',
903 ))
904 return row
905
906 def handle_search(_):
907 # SQLite objects created in a thread can only be used in that same thread
908 con = sqlite3.connect(DB_PATH)
909 res = None
910 if name.value != '':
911 # If given an exact script, just pull it up by name
912 res = con.execute(SQL_EXTRACTION + 'name = ?', (name.value, ))
913 else:
914 # Otherwise we need to do a proper search
915 CONDITON = " AND ".join(
916 ['title LIKE ?'] + # The title is like the given substring
917 # Each keyword given must be a substring of the one within the DB
918 # Since keywords are 'foo bar baz'. So ex. we need to match keywords LIKE foo AND keywords like bar
919 ['keywords LIKE ?'] * len(keyword_picker.value)
920 )
921 params = tuple('%'+p+'%' for p in (title.value,
922 )+tuple(keyword_picker.value))
923 res = con.execute(SQL_EXTRACTION + CONDITON, params)
924 table.children = [make_row(*r) for r in res]
925 side_by_side_viewer.children = [table]
926 search_button.on_click(handle_search)
927 name.on_submit(handle_search)
928 title.on_submit(handle_search)
929
930 def handle_clear(_):
931 title.value = ''
932 name.value = ''
933 keyword_picker.value = []
934 table.children = []
935 side_by_side_viewer.children = [table]
936 clear_button.on_click(handle_clear)
937
938 self.kernel.display(VBox([search_controls, side_by_side_viewer]))
939
940 def _run_probe_list(self):
941 query = Text()
942 search_button = Button(button_style='info', tooltip='Search', icon='search')
943 clear_button = Button(button_style='info', tooltip='Clear', icon='refresh')
944 results = VBox([])
945 def list_probes(_):
946 # Just visual feedback since the loading can take a second. This serves no real purpose
947 progress = IntProgress(value=2, min=0, max=5, step=1,
948 bar_style='info', orientation='horizontal')
949 results.children = [progress]
950
951 rows = []
952 p = subprocess.run(format_subprocess_args("stap", '-L', query.value),
953 capture_output=True, text=True, env=SUBPROCESS_ENV)
954 progress.value = 4 # Again just a visual thing
955 if p.returncode == 0:
956 for line in str(p.stdout).split('\n'):
957 pp_end = line.find(' ')
958 if pp_end != -1: # There is a space i.e context variables
959 probe_point, context_vars = line[:pp_end], line[pp_end+1:]
960 else:
961 probe_point, context_vars = line, ""
962 rows.append(VBox([
963 Label(value=probe_point, style={
964 'font_weight': 'bold', 'font_size': '1.1em'}),
965 # We use HTML, since Labels render LaTex (so $arg: ... $arg2 is math mode causing issues)
966 HTML(f'<em>{context_vars}</em>')
967 ]))
968 results.children = rows
969 search_button.on_click(list_probes)
970 query.on_submit(list_probes)
971
972 def clear_list(_):
973 query.value = ""
974 results.children = []
975 clear_button.on_click(clear_list)
976
977 self.kernel.display(VBox([
978 Label(value="Systemtap Probe List", style={
979 'font_weight': 'bold', 'font_size': '1.5em'}),
980 Label(value="List all available probe points (and context variables) matching the given single probe point. The pattern may include wildcards and aliases",
981 style={'font_style': 'italic'}),
982 HBox([Label(value="probe", style={'font_weight': 'bold'}), query]),
983 HBox([search_button, clear_button]),
984 results
985 ]))
986
987 def execute(self, code):
988 if not self._load_script(code):
989 status = 'error'
990 else:
991 status = 'ok'
992 match self.mode:
993 case CellExeMode.EDIT:
994 actions = [self._validate_syntax,
995 self._validate_symbols_and_types]
996 case CellExeMode.RUN:
997 actions = [self._run_namespace]
998 case CellExeMode.PYTHON:
999 actions = [self._run_python]
1000 case CellExeMode.SCRIPT:
1001 def add_to_ns():
1002 # As a script there are no other cells to stitch,
1003 # so we want remove the previous scripts (if they exist)
1004 self.namespace.remove_all_cells()
1005 self.namespace.add_cell(self)
1006 return True
1007 actions = [add_to_ns, self._run_namespace]
1008 case CellExeMode.EXAMPLES:
1009 actions = [self._run_examples]
1010 case CellExeMode.PROBES:
1011 actions = [self._run_probe_list]
1012 case CellExeMode.HELP:
1013 actions = [self._run_help]
1014 case _:
1015 actions = []
1016
1017 for a in actions:
1018 if not a():
1019 status = 'error'
1020 break
1021
1022 return {'status': status,
1023 'execution_count': self.kernel.execution_count,
1024 'payload': [],
1025 'user_expressions': {},
1026 }