Debian Codegraph @

testsuite/systemtap.stress/passlist.exp

raw
1 # An implementation of generating passlist for safe-probes
2 # in systemtap based on the discussion in thread:
3 # http://sourceware.org/ml/systemtap/2006-q3/msg00574.html
4 #
5 # Copyright (C) 2006 IBM Corp.
6 #
7 # This file is part of systemtap, and is free software. You can
8 # redistribute it and/or modify it under the terms of the GNU General
9 # Public License (GPL); either version 2, or (at your option) any
10 # later version.
11 #
12 # Main ideas:
13 # 1)Fetch a group of probe points from probes.pending, probe them
14 # and run some workloads(e.g. runltp) parallely meanwhile.
15 #
16 # 2)If the probe test ends without crash, those actually triggered
17 # probe points are moved into probes.passed and those
18 # untriggered are into probes.untriggered; if the probe test
19 # crashes the system, it will be resumed automatically after
20 # system reboot. Those probe points which have been triggered are
21 # also moved into probes.passed, but those untriggered ones are
22 # moved into probes.failed.
23 #
24 # 3)Repeat the above until probes.pending becomes empty, then:
25 # Normally, probes.pending is reinitialized from probes.failed
26 # and probes.untriggered, then start the next iteration;
27 # But if max running level (now 3) is reached, or probes.pending,
28 # probes.failed and probes.untriggered are all empty, stop the
29 # whole test.
30 #
31 # http://sourceware.org/ml/systemtap/2006-q4/msg00143.html
32 # http://sourceware.org/ml/systemtap/2006-q4/msg00435.html
33 #
34 set TESTNAME "passlist"
35
36 # Because this test will take several hours (depends on workloads
37 # and max_running_level) and probably cause more than once system
38 # crashes, it is disabled by default. It's strongly suggested
39 # to run it manually as a single test case if necessary.
40 # Comment the following two lines to enable it.
41 untested "$TESTNAME is disabled"
42 return
43
44 #################################################################
45 # supporting constants and procedures #
46 #################################################################
47 set MAX_RUNNING_LEVEL 2
48
49 set STP_GENWHITELIST_RUNNING "/stp_genpasslist_running"
50 set STAP_RESULT "probe.out"
51 set PROBES_ALL "probes.all"
52 set PROBES_PENDING "probes.pending"
53 set PROBES_CURRENT "probes.current"
54 set PROBES_PASSED "probes.passed"
55 set PROBES_FAILED "probes.failed"
56 set PROBES_UNTRIGGERED "probes.untriggered"
57
58 set systemtap_script {
59 global stat
60 probe %s {
61 stat[pp()] <<< 1
62 }
63 probe timer.ms(30000) {
64 foreach (pp in stat)
65 printf("%%d %%s\n", @count(stat[pp]), pp)
66 }
67 probe end {
68 foreach (pp in stat)
69 printf("%%d %%s\n", @count(stat[pp]), pp)
70 }
71 }
72
73 set benchs {
74 {/usr/local/ltp/runltp -t 300s -f syscalls}
75 {/usr/local/ltp/runltp -t 300s -f nfs}
76 {/usr/local/ltp/runltp -t 300s -f ipc}
77 {/usr/local/ltp/runltp -t 300s -f dio}
78 {/usr/local/ltp/runltp -t 300s -f fs}
79 {/usr/local/ltp/runltp -t 300s -f mm}
80 {/usr/local/ltp/runltp -t 300s -f tcp_cmds}
81 {/usr/local/ltp/runltp -t 300s -f ltp-aio-stress.part1}
82 }
83
84 set init_probes_all_script {
85 probe scheduler.*,
86 ioscheduler.*,
87 ioscheduler.*.return,
88 syscall.*,
89 syscall.*.return,
90 ioblock.*,
91 netdev.*,
92 vm.*,
93 signal.*,
94 signal.*.return,
95 udp.*,
96 udp.*.return,
97 tcp.*,
98 tcp.*.return,
99 kprocess.*,
100 process.*,
101 nfs.fop.*,
102 nfs.aop.*,
103 nfsd.proc.*,
104 nfsd.*,
105 nfs.proc.*,
106 scsi.*,
107 sunrpc.*,
108 generic.fop.*,
109 generic.fop.*.return,
110 vfs.*,
111 vfs.*.return {}
112 }
113
114 proc do_current_test {} {
115 global PROBES_CURRENT
116 global STAP_RESULT
117 global systemtap_script
118 if [catch {open $PROBES_CURRENT r} Infile] {
119 puts "Failed to open $PROBES_CURRENT"
120 return
121 }
122 set probepoint [gets $Infile]
123 while {![eof $Infile]} {
124 set extra [gets $Infile]
125 if {[string length $extra] > 0} {
126 set probepoint [concat "$probepoint, " $extra]
127 set extra [gets $Infile]
128 }
129 }
130 catch {close $Infile}
131
132 set testname "L[get_running_level]_Grp_[string range $probepoint 0 32]..."
133 if [probe_ok $probepoint] {
134 set script [format $systemtap_script $probepoint]
135 passlist_run $testname runbenchs -e $script -o $STAP_RESULT
136 } else {
137 puts "ERROR no match in probe_ok() $testname no match"
138 set script [format $systemtap_script $probepoint]
139 exec echo $script > sc.stp
140 }
141 }
142
143 proc get_linesize {filename} {
144 if [file readable $filename] {
145 scan [exec wc -l $filename] "%d" lines
146 if {[info exists lines] && $lines > 0} {
147 return $lines
148 }
149 }
150 return 0
151 }
152
153 proc get_running_level {} {
154 global STP_GENWHITELIST_RUNNING
155 if [file readable $STP_GENWHITELIST_RUNNING] {
156 scan [exec cat $STP_GENWHITELIST_RUNNING] "%d" current_running_level
157 if {[info exists current_running_level] && $current_running_level > 0} {
158 return $current_running_level
159 }
160 }
161 return 0
162 }
163
164 set NO_CRASH 0
165 set ALREADY_CRASHED 1
166 proc garbage_collect {{already_crashed $NO_CRASH}} {
167 global STAP_RESULT
168 global PROBES_CURRENT
169 global PROBES_PASSED
170 global PROBES_FAILED
171 global PROBES_UNTRIGGERED
172
173 if {[get_linesize $PROBES_CURRENT] == 0} {
174 exec rm -f $PROBES_CURRENT $STAP_RESULT
175 return
176 }
177 if {[get_linesize $STAP_RESULT] == 0} {
178 if {$already_crashed} {
179 exec cat $PROBES_CURRENT >> $PROBES_FAILED
180 } else {
181 exec cat $PROBES_CURRENT >> $PROBES_UNTRIGGERED
182 }
183 exec rm -f $PROBES_CURRENT $STAP_RESULT
184 return
185 }
186
187 # both probes.current and probe.out are non-empty
188 if [catch {open $PROBES_CURRENT r} Infile] {
189 puts "Failed to open $PROBES_CURRENT"
190 exec rm -f $PROBES_CURRENT $STAP_RESULT
191 return
192 }
193 set probepoint [gets $Infile]
194 while {![eof $Infile]} {
195 if {[is_probed $probepoint $STAP_RESULT]} {
196 exec echo $probepoint >> $PROBES_PASSED
197 } else {
198 if {$already_crashed} {
199 exec echo $probepoint >> $PROBES_FAILED
200 } else {
201 exec echo $probepoint >> $PROBES_UNTRIGGERED
202 }
203 }
204 set probepoint [gets $Infile]
205 }
206 catch {close $Infile}
207 exec rm -f $PROBES_CURRENT $STAP_RESULT
208 return
209 }
210
211 proc incr_running_level {} {
212 global STP_GENWHITELIST_RUNNING
213 set newlevel [expr [get_running_level]+1]
214 if { $newlevel > 0 } {
215 exec echo $newlevel > $STP_GENWHITELIST_RUNNING
216 } else {
217 exec echo 0 > $STP_GENWHITELIST_RUNNING
218 }
219 return
220 }
221
222 proc init_probes_all {} {
223 global PROBES_ALL
224 global init_probes_all_script
225 catch {exec stap -p2 -e $init_probes_all_script > /tmp/passlist_tmpfile}
226 catch {exec grep "^kernel.function" /tmp/passlist_tmpfile > $PROBES_ALL }
227 catch {exec rm -f /tmp/passlist_tmpfile}
228 if {[get_linesize $PROBES_ALL] == 0} {
229 return 1
230 }
231 return 0
232 }
233
234 proc init_running_level {} {
235 global STP_GENWHITELIST_RUNNING
236 exec echo 1 > $STP_GENWHITELIST_RUNNING
237 }
238
239 proc is_probed {probepoint stap_result} {
240 if {[string length $probepoint] == 0 || [get_linesize $stap_result] == 0 } {
241 return 0
242 }
243 if [catch {open $stap_result r} Resfile] {
244 puts "Failed to open $stap_result in is_probed() proc"
245 return 0
246 }
247 set probed 0
248 set resline [gets $Resfile]
249 while {![eof $Resfile]} {
250 if {[string match "* $probepoint" $resline]} {
251 set probed 1
252 break
253 }
254 set resline [gets $Resfile]
255 }
256 catch {close $Resfile}
257 return $probed
258 }
259
260 proc probe_ok {probepoint} {
261 set cmd {exec stap -u -p2 -e }
262 lappend cmd "probe $probepoint {}"
263 exec echo $cmd > cmdfile
264 return ![catch $cmd]
265 }
266
267 proc proper_current_size {level inputfile} {
268 set totalsize [get_linesize $inputfile]
269 switch $level {
270 {1} {set currentsize [expr $totalsize/9]}
271 {2} {set currentsize [expr $totalsize/49]}
272 {3} {set currentsize 50}
273 {4} {set currentsize 10}
274 default {
275 puts "Reached unexpected iteration level: $level"
276 set currentsize $totalsize
277 }
278 }
279 if {$currentsize <= 0} {
280 set currentsize 5
281 }
282 return $currentsize
283 }
284
285 set startup_line_in_RCLOCAL "cd $env(PWD); runtest passlist.exp&"
286 set RCLOCAL "/etc/rc.d/rc.local"
287
288 proc register_service {} {
289 global startup_line_in_RCLOCAL
290 global RCLOCAL
291 exec sed -i -n -e "/runtest passlist.exp/!p" $RCLOCAL
292 exec echo $startup_line_in_RCLOCAL >> $RCLOCAL
293 }
294
295 proc unregister_service {} {
296 global RCLOCAL
297 exec sed -i -n -e "/runtest passlist.exp/!p" $RCLOCAL
298 }
299
300 proc passlist_run { TEST_NAME {LOAD_GEN_FUNCTION ""} args } {
301 set cmd [concat {stap -DMAXSKIPPED=200000 -v } $args]
302 catch {eval spawn $cmd}
303 set stap_id $spawn_id
304 set failed 1
305 expect {
306 -timeout 1800
307 -i $stap_id -re {^Pass\ ([1234]):\ [^\r]*\r\n} {
308 set error_msg "pass$expect_out(1,string)";
309 exp_continue
310 }
311 -re {Pass\ 5:\ starting\ run.\r\n} {
312 set error_msg "stap runtime"
313 runbenchs
314 kill -INT -[exp_pid -i $stap_id]
315 exp_continue
316 }
317 -re {Pass\ 5:\ run\ completed} {
318 set failed 0
319 }
320 -re {parse\ error|semantic\ error} { set detail "$expect_out(0,string)" }
321 timeout { set detail "stap timeout"; kill -INT -[exp_pid -i $stap_id] }
322 eof { set failed 0 }
323 }
324 catch {close -i $stap_id}
325 wait -i $stap_id
326 if {$failed} { puts "passlist_run failure \($detail\)" }
327 }
328
329 proc runbenchs {} {
330 global benchs
331 set runningcount 0
332
333 foreach bench $benchs {
334 set benchexec [lrange $bench 0 0]
335 set benchargs [join [lrange $bench 1 [llength $bench]]]
336 if {[file executable $benchexec]} {
337 catch {eval spawn $benchexec $benchargs}
338 set benchname($spawn_id) $benchexec
339 } else {
340 catch {eval spawn sleep 30}
341 set benchname($spawn_id) "sleep"
342 }
343 lappend idlist $spawn_id
344 incr runningcount
345 }
346
347 while {$runningcount > 0} {
348 expect {
349 -timeout 900
350 -i $idlist -re {LTP\ Version:\ LTP-([0-9])+\r\n$} {
351 set from $expect_out(spawn_id)
352 lappend benchres($from) $expect_out(buffer)
353 } eof {
354 set donepos [lsearch $idlist $expect_out(spawn_id)]
355 set idlist [lreplace $idlist $donepos $donepos]
356 incr runningcount -1
357 catch { close -i $expect_out(spawn_id) }
358 wait -i $expect_out(spawn_id)
359 } timeout {
360 break
361 }
362 }
363 }
364 }
365
366 ###################################################################
367 # Main routine of the whole test #
368 ###################################################################
369 if {[info procs installtest_p] != "" && ![installtest_p]} {
370 untested $TESTNAME;
371 return
372 }
373
374 if {[get_running_level] == 0} {
375 # Not started yet, start the whole test from the scratch
376 # Append the startup code to /etc/rc.d/rc.local if not yet
377 register_service
378 # Check whether probes.all is empty or not given
379 if {[get_linesize $PROBES_ALL] == 0} {
380 if {[init_probes_all] != 0} {
381 fail "$TESTNAME unable to initialize the probe point list"
382 return
383 }
384 }
385 # Set current_running_level as 1 to indicate a new test started
386 init_running_level
387 # Initialize intermediate files based on probe.all
388 exec rm -f $PROBES_PENDING $PROBES_CURRENT
389 exec rm -f $PROBES_PASSED $PROBES_UNTRIGGERED $PROBES_FAILED
390 file copy $PROBES_ALL $PROBES_PENDING
391 exec touch $PROBES_PASSED $PROBES_UNTRIGGERED $PROBES_FAILED
392 puts "Start a fresh stp_genpasslist test."
393 } else {
394 # Maybe started already, so do some cleaning if necessary
395 garbage_collect $ALREADY_CRASHED
396 puts "Recovered from last maybe crashed probe test."
397 }
398
399 set current_size_const [proper_current_size [get_running_level] $PROBES_ALL]
400 puts "current_size_const is initialized as $current_size_const"
401
402 while {1} {
403 puts "Current size of probes.pending is [get_linesize $PROBES_PENDING]"
404 if {[get_linesize $PROBES_PENDING] == 0} {
405 # Check whether we need the next iteration or not
406 global MAX_RUNNING_LEVEL
407 # incr running_level for the start of a new iteration
408 incr_running_level
409 puts "Running level increased to [get_running_level]"
410 if {[get_running_level] > $MAX_RUNNING_LEVEL} {
411 puts "Exceed max running level limit."
412 break
413 } else {
414 puts "Current running level is [get_running_level]"
415 exec rm -f $PROBES_PENDING
416 if {[get_linesize $PROBES_FAILED] > 0} {
417 # Append probes.failed to probes.pending
418 exec cat $PROBES_FAILED >> $PROBES_PENDING
419 file delete $PROBES_FAILED
420 exec touch $PROBES_FAILED
421 puts "Append $PROBES_FAILED to $PROBES_PENDING"
422 }
423 if {[get_linesize $PROBES_UNTRIGGERED] > 0} {
424 # Append probes.untriggered to probes.pending
425 exec cat $PROBES_UNTRIGGERED >> $PROBES_PENDING
426 file delete $PROBES_UNTRIGGERED
427 exec touch $PROBES_UNTRIGGERED
428 puts "Append $PROBES_UNTRIGGERED to $PROBES_PENDING"
429 }
430 if {[get_linesize $PROBES_PENDING] == 0} {
431 # No more pending probe points
432 puts "No more iterations needed. Stopped."
433 break
434 }
435 # set new value of current_size_const for new iteration level
436 set current_size_const [proper_current_size [get_running_level] $PROBES_ALL]
437 puts "current_size_const is set as $current_size_const now"
438 continue
439 }
440 }
441 # Now, non-empty probes.pending should be ready
442 # Generate probes.current
443 exec rm -f $PROBES_CURRENT
444 exec head -n $current_size_const $PROBES_PENDING > $PROBES_CURRENT
445 exec tail -n+[expr $current_size_const+1] $PROBES_PENDING > /tmp/passlist_tmpfile
446 exec mv /tmp/passlist_tmpfile $PROBES_PENDING
447
448 puts "Start a probe test..."
449 # Do actual probe test
450 do_current_test
451 puts "Completed one probe test."
452
453 # No crash fortunately, so do some cleaning to prepare for next test
454 garbage_collect $NO_CRASH
455 }
456
457 # Congratulations for arriving here
458 # Remove all temporary files and unregister myself
459 puts "Remove all temporary files, unregister the service and return."
460 exec rm -f $PROBES_PENDING $PROBES_CURRENT $STP_GENWHITELIST_RUNNING
461 exec rm -f $STAP_RESULT /tmp/passlist_tmpfile
462 unregister_service
463
464 pass "$TESTNAME completed"
465 return