set test "sdt_misc" # Test miscellaneous features of .mark probes # 1. Test executable built with dtrace generated header file # 2. Test attaching to a running process # 3. Test passing various C types to .mark probes # 4. Test probe in shared object # 5. Test attaching to a running process with markers in a shared object # 6. Test .mark probe wildcard matching # 7. Test V3 $$parms support set v_flags {{-DSTAP_SDT_V1} {-DSTAP_SDT_V2} {-DSTAP_SDT_V3}} set pbtype_mssgs {{V1_uprobe} {V2_uprobe} {V3_uprobe}} set pbtype_flags {{additional_flags=-g} {} {}} set fail_count 0 proc cleanup_handler { verbose } { global pbtype_mssgs global fail_count if { $verbose == 0 && $fail_count == 0 } { foreach x $pbtype_mssgs { catch {exec rm -f libsdt_${x}.so libsdt_${x}_shared.so \ sdt_misc_${x}.x sdt_misc_${x}_loop.x \ sdt_misc_${x}_shared.x sdt_misc_${x}_loop_shared.x \ sdt_misc_${x}.x sdt_misc_${x}_compatibility.x \ sdt_misc_${x}_types.x libsdt_${x}_loop.so } } catch {exec rm -f sdt_misc_.c sdt_misc_.h sdt_misc_.o sdt_types.x \ ,semclean ,semout} } else { verbose -log "kept sdt_misc*.* test files due to failing tests" } } # Start with a clean slate cleanup_handler 0 # Compile a C program to use as the user-space probing target set stap_path $env(SYSTEMTAP_PATH)/stap set sup_dpath "[pwd]/sdt_misc_.d" set sup_hpath "[pwd]/sdt_misc_.h" set sup_opath "[pwd]/sdt_misc_.o" # Iterate pbtype_flags for {set i 0} {$i < [llength $pbtype_flags]} {incr i} { set v_flag [lindex $v_flags $i] set pbtype_flag "[lindex $pbtype_flags $i] additional_flags=$v_flag" set pbtype_mssg [lindex $pbtype_mssgs $i] set sup_exepath "[pwd]/sdt_misc_$pbtype_mssg.x" # Run dtrace if {[installtest_p]} { set dtrace $env(SYSTEMTAP_PATH)/dtrace } else { set dtrace ../dtrace } verbose -log "$dtrace $v_flag --types -h -s $srcdir/$subdir/sdt_misc_.d" if {[catch {exec $dtrace $v_flag --types -h -s \ $srcdir/$subdir/sdt_misc_.d} res]} { verbose -log "unable to run $dtrace: $res" } verbose -log "$dtrace $v_flag --types -G -s $srcdir/$subdir/sdt_misc_.d" if {[catch {exec $dtrace $v_flag --types -G -s \ $srcdir/$subdir/sdt_misc_.d} res]} { verbose -log "unable to run $dtrace: $res" } if {[file exists $sup_hpath] && [file exists $sup_opath]} then { pass "$test dtrace $pbtype_mssg" } else { incr fail_count fail "$test dtrace $pbtype_mssg" cleanup_handler $verbose return } set sup_flags [sdt_includes] set sup_flags "$sup_flags additional_flags=-Wall" set sup_flags "$sup_flags additional_flags=-Werror" set sup_flags "$sup_flags additional_flags=$sup_opath" set sup_flags "$sup_flags additional_flags=-I. $pbtype_flag" set res [target_compile $srcdir/$subdir/sdt_misc.c $sup_exepath \ executable $sup_flags] if { $res != "" } { incr fail_count verbose "target_compile failed: $res" 2 fail "$test compiling $pbtype_mssg" cleanup_handler $verbose return } else { pass "$test compiling $pbtype_mssg" } if {![installtest_p]} { untested $test cleanup_handler $verbose return } # 1. Test executable built with dtrace generated header file foreach runtime [get_runtime_list] { if {$runtime == "dyninst" && $pbtype_mssg != "V3_uprobe"} { continue } set test_xtra "" if {$runtime != ""} { set test_xtra " ($runtime)" verbose -log "spawn $stap_path --runtime=$runtime -c $sup_exepath $srcdir/$subdir/sdt_misc.stp $sup_exepath" spawn $stap_path --runtime=$runtime -c $sup_exepath \ $srcdir/$subdir/sdt_misc.stp $sup_exepath } elseif {[uprobes_p]} { verbose -log "spawn $stap_path -c $sup_exepath $srcdir/$subdir/sdt_misc.stp $sup_exepath" spawn $stap_path -c $sup_exepath $srcdir/$subdir/sdt_misc.stp \ $sup_exepath } else { untested "$test ${pbtype_mssg}" continue } set ok 0 expect { -timeout 180 -re {In test_probe_2 probe 0x2} { incr ok; exp_continue } -re {In test_probe_0 probe 0x3} { incr ok; exp_continue } -re {In test_probe_3 probe 0x3 0x[0-9a-f][0-9a-f]} { incr ok; exp_continue } -re {In test_probe_4 dtrace probe 0x[0-9a-f]} { incr ok; exp_continue } -re {In test_probe_1 probe 0x0} { incr ok; exp_continue } timeout { fail "$test (timeout)" } eof { } } catch {close}; catch {wait} if {$ok == 6} { pass "$test ${pbtype_mssg}${test_xtra}" } else { if {[istarget "arm*-*-*"] && $pbtype_mssg == "V2_uprobe"} { xfail "$test ($ok) ${pbtype_mssg}${test_xtra}" } else { incr fail_count fail "$test ($ok) ${pbtype_mssg}${test_xtra}" } } } # 2. Test attaching to a running process set loop_flags "$sup_flags additional_flags=-DLOOP" set loop_exepath "[pwd]/sdt_misc_${pbtype_mssg}_loop.x" set res [target_compile $srcdir/$subdir/sdt_misc.c $loop_exepath \ executable $loop_flags] if { $res != "" } { incr fail_count verbose "target_compile failed: $res" 2 fail "$test compiling $pbtype_mssg -DLOOP" cleanup_handler $verbose return } else { pass "$test compiling $pbtype_mssg -DLOOP" } foreach runtime [get_runtime_list] { if {$runtime == "dyninst" && $pbtype_mssg != "V3_uprobe"} { continue } set test_xtra "" if {$runtime != ""} { set test_xtra " ($runtime)" if {$runtime == "dyninst"} { # With --runtime=dyninst these tests will fail since # the dyninst runtime needs a target executable # (either '-c exe' or '-x pid'). So, skip these tests # to speed up test execution. xfail "$test ${pbtype_mssg}${test_xtra} attach" continue } # This is perhaps a wee bit clumsy verbose -log "$loop_exepath > ,semclean 2>&1 & TEST=\$! ; \ $stap_path --runtime=$runtime $srcdir/$subdir/sdt_misc.stp $loop_exepath >| ,semout ; \ /bin/kill -SIGINT \$TEST" catch {system "$loop_exepath > ,semclean 2>&1 & TEST=\$! ; \ $stap_path --runtime=$runtime $srcdir/$subdir/sdt_misc.stp $loop_exepath >| ,semout ; \ /bin/kill -SIGINT \$TEST"} } elseif {[uprobes_p]} { # This is perhaps a wee bit clumsy verbose -log "$loop_exepath > ,semclean 2>&1 & TEST=\$! ; \ $stap_path $srcdir/$subdir/sdt_misc.stp $loop_exepath >| ,semout ; \ /bin/kill -SIGINT \$TEST" catch {system "$loop_exepath > ,semclean 2>&1 & TEST=\$! ; \ $stap_path $srcdir/$subdir/sdt_misc.stp $loop_exepath >| ,semout ; \ /bin/kill -SIGINT \$TEST"} } else { untested "$test $pbtype_mssg attach" continue } set ok 0 set ko 0 spawn cat ,semout expect { -re {In test_probe_2 probe 0x2} { incr ok; exp_continue } -re {In test_probe_0 probe 0x3} { incr ok; exp_continue } -re {In test_probe_3 probe 0x3 0x[0-9a-f][0-9a-f]} { incr ok; exp_continue } -re {In test_probe_4 dtrace probe 0x[0-9a-f]} { incr ok; exp_continue } -re {In test_probe_1 probe 0x0} { incr ok; exp_continue } eof { } } catch {close}; catch {wait} if {$ok == 6} { pass "$test ${pbtype_mssg}${test_xtra} attach" } else { if {[istarget "arm*-*-*"] && $pbtype_mssg == "V2_uprobe"} { xfail "$test ($ok) ${pbtype_mssg}${test_xtra} attach" } else { incr fail_count fail "$test ($ok) ${pbtype_mssg}${test_xtra} attach" } } set ok 0 spawn cat ,semclean expect { -timeout 180 -re {PASS} { incr ok; exp_continue } -re {FAIL} { incr ko; exp_continue } timeout { fail "$test (timeout)" } eof { } } catch {close}; catch {wait} if {$ok == 4 && $ko == 0} { pass "$test ${pbtype_mssg}${test_xtra} epilogue" } else { incr fail_count fail "$test ($ok) ${pbtype_mssg}${test_xtra} epilogue" } } # 3. Test passing various C types to .mark probes # same set of flags as sdt.exp set extra_type_flags {{additional_flags=-O2} {additional_flags=-O3} {additional_flags=-std=c89} {additional_flags=-std=c99} {additional_flags=-std=gnu99} {c++ additional_flags=-std=c++98 additional_flags=-x \ additional_flags=c++} {c++ additional_flags=-std=gnu++98 additional_flags=-x \ additional_flags=c++} {c++ additional_flags=-std=c++0x additional_flags=-x \ additional_flags=c++} {c++ additional_flags=-std=gnu++0x additional_flags=-x \ additional_flags=c++} } set extra_type_mssgs {-O2 -O3 c89 c99 gnu99 c++98 gnu++98 c++0x gnu++0x} # __STDC_VERSION__ could be used, but g++ has no equivalent set has_long_long {{} {} additional_flags=-DNO_LONG_LONG {} {} \ additional_flags=-DNO_LONG_LONG \ additional_flags=-DNO_LONG_LONG {} {}} set type_xfail {1 1 0 0 0 0 0 0 0} set type_pedantics_idx {1 1 2 2 2 2 2 2 2} # don't compile -O2 -O3 with -pedantic set type_pedantics {{} additional_flags=-pedantic} set type_pedantic_mssgs {{} -pedantic} for {set j 0} {$j < [llength $extra_type_flags]} {incr j} { set extra_type_mssg [lindex $extra_type_mssgs $j] for {set p 0} {$p < [lindex $type_pedantics_idx $j]} {incr p} { set type_pedantic_mssg [lindex $type_pedantic_mssgs $p] set type_flags "[sdt_includes] additional_flags=-Wall additional_flags=-Werror" set type_flags "$type_flags additional_flags=-I. $pbtype_flag" set type_flags "$type_flags [lindex $extra_type_flags $j] [lindex $type_pedantics $p] [lindex $has_long_long $j]" if {![istarget "i*86-*-*"] && ![istarget "arm*-*-*"] && ![istarget "aarch64-*-*"]} { set type_flags "$type_flags additional_flags=-m64" } set test_suffix "$pbtype_mssg $extra_type_mssg $type_pedantic_mssg" # On ARM these tests are known to fail with timeout for V1 # and V2 probes. Skip them altogether to speed execution. if {[istarget "arm*-*-*"] \ && ($pbtype_mssg == "V1_uprobe" || $pbtype_mssg == "V2_uprobe")} { xfail "$test types $test_suffix" continue } set res [target_compile $srcdir/$subdir/sdt_types.c \ sdt_types.x executable $type_flags] if { $res != "" } { verbose "target_compile failed: $res" 2 # -std=gnu++0x and -std=c++0x are not universally accepted if {[string first "unrecognized command line option" $res] == -1} { incr fail_count fail "$test compiling types $test_suffix" } else { untested "$test compiling types $test_suffix" } untested "$test types $test_suffix" continue ; # type_pedantics_idx } else { pass "$test compiling types $test_suffix" } foreach runtime [get_runtime_list] { if {$runtime == "dyninst" && $pbtype_mssg != "V3_uprobe"} { continue } set test_xtra "" if {$runtime != ""} { set test_xtra " ($runtime)" verbose -log "spawn $stap_path --runtime=$runtime -g -c ./sdt_types.x $srcdir/$subdir/sdt_types.stp ./sdt_types.x # $pbtype_mssg $extra_type_mssg" spawn $stap_path --runtime=$runtime -g -c ./sdt_types.x $srcdir/$subdir/sdt_types.stp ./sdt_types.x } elseif {[uprobes_p]} { verbose -log "spawn $stap_path -g -c ./sdt_types.x $srcdir/$subdir/sdt_types.stp ./sdt_types.x # $pbtype_mssg $extra_type_mssg" spawn $stap_path -g -c ./sdt_types.x $srcdir/$subdir/sdt_types.stp ./sdt_types.x } else { untested "$test types $test_suffix" continue } set test_suffix "${pbtype_mssg}${test_xtra} $extra_type_mssg $type_pedantic_mssg" set ok 0 set notok 0 set fail "" set uns_notok 0 set float_notok 0 expect { -timeout 180 # i386 unsigned may xfail because of sign extension # but fail anything else -re {^sdt_types PASS: [^\r]*\r\n} { incr ok; exp_continue } -indices -re {^sdt_types FAIL: ([^\r ]*)[^\r]*\r\n} { set s $expect_out(1,string); if {[regexp "unsigned" $s] && [istarget "i*86-*-*"]} { incr uns_notok } if {[regexp "float" $s] || [regexp "double" $s]} { incr float_notok } incr notok; set fail "$fail $s"; exp_continue } -re {^[^\r]*\r\n} { incr notok; exp_continue } timeout { fail "$test types (timeout) $test_suffix" } eof { } } catch {close}; catch {wait} if { $notok != 0 || $ok == 0} { if { $notok == $float_notok } { # Ignore float failures for V1, V2, and dyninst if { $pbtype_mssg != "V3_uprobe" || $runtime == "dyninst" } { pass "$test types $test_suffix" } else { incr fail_count fail "$test types $fail $test_suffix" } } elseif { [lindex $type_xfail $j] || $notok == $uns_notok } { xfail "$test types $fail $test_suffix" } else { incr fail_count fail "$test types $fail $test_suffix" } } else { pass "$test types $test_suffix" } } } ; # end type_pedantics_idx } ; # end extra_type_flags # 4. Test probe in shared object set sup41_flags "$sup_flags additional_flags=-shared" set sup41_flags "$sup41_flags additional_flags=-fPIC" set sup41_flags "$sup41_flags additional_flags=-DNO_MAIN" set sup_sopath "[pwd]/libsdt_$pbtype_mssg.so" set sup_exepath "[pwd]/sdt_misc_${pbtype_mssg}_shared.x" set res0 [target_compile $srcdir/$subdir/sdt_misc.c $sup_sopath \ executable $sup41_flags ] set sup42_flags "additional_flags=-Wl,-rpath,[pwd]" set sup42_flags "$sup42_flags additional_flags=-L[pwd] additional_flags=-lsdt_${pbtype_mssg}" set sup42_flags "$sup42_flags additional_flags=-DONLY_MAIN" set res [target_compile $srcdir/$subdir/sdt_misc.c $sup_exepath \ executable $sup42_flags ] if { $res0 != "" || $res != "" } { incr fail_count verbose "target_compile failed: $res0 $res" 2 fail "$test compiling -shared $pbtype_mssg" cleanup_handler $verbose return } else { pass "$test compiling -shared $pbtype_mssg" } foreach runtime [get_runtime_list] { if {$runtime == "dyninst" && $pbtype_mssg != "V3_uprobe"} { continue } set test_xtra "" if {$runtime != ""} { set test_xtra " ($runtime)" verbose -log "spawn $stap_path --runtime=$runtime -c $sup_exepath $srcdir/$subdir/sdt_misc.stp $sup_exepath $sup_sopath" spawn $stap_path --runtime=$runtime -c $sup_exepath $srcdir/$subdir/sdt_misc.stp $sup_exepath $sup_sopath } elseif {[uprobes_p]} { verbose -log "spawn $stap_path -c $sup_exepath $srcdir/$subdir/sdt_misc.stp $sup_exepath $sup_sopath" spawn $stap_path -c $sup_exepath $srcdir/$subdir/sdt_misc.stp $sup_exepath $sup_sopath } else { untested "$test shared $pbtype_mssg" continue } set ok 0 expect { -timeout 180 -re {In test_probe_2 probe 0x2} { incr ok; exp_continue } -re {In test_probe_0 probe 0x3} { incr ok; exp_continue } -re {In test_probe_3 probe 0x3 0x[0-9a-f][0-9a-f]} { incr ok; exp_continue } -re {In test_probe_4 dtrace probe 0x[0-9a-f]} { incr ok; exp_continue } -re {In test_probe_1 probe 0x0} { incr ok; exp_continue } timeout { fail "$test (timeout)" } eof { } } catch {close}; catch {wait} if {$ok == 6} { pass "$test shared ${pbtype_mssg}${test_xtra}" } else { if {[istarget "arm*-*-*"] && $pbtype_mssg == "V2_uprobe"} { xfail "$test ($ok) shared ${pbtype_mssg}${test_xtra}" } else { incr fail_count fail "$test ($ok) shared ${pbtype_mssg}${test_xtra}" } } } # 5. Test attaching to a running process with markers in a shared object set supl_flags "$sup41_flags additional_flags=-DLOOP" set supl_sopath "[pwd]/libsdt_${pbtype_mssg}_loop.so" set res0 [target_compile $srcdir/$subdir/sdt_misc.c $supl_sopath \ executable $supl_flags ] set loop_flags [sdt_includes] set loop_flags "$loop_flags additional_flags=-I. $pbtype_flag" set loop_flags "$loop_flags additional_flags=-DLOOP" set loop_flags "$loop_flags additional_flags=-DONLY_MAIN" set loop_flags "$loop_flags additional_flags=-Wl,-rpath,[pwd]" set loop_flags "$loop_flags additional_flags=-L[pwd] additional_flags=-lsdt_${pbtype_mssg}_loop" set loop_exepath "[pwd]/sdt_misc_${pbtype_mssg}_loop_shared.x" set res [target_compile $srcdir/$subdir/sdt_misc.c $loop_exepath \ executable $loop_flags] set test_suffix "$pbtype_mssg -DLOOP with markers in a shared object" if { $res != "" } { incr fail_count verbose "target_compile failed: $res" 2 fail "$test compiling $test_suffix" cleanup_handler $verbose return } else { pass "$test compiling $test_suffix" } foreach runtime [get_runtime_list] { if {$runtime == "dyninst" && $pbtype_mssg != "V3_uprobe"} { continue } set test_xtra "" if {$runtime != ""} { set test_xtra " ($runtime)" if {$runtime == "dyninst"} { # With --runtime=dyninst these tests will fail since # the dyninst runtime needs a target executable # (either '-c exe' or '-x pid'). So, skip these tests # to speed up test execution. xfail "$test ${pbtype_mssg}${test_xtra} shared attach" continue } # This is perhaps a wee bit clumsy verbose -log "$loop_exepath > ,semclean 2>&1 & TEST=\$! ; \ $stap_path --runtime=$runtime $srcdir/$subdir/sdt_misc.stp $loop_exepath $supl_sopath >| ,semout ; \ /bin/kill -SIGINT \$TEST" catch {system "$loop_exepath > ,semclean 2>&1 & TEST=\$! ; \ $stap_path --runtime=$runtime $srcdir/$subdir/sdt_misc.stp $loop_exepath $supl_sopath >| ,semout ; \ /bin/kill -SIGINT \$TEST"} } elseif {[uprobes_p]} { # This is perhaps a wee bit clumsy verbose -log "$loop_exepath > ,semclean 2>&1 & TEST=\$! ; \ $stap_path $srcdir/$subdir/sdt_misc.stp $loop_exepath $supl_sopath >| ,semout ; \ /bin/kill -SIGINT \$TEST" catch {system "$loop_exepath > ,semclean 2>&1 & TEST=\$! ; \ $stap_path $srcdir/$subdir/sdt_misc.stp $loop_exepath $supl_sopath >| ,semout ; \ /bin/kill -SIGINT \$TEST"} } else { untested "$test $pbtype_mssg shared attach" continue } set ok 0 set ko 0 spawn cat ,semout expect { -re {In test_probe_2 probe 0x2} { incr ok; exp_continue } -re {In test_probe_0 probe 0x3} { incr ok; exp_continue } -re {In test_probe_3 probe 0x3 0x[0-9a-f][0-9a-f]} { incr ok; exp_continue } -re {In test_probe_4 dtrace probe 0x[0-9a-f]} { incr ok; exp_continue } -re {In test_probe_1 probe 0x0} { incr ok; exp_continue } eof { } } catch {close}; catch {wait} if {$ok == 6} { pass "$test ${pbtype_mssg}${test_xtra} shared attach" } else { if {[istarget "arm*-*-*"] && $pbtype_mssg == "V2_uprobe"} { xfail "$test ($ok) ${pbtype_mssg}${test_xtra} shared attach" } else { incr fail_count fail "$test ($ok) ${pbtype_mssg}${test_xtra} shared attach" } } set ok 0 spawn cat ,semclean expect { -timeout 180 -re {PASS} { incr ok; exp_continue } -re {FAIL} { incr ko; exp_continue } timeout { fail "$test (timeout)" } eof { } } catch {close}; catch {wait} if {$ok == 4 && $ko == 0} { pass "$test ${pbtype_mssg}${test_xtra} shared shared epilogue" } else { incr fail_count fail "$test ($ok) ${pbtype_mssg}${test_xtra} shared epilogue" } } # 6. Test .mark probe wildcard matching set ok 0 if {[uprobes_p]} { verbose -log "spawn $stap_path -L \"process(\"./sdt_types.x\").mark(\"*\")" spawn $stap_path -L "process(\"./sdt_types.x\").mark(\"*\")" expect { -timeout 180 -re {mark\(\"[a-z_]+\"\) .arg1:long .arg2:long .arg3:long .arg4:long .arg5:long .arg6:long .arg7:long .arg8:long\n} { incr ok; exp_continue } -re {mark\(\"[a-z_]+\"\) .arg1:long .arg2:long\n} { incr ok; exp_continue } -re {mark\(\"[a-z_]+\"\) .arg1:long\n} { incr ok; exp_continue } -re {mark\(\"[a-z_]+\"\)} { incr ok; exp_continue } timeout { fail "$test (timeout)" } eof { } } catch {close}; catch {wait} # PR10245: match the __ one here spawn $stap_path -l "process(\"./sdt_types.x\").mark(\"*-*-*\")" expect { -timeout 180 -re {mark\(\"[a-z_]+\"\)} { incr ok; exp_continue } timeout { fail "$test (timeout)" } eof { } } catch {close}; catch {wait} if { $ok == 53 || $ok == 50 \ || (($ok == 44 || $ok == 41 ) && ([istarget "i*86-*-*"] \ || [istarget "arm*-*-*"]))} { pass "$test wildcard $pbtype_mssg" } else { incr fail_count fail "$test wildcard ($ok) $pbtype_mssg" } } else { untested "$test wildcard $pbtype_mssg" } } ; # end pbtype_flags # 7. Test V3 $$parms support foreach runtime [get_runtime_list] { if {$runtime == "dyninst" && $pbtype_mssg != "V3_uprobe"} { continue } set test_xtra "" if {$runtime != ""} { set test_xtra " ($runtime)" verbose -log "spawn $stap_path --runtime=$runtime -c ./sdt_misc_V3_uprobe.x -e \"probe process(\"./sdt_misc_V3_uprobe.x\").mark(\"test_probe_5\") {printf(\"%s %s %s\\n\", \$\$provider, \$\$name, \$\$parms)}\"" spawn $stap_path --runtime=$runtime -c ./sdt_misc_V3_uprobe.x -e "probe process(\"./sdt_misc_V3_uprobe.x\").mark(\"test_probe_3\") {printf(\"%s %s %s\\n\", \$\$provider, \$\$name, \$\$parms)}" } elseif {[uprobes_p]} { verbose -log "spawn $stap_path -c ./sdt_misc_V3_uprobe.x -e \"probe process(\"./sdt_misc_V3_uprobe.x\").mark(\"test_probe_5\") {printf(\"%s %s %s\\n\", \$\$provider, \$\$name, \$\$parms)}\"" spawn $stap_path -c ./sdt_misc_V3_uprobe.x -e "probe process(\"./sdt_misc_V3_uprobe.x\").mark(\"test_probe_3\") {printf(\"%s %s %s\\n\", \$\$provider, \$\$name, \$\$parms)}" } else { untested "$test \$\$parms" continue } set ok 0 expect { -timeout 180 -re {sdt_misc test_probe_3 \$arg1=0x3 \$arg2=0x[0-9a-f][0-9a-f]} { incr ok; exp_continue } timeout { fail "$test (timeout)" } eof { } } catch {close}; catch {wait} if {$ok == 1} { pass "$test \$\$parms${test_xtra}" } else { incr fail_count fail "$test ($ok) \$\$parms${test_xtra}" } } cleanup_handler $verbose