Debian Codegraph @

perf/vg_perf

raw
1 #! /usr/bin/perl
2 ##--------------------------------------------------------------------##
3 ##--- Valgrind performance testing script vg_perf ---##
4 ##--------------------------------------------------------------------##
5
6 # This file is part of Valgrind, a dynamic binary instrumentation
7 # framework.
8 #
9 # Copyright (C) 2005-2017 Nicholas Nethercote
10 # [email protected]
11 #
12 # This program is free software; you can redistribute it and/or
13 # modify it under the terms of the GNU General Public License as
14 # published by the Free Software Foundation; either version 2 of the
15 # License, or (at your option) any later version.
16 #
17 # This program is distributed in the hope that it will be useful, but
18 # WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 # General Public License for more details.
21 #
22 # You should have received a copy of the GNU General Public License
23 # along with this program; if not, see <http://www.gnu.org/licenses/>.
24 #
25 # The GNU General Public License is contained in the file COPYING.
26
27 #----------------------------------------------------------------------------
28 # usage: see usage message.
29 #
30 # You can specify individual files to test, or whole directories, or both.
31 # Directories are traversed recursively, except for ones named, for example,
32 # CVS/ or docs/.
33 #
34 # Each test is defined in a file <test>.vgperf, containing one or more of the
35 # following lines, in any order:
36 # - prog: <prog to run> (compulsory)
37 # - args: <args for prog> (default: none)
38 # - vgopts: <Valgrind options> (default: none)
39 # - prereq: <prerequisite command> (default: none)
40 # - cleanup: <post-test cleanup cmd to run> (default: none)
41 #
42 # The prerequisite command, if present, must return 0 otherwise the test is
43 # skipped.
44 # Sometimes it is useful to run all the tests at a high sanity check
45 # level or with arbitrary other flags. To make this simple, extra
46 # options, applied to all tests run, are read from $EXTRA_REGTEST_OPTS,
47 # and handed to valgrind prior to any other flags specified by the
48 # .vgperf file. Note: the env var is the same as vg_regtest.
49 #----------------------------------------------------------------------------
50
51 use warnings;
52 use strict;
53
54 #----------------------------------------------------------------------------
55 # Global vars
56 #----------------------------------------------------------------------------
57 my $usage = <<END
58 usage: vg_perf [options] [files or dirs]
59
60 options for the user, with defaults in [ ], are:
61 -h --help show this message
62 --reps=<n> number of repeats for each program [1]
63 --tools=<t1,t2,t3> tools to run [Nulgrind and Memcheck]
64 --vg=<dir> top-level directory containing Valgrind to measure
65 [Valgrind in the current directory, i.e. --vg=.]
66 Can be specified multiple times.
67 The "in-place" build is used.
68 --terse: terse output. Prints only program name and speedup's for specified
69 tools.
70
71 --outer-valgrind: run these Valgrind(s) under the given outer valgrind.
72 These Valgrind(s) must be configured with --enable-inner.
73 --outer-tool: tool to use by the outer valgrind (default cachegrind).
74 --outer-args: use this as outer tool args. If the outer args are starting
75 with +, the given outer args are appended to the outer args predefined
76 by vg_perf.
77
78 Any tools named in --tools must be present in all directories specified
79 with --vg. (This is not checked.)
80 Use EXTRA_REGTEST_OPTS to supply extra args for all tests
81 END
82 ;
83
84 # Test variables
85 my $vgopts; # valgrind options
86 my $prog; # test prog
87 my $args; # test prog args
88 my $prereq; # prerequisite test to satisfy before running test
89 my $cleanup; # cleanup command to run
90
91 # Command line options
92 my $n_reps = 1; # Run each test $n_reps times and choose the best one.
93 my @vgdirs; # Dirs of the various Valgrinds being measured.
94 my @tools = ("none", "memcheck"); # tools being measured
95 my $terse = 0; # Terse output.
96
97 # Outer valgrind to use, and args to use for it.
98 # If this is set, --valgrind should be set to the installed inner valgrind,
99 # and --valgrind-lib will be ignore
100 my $outer_valgrind;
101 my $outer_tool = "cachegrind";
102 my $outer_args;
103
104
105 my $num_tests_done = 0;
106 my $num_timings_done = 0;
107
108 # Starting directory
109 chomp(my $tests_dir = `pwd`);
110
111 #----------------------------------------------------------------------------
112 # Process command line, setup
113 #----------------------------------------------------------------------------
114
115 # If $prog is a relative path, it prepends $dir to it. Useful for two reasons:
116 #
117 # 1. Can prepend "." onto programs to avoid trouble with users who don't have
118 # "." in their path (by making $dir = ".")
119 # 2. Can prepend the current dir to make the command absolute to avoid
120 # subsequent trouble when we change directories.
121 #
122 # Also checks the program exists and is executable.
123 sub validate_program ($$$$)
124 {
125 my ($dir, $prog, $must_exist, $must_be_executable) = @_;
126
127 # If absolute path, leave it alone. If relative, make it
128 # absolute -- by prepending current dir -- so we can change
129 # dirs and still use it.
130 $prog = "$dir/$prog" if ($prog !~ /^\//);
131 if ($must_exist) {
132 (-f $prog) or die "vg_perf: '$prog' not found or not a file ($dir)\n";
133 }
134 if ($must_be_executable) {
135 (-x $prog) or die "vg_perf: '$prog' not executable ($dir)\n";
136 }
137
138 return $prog;
139 }
140
141 sub add_vgdir($)
142 {
143 my ($vgdir) = @_;
144 if ($vgdir !~ /^\//) { $vgdir = "$tests_dir/$vgdir"; }
145 push(@vgdirs, $vgdir);
146 }
147
148 sub process_command_line()
149 {
150 my @fs;
151
152 for my $arg (@ARGV) {
153 if ($arg =~ /^-/) {
154 if ($arg =~ /^--reps=(\d+)$/) {
155 $n_reps = $1;
156 if ($n_reps < 1) { die "bad --reps value: $n_reps\n"; }
157 } elsif ($arg =~ /^--vg=(.+)$/) {
158 # Make dir absolute if not already
159 add_vgdir($1);
160 } elsif ($arg =~ /^--tools=(.+)$/) {
161 @tools = split(/,/, $1);
162 } elsif ($arg =~ /^--terse$/) {
163 $terse = 1;
164 } elsif ($arg =~ /^--outer-valgrind=(.*)$/) {
165 $outer_valgrind = $1;
166 } elsif ($arg =~ /^--outer-tool=(.*)$/) {
167 $outer_tool = $1;
168 } elsif ($arg =~ /^--outer-args=(.*)$/) {
169 $outer_args = $1;
170 } else {
171 die $usage;
172 }
173 } else {
174 push(@fs, $arg);
175 }
176 }
177
178 # If no --vg options were specified, use the current tree.
179 if (0 == @vgdirs) {
180 add_vgdir($tests_dir);
181 }
182
183 (0 != @fs) or die "No test files or directories specified\n";
184
185 return @fs;
186 }
187
188 #----------------------------------------------------------------------------
189 # Read a .vgperf file
190 #----------------------------------------------------------------------------
191 sub read_vgperf_file($)
192 {
193 my ($f) = @_;
194
195 # Defaults.
196 ($vgopts, $prog, $args, $prereq, $cleanup)
197 = ("", undef, "", undef, undef, undef, undef);
198
199 open(INPUTFILE, "< $f") || die "File $f not openable\n";
200
201 while (my $line = <INPUTFILE>) {
202 if ($line =~ /^\s*#/ || $line =~ /^\s*$/) {
203 next;
204 } elsif ($line =~ /^\s*vgopts:\s*(.*)$/) {
205 $vgopts = $1;
206 } elsif ($line =~ /^\s*prog:\s*(.*)$/) {
207 $prog = validate_program(".", $1, 1, 1);
208 } elsif ($line =~ /^\s*args:\s*(.*)$/) {
209 $args = $1;
210 } elsif ($line =~ /^\s*prereq:\s*(.*)$/) {
211 $prereq = $1;
212 } elsif ($line =~ /^\s*cleanup:\s*(.*)$/) {
213 $cleanup = $1;
214 } else {
215 die "Bad line in $f: $line\n";
216 }
217 }
218 close(INPUTFILE);
219
220 if (!defined $prog) {
221 $prog = ""; # allow no prog for testing error and --help cases
222 }
223 if (0 == @tools) {
224 die "vg_perf: missing 'tools' line in $f\n";
225 }
226 }
227
228 #----------------------------------------------------------------------------
229 # Do one test
230 #----------------------------------------------------------------------------
231 # Since most of the program time is spent in system() calls, need this to
232 # propagate a Ctrl-C enabling us to quit.
233 sub mysystem($)
234 {
235 my ($cmd) = @_;
236 my $retval = system($cmd);
237 if ($retval == 2) {
238 exit 1;
239 } else {
240 return $retval;
241 }
242 }
243
244 # Run program N times, return the best user time. Use the POSIX
245 # -p flag on /usr/bin/time so as to get something parseable on AIX.
246 sub time_prog($$)
247 {
248 my ($cmd, $n) = @_;
249 my $tmin = 999999;
250 for (my $i = 0; $i < $n; $i++) {
251 mysystem("echo '$cmd' > perf.cmd");
252 my $retval = mysystem("$cmd > perf.stdout 2> perf.stderr");
253 (0 == $retval) or
254 die "\n*** Command returned non-zero ($retval)"
255 . "\n*** See perf.{cmd,stdout,stderr} to determine what went wrong.\n";
256 my $out = `cat perf.stderr`;
257 ($out =~ /[Uu]ser +([\d\.]+)/) or
258 die "\n*** missing usertime in perf.stderr\n";
259 $tmin = $1 if ($1 < $tmin);
260 }
261
262 # Successful run; cleanup
263 unlink("perf.cmd");
264 unlink("perf.stderr");
265 unlink("perf.stdout");
266
267 # Avoid divisions by zero!
268 return (0 == $tmin ? 0.01 : $tmin);
269 }
270
271 sub do_one_test($$)
272 {
273 my ($dir, $vgperf) = @_;
274 $vgperf =~ /^(.*)\.vgperf/;
275 my $name = $1;
276 my %first_tTool; # For doing percentage speedups when comparing
277 # multiple Valgrinds
278
279 read_vgperf_file($vgperf);
280
281 if (defined $prereq) {
282 if (system("$prereq") != 0) {
283 printf("%-16s (skipping, prereq failed: $prereq)\n", "$name:");
284 return;
285 }
286 }
287
288 my $timecmd = "/usr/bin/time -p";
289
290 # Do the native run(s).
291 printf("-- $name --\n") if (@vgdirs > 1);
292 my $cmd = "$timecmd $prog $args";
293 my $tNative = time_prog($cmd, $n_reps);
294
295 if (defined $outer_valgrind) {
296 $outer_valgrind = validate_program($tests_dir, $outer_valgrind, 1, 1);
297 foreach my $vgdir (@vgdirs) {
298 validate_program($vgdir, "./coregrind/valgrind", 1, 1);
299 }
300 } else {
301 foreach my $vgdir (@vgdirs) {
302 validate_program($vgdir, "./coregrind/valgrind", 1, 1);
303 }
304 }
305
306 # Pull any extra options (for example, --sanity-level=4)
307 # from $EXTRA_REGTEST_OPTS.
308 my $maybe_extraopts = $ENV{"EXTRA_REGTEST_OPTS"};
309 my $extraopts = $maybe_extraopts ? $maybe_extraopts : "";
310
311 foreach my $vgdir (@vgdirs) {
312 # Benchmark name
313 if (!$terse) {
314 printf("%-8s ", $name);
315 }
316
317 # Print the Valgrind version if we are measuring more than one.
318 my $vgdirname = $vgdir;
319 chomp($vgdirname = `basename $vgdir`);
320 printf("%-10s:", $vgdirname);
321
322 # Native execution time
323 if (!$terse) {
324 printf("%4.2fs", $tNative);
325 }
326
327 foreach my $tool (@tools) {
328 # First two chars of toolname for abbreviation
329 my $tool_abbrev = $tool;
330 $tool_abbrev =~ s/(..).*/$1/;
331 printf(" %s:", $tool_abbrev);
332 my $run_outer_args = "";
333 if ((not defined $outer_args) || ($outer_args =~ /^\+/)) {
334 $run_outer_args =
335 " -v --command-line-only=yes"
336 . " --sim-hints=enable-outer"
337 . " --run-libc-freeres=no --run-cxx-freeres=no"
338 . " --smc-check=all-non-file"
339 . " --vgdb=no --trace-children=yes --read-var-info=no"
340 . " --suppressions=../tests/outer_inner.supp"
341 . " --memcheck:leak-check=full --memcheck:show-reachable=no"
342 . " --cachegrind:cache-sim=yes --cachegrind:branch-sim=yes"
343 . " --cachegrind:cachegrind-out-file=cachegrind.out.$vgdirname.$tool_abbrev.$name.%p"
344 . " --callgrind:cache-sim=yes --callgrind:branch-sim=yes"
345 . " --callgrind:dump-instr=yes --callgrind:collect-jumps=yes"
346 . " --callgrind:callgrind-out-file=callgrind.out.$vgdirname.$tool_abbrev.$name.%p"
347 . " ";
348 if (defined $outer_args) {
349 $outer_args =~ s/^\+(.*)/$1/;
350 $run_outer_args = $run_outer_args . $outer_args;
351 }
352 } else {
353 $run_outer_args = $outer_args;
354 }
355
356 my $vgsetup = "";
357 my $vgcmd = "$vgdir/coregrind/valgrind "
358 . "--command-line-only=yes --tool=$tool $extraopts -q "
359 . "--memcheck:leak-check=no "
360 . "--trace-children=yes "
361 . "$vgopts ";
362 # Do the tool run(s).
363 if (defined $outer_valgrind ) {
364 # in an outer-inner setup, only set VALGRIND_LIB_INNER
365 $vgsetup = "VALGRIND_LIB_INNER=$vgdir/.in_place ";
366 $vgcmd = "$outer_valgrind "
367 . "--tool=" . $outer_tool . " "
368 . "--log-file=" . "$outer_tool.outer.log.$vgdirname.$tool_abbrev.$name.%p "
369 . "$run_outer_args "
370 . $vgcmd;
371 } else {
372 # Set both VALGRIND_LIB and VALGRIND_LIB_INNER
373 # in case this Valgrind was configured with --enable-inner. And
374 # also VALGRINDLIB, which was the old name for the variable, to
375 # allow comparison against old Valgrind versions (eg. 2.4.X).
376 $vgsetup = "VALGRINDLIB=$vgdir/.in_place "
377 . "VALGRIND_LIB=$vgdir/.in_place "
378 . "VALGRIND_LIB_INNER=$vgdir/.in_place ";
379 }
380 my $cmd = "$vgsetup $timecmd $vgcmd $prog $args";
381 my $tTool = time_prog($cmd, $n_reps);
382 if (!$terse) {
383 printf("%4.1fs (%4.1fx,", $tTool, $tTool/$tNative);
384 }
385
386 # If it's the first timing for this tool on this benchmark,
387 # record the time so we can get the percentage speedup of the
388 # subsequent Valgrinds. Otherwise, compute and print
389 # the speedup.
390 if (not defined $first_tTool{$tool}) {
391 $first_tTool{$tool} = $tTool;
392 print(" -----");
393 } else {
394 my $speedup = 100 - (100 * $tTool / $first_tTool{$tool});
395 printf("%5.1f%%", $speedup);
396 }
397 if (!$terse) {
398 print(")");
399 }
400
401 $num_timings_done++;
402
403 if (defined $cleanup) {
404 (system("$cleanup") == 0) or
405 print(" ($name cleanup operation failed: $cleanup)\n");
406 }
407 }
408 printf("\n");
409 }
410
411 $num_tests_done++;
412 }
413
414 #----------------------------------------------------------------------------
415 # Test one directory (and any subdirs)
416 #----------------------------------------------------------------------------
417 sub test_one_dir($$); # forward declaration
418
419 sub test_one_dir($$)
420 {
421 my ($dir, $prev_dirs) = @_;
422 $dir =~ s/\/$//; # trim a trailing '/'
423
424 chomp(my $initial_dir = `pwd`); # record where we started
425
426 # Ignore dirs into which we should not recurse.
427 if ($dir =~ /^(BitKeeper|CVS|SCCS|docs|doc)$/) { return; }
428
429 chdir($dir) or die "Could not change into $dir\n";
430
431 # Nb: Don't prepend a '/' to the base directory
432 my $full_dir = $prev_dirs . ($prev_dirs eq "" ? "" : "/") . $dir;
433 my $dashes = "-" x (50 - length $full_dir);
434
435 my @fs = glob "*";
436 my $found_tests = (0 != (grep { $_ =~ /\.vgperf$/ } @fs));
437
438 if ($found_tests) {
439 print "-- Running tests in $full_dir $dashes\n";
440 }
441 foreach my $f (@fs) {
442 if (-d $f) {
443 test_one_dir($f, $full_dir);
444 } elsif ($f =~ /\.vgperf$/) {
445 do_one_test($full_dir, $f);
446 }
447 }
448 if ($found_tests) {
449 print "-- Finished tests in $full_dir $dashes\n";
450 }
451
452 chdir("$initial_dir");
453 }
454
455 #----------------------------------------------------------------------------
456 # Summarise results
457 #----------------------------------------------------------------------------
458 sub summarise_results
459 {
460 printf("\n== %d programs, %d timings =================\n\n",
461 $num_tests_done, $num_timings_done);
462 }
463
464 #----------------------------------------------------------------------------
465 # main()
466 #----------------------------------------------------------------------------
467 sub warn_about_EXTRA_REGTEST_OPTS()
468 {
469 print "WARNING: \$EXTRA_REGTEST_OPTS is set. You probably don't want\n";
470 print "to run the perf tests with it set, unless you are doing some\n";
471 print "strange experiment, and/or you really know what you are doing.\n";
472 print "\n";
473 }
474
475 # nuke VALGRIND_OPTS
476 $ENV{"VALGRIND_OPTS"} = "";
477
478 if ($ENV{"EXTRA_REGTEST_OPTS"}) {
479 print "\n";
480 warn_about_EXTRA_REGTEST_OPTS();
481 }
482
483 my @fs = process_command_line();
484 foreach my $f (@fs) {
485 if (-d $f) {
486 test_one_dir($f, "");
487 } else {
488 # Allow the .vgperf suffix to be given or omitted
489 if ($f =~ /.vgperf$/ && -r $f) {
490 # do nothing
491 } elsif (-r "$f.vgperf") {
492 $f = "$f.vgperf";
493 } else {
494 die "`$f' neither a directory nor a readable test file/name\n"
495 }
496 my $dir = `dirname $f`; chomp $dir;
497 my $file = `basename $f`; chomp $file;
498 chdir($dir) or die "Could not change into $dir\n";
499 do_one_test($dir, $file);
500 chdir($tests_dir);
501 }
502 }
503 summarise_results();
504
505 if ($ENV{"EXTRA_REGTEST_OPTS"}) {
506 warn_about_EXTRA_REGTEST_OPTS();
507 }
508
509 ##--------------------------------------------------------------------##
510 ##--- end ---##
511 ##--------------------------------------------------------------------##