| 1 |
(* |
| 2 |
* This file is part of Coccinelle, lincensed under the terms of the GPL v2. |
| 3 |
* See copyright.txt in the Coccinelle source code for more information. |
| 4 |
* The Coccinelle source code can be obtained at http://coccinelle.lip6.fr |
| 5 |
*) |
| 6 |
|
| 7 |
(* ------------------------------------------------------------------------- *) |
| 8 |
|
| 9 |
(* Regression tests. |
| 10 |
* Process for spgen: |
| 11 |
* - file.cocci (original cocci file) |
| 12 |
* - file.config (config file for running spgen) |
| 13 |
* - file.expected (expected spgenerated file) |
| 14 |
* |
| 15 |
* Run <spgen file.cocci --config file.config -o file.actual.cocci> |
| 16 |
* Compare file.actual.cocci with file.expected |
| 17 |
* Run <spatch --parse-cocci file.actual.cocci -D context>. |
| 18 |
*) |
| 19 |
|
| 20 |
(* ------------------------------------------------------------------------- *) |
| 21 |
|
| 22 |
(* some common functions *) |
| 23 |
let spf = Common.spf (* Printf.sprintf *) |
| 24 |
let perr_nl = prerr_endline (* Common.pr2 *) |
| 25 |
let perr = prerr_string (* Common.pr_no_nl *) |
| 26 |
let dbe2nm = Common.filename_of_dbe (* (path,file,ext) -> path/file.ext *) |
| 27 |
let nm2dbe = Common.dbe_of_filename (* path/file.ext -> (path,file,ext) *) |
| 28 |
|
| 29 |
(* hardcoded values; timeout for testing + extension names *) |
| 30 |
let timeout_per_file = 60 |
| 31 |
let exp_ext = "expected" |
| 32 |
let act_ext = "actual.cocci" |
| 33 |
let score_ext = "score" (* marshalling format used by Common *) |
| 34 |
|
| 35 |
|
| 36 |
(* ------------------------------------------------------------------------- *) |
| 37 |
|
| 38 |
(* more or less the same as in coccinelle/parsing_c/compare_c.ml. |
| 39 |
* diff flags are |
| 40 |
* -u: unified format (ie. output diffs with - and +) |
| 41 |
* -b: ignore changes in amount of whitespace |
| 42 |
* -B: ignore changes in blank lines |
| 43 |
*) |
| 44 |
let get_diff filename1 filename2 = |
| 45 |
let com = spf "diff -u -b -B %s %s" filename1 filename2 in |
| 46 |
let xs = Common.cmd_to_list com in |
| 47 |
|
| 48 |
(* get rid of the --- and +++ lines *) |
| 49 |
if xs = [] then xs else Common.drop 2 xs |
| 50 |
|
| 51 |
(* Run spgen on <file>.cocci with <file>.config, |
| 52 |
* compare to <file>.expected, |
| 53 |
* add result to <score>. |
| 54 |
*) |
| 55 |
let compare_one score expected = |
| 56 |
|
| 57 |
let (dir, base, ext) = nm2dbe expected in |
| 58 |
let hashkey = base ^ "." ^ ext in |
| 59 |
let actual = dbe2nm (dir, base, act_ext) in |
| 60 |
let cocci = dbe2nm (dir, base, "cocci") in |
| 61 |
let config = dbe2nm (dir, base, "config") in |
| 62 |
|
| 63 |
if ext <> exp_ext then failwith ("expected extension "^exp_ext^", not "^ext); |
| 64 |
if not(Sys.file_exists cocci) then failwith ("no cocci for " ^ expected); |
| 65 |
if not(Sys.file_exists config) then failwith ("no config for " ^ expected); |
| 66 |
|
| 67 |
try |
| 68 |
|
| 69 |
Common.timeout_function "spgen_test" timeout_per_file ( |
| 70 |
fun () -> |
| 71 |
|
| 72 |
perr_nl cocci; |
| 73 |
|
| 74 |
(* spgenerate the file *) |
| 75 |
let options = Spgen.make_options ~year:2000 ~output:actual cocci in |
| 76 |
let _ = Spgen.run options in |
| 77 |
|
| 78 |
(* check that the spgenerated file is parsable. Note that the parsing |
| 79 |
* flag generating_mode must be false (this should be done in spgen.ml). |
| 80 |
*) |
| 81 |
Flag.set_defined_virtual_rules "context"; |
| 82 |
let _ = Parse_cocci.process actual None false in |
| 83 |
|
| 84 |
match get_diff actual expected with |
| 85 |
| [] -> |
| 86 |
|
| 87 |
let _ = if Sys.file_exists actual then Sys.remove actual in |
| 88 |
Hashtbl.add score hashkey Common.Ok |
| 89 |
|
| 90 |
| difflist -> |
| 91 |
|
| 92 |
let difflist = List.map (spf " %s\n") difflist in |
| 93 |
let difflist = String.concat "" difflist in |
| 94 |
let diff = |
| 95 |
spf "INCORRECT: %s\n diff (actual vs expected) = \n%s" |
| 96 |
actual difflist in |
| 97 |
Hashtbl.add score hashkey (Common.Pb diff) |
| 98 |
) |
| 99 |
|
| 100 |
with exn -> |
| 101 |
|
| 102 |
let s = spf "PROBLEM\n exn = %s\n" (Printexc.to_string exn) in |
| 103 |
Hashtbl.add score hashkey (Common.Pb s) |
| 104 |
|
| 105 |
(* Prints regression test statistics and information + updates score files. |
| 106 |
* (perhaps split, but then also have to refactor Common.regression_testing_vs) |
| 107 |
* |
| 108 |
* <test_dir> The directory in which the regression test files are stored |
| 109 |
* <score> The new test information |
| 110 |
* Similar to coccinelle/testing.ml, but with less stuff. |
| 111 |
*) |
| 112 |
let print_update_regression test_dir score = |
| 113 |
|
| 114 |
if Hashtbl.length score <= 0 then failwith "There are no tests results ..."; |
| 115 |
|
| 116 |
let expected_score_file = dbe2nm (test_dir, "SCORE_expected", score_ext) in |
| 117 |
let best_of_both_file = dbe2nm (test_dir, "SCORE_best_of_both", score_ext) in |
| 118 |
let actual_score_file = dbe2nm (test_dir, "SCORE_actual", score_ext) in |
| 119 |
|
| 120 |
perr_nl "--------------------------------"; |
| 121 |
perr_nl "statistics"; |
| 122 |
perr_nl "--------------------------------"; |
| 123 |
|
| 124 |
let print_result (filename, result) = |
| 125 |
perr (spf "%-40s: " filename); |
| 126 |
perr ( |
| 127 |
match result with |
| 128 |
| Common.Ok -> "CORRECT\n" |
| 129 |
| Common.Pb s -> s |
| 130 |
) in |
| 131 |
|
| 132 |
(* hash_to_list also sorts the entries by filename *) |
| 133 |
List.iter print_result (Common.hash_to_list score); |
| 134 |
|
| 135 |
perr_nl "--------------------------------"; |
| 136 |
perr_nl "regression testing information"; |
| 137 |
perr_nl "--------------------------------"; |
| 138 |
perr_nl ("regression file: "^ expected_score_file); |
| 139 |
|
| 140 |
let expected_score = |
| 141 |
if Sys.file_exists expected_score_file then |
| 142 |
Common.load_score expected_score_file() |
| 143 |
else |
| 144 |
let s = Common.empty_score() in |
| 145 |
let _ = Common.save_score s expected_score_file in |
| 146 |
s |
| 147 |
in |
| 148 |
|
| 149 |
(* find and print changes in test results since last time *) |
| 150 |
let new_bestscore = Common.regression_testing_vs score expected_score in |
| 151 |
|
| 152 |
Common.save_score score actual_score_file; |
| 153 |
Common.save_score new_bestscore best_of_both_file; |
| 154 |
Common.print_total_score score; |
| 155 |
|
| 156 |
let (good, total) = Common.total_scores score in |
| 157 |
let (expected_good, expected_total) = Common.total_scores expected_score in |
| 158 |
|
| 159 |
if good = expected_good then begin |
| 160 |
|
| 161 |
perr_nl "Current score is equal to expected score; everything is fine" |
| 162 |
|
| 163 |
end else |
| 164 |
if good < expected_good then begin |
| 165 |
|
| 166 |
perr_nl "Current score is lower than expected :("; |
| 167 |
perr_nl (spf "(was expecting %d but got %d)\n" expected_good good); |
| 168 |
perr_nl "If you think it's normal, then maybe you need to update the"; |
| 169 |
perr_nl (spf "score file %s, copying info from %s." |
| 170 |
expected_score_file actual_score_file) |
| 171 |
|
| 172 |
end else |
| 173 |
(* if good > expected_good then *) begin |
| 174 |
|
| 175 |
perr_nl "Current score is greater than expected :)"; |
| 176 |
perr_nl (spf "(was expecting %d but got %d)" expected_good good); |
| 177 |
perr_nl "Generating new expected score file and saving old one"; |
| 178 |
Common.command2_y_or_no_exit_if_no |
| 179 |
(spf "mv %s %s" expected_score_file (expected_score_file ^ ".save")); |
| 180 |
Common.command2_y_or_no_exit_if_no |
| 181 |
(spf "mv %s %s" best_of_both_file expected_score_file) |
| 182 |
|
| 183 |
end |
| 184 |
|
| 185 |
|
| 186 |
(* ------------------------------------------------------------------------- *) |
| 187 |
(* ENTRY POINT *) |
| 188 |
|
| 189 |
let regression_test ~test_dir = |
| 190 |
|
| 191 |
(* sort expected result files by name *) |
| 192 |
let test_files = dbe2nm (test_dir, "*", exp_ext) in |
| 193 |
let e = Common.glob test_files in |
| 194 |
let e = List.filter (fun f -> Common.filesize f > 0) e in |
| 195 |
let expected_files = List.sort compare e in |
| 196 |
if e = [] then failwith ( |
| 197 |
(spf "No test files with expected extension <.%s> found." exp_ext) ^ |
| 198 |
" Are you sure this is the right directory?" |
| 199 |
); |
| 200 |
|
| 201 |
(* populate score table *) |
| 202 |
let actual_score = Common.empty_score() in |
| 203 |
List.iter (compare_one actual_score) expected_files; |
| 204 |
|
| 205 |
print_update_regression test_dir actual_score |