Debian Codegraph @

tools/spgen/source/spgen_test.ml

raw
1 (*
2 * This file is part of Coccinelle, lincensed under the terms of the GPL v2.
3 * See copyright.txt in the Coccinelle source code for more information.
4 * The Coccinelle source code can be obtained at http://coccinelle.lip6.fr
5 *)
6
7 (* ------------------------------------------------------------------------- *)
8
9 (* Regression tests.
10 * Process for spgen:
11 * - file.cocci (original cocci file)
12 * - file.config (config file for running spgen)
13 * - file.expected (expected spgenerated file)
14 *
15 * Run <spgen file.cocci --config file.config -o file.actual.cocci>
16 * Compare file.actual.cocci with file.expected
17 * Run <spatch --parse-cocci file.actual.cocci -D context>.
18 *)
19
20 (* ------------------------------------------------------------------------- *)
21
22 (* some common functions *)
23 let spf = Common.spf (* Printf.sprintf *)
24 let perr_nl = prerr_endline (* Common.pr2 *)
25 let perr = prerr_string (* Common.pr_no_nl *)
26 let dbe2nm = Common.filename_of_dbe (* (path,file,ext) -> path/file.ext *)
27 let nm2dbe = Common.dbe_of_filename (* path/file.ext -> (path,file,ext) *)
28
29 (* hardcoded values; timeout for testing + extension names *)
30 let timeout_per_file = 60
31 let exp_ext = "expected"
32 let act_ext = "actual.cocci"
33 let score_ext = "score" (* marshalling format used by Common *)
34
35
36 (* ------------------------------------------------------------------------- *)
37
38 (* more or less the same as in coccinelle/parsing_c/compare_c.ml.
39 * diff flags are
40 * -u: unified format (ie. output diffs with - and +)
41 * -b: ignore changes in amount of whitespace
42 * -B: ignore changes in blank lines
43 *)
44 let get_diff filename1 filename2 =
45 let com = spf "diff -u -b -B %s %s" filename1 filename2 in
46 let xs = Common.cmd_to_list com in
47
48 (* get rid of the --- and +++ lines *)
49 if xs = [] then xs else Common.drop 2 xs
50
51 (* Run spgen on <file>.cocci with <file>.config,
52 * compare to <file>.expected,
53 * add result to <score>.
54 *)
55 let compare_one score expected =
56
57 let (dir, base, ext) = nm2dbe expected in
58 let hashkey = base ^ "." ^ ext in
59 let actual = dbe2nm (dir, base, act_ext) in
60 let cocci = dbe2nm (dir, base, "cocci") in
61 let config = dbe2nm (dir, base, "config") in
62
63 if ext <> exp_ext then failwith ("expected extension "^exp_ext^", not "^ext);
64 if not(Sys.file_exists cocci) then failwith ("no cocci for " ^ expected);
65 if not(Sys.file_exists config) then failwith ("no config for " ^ expected);
66
67 try
68
69 Common.timeout_function "spgen_test" timeout_per_file (
70 fun () ->
71
72 perr_nl cocci;
73
74 (* spgenerate the file *)
75 let options = Spgen.make_options ~year:2000 ~output:actual cocci in
76 let _ = Spgen.run options in
77
78 (* check that the spgenerated file is parsable. Note that the parsing
79 * flag generating_mode must be false (this should be done in spgen.ml).
80 *)
81 Flag.set_defined_virtual_rules "context";
82 let _ = Parse_cocci.process actual None false in
83
84 match get_diff actual expected with
85 | [] ->
86
87 let _ = if Sys.file_exists actual then Sys.remove actual in
88 Hashtbl.add score hashkey Common.Ok
89
90 | difflist ->
91
92 let difflist = List.map (spf " %s\n") difflist in
93 let difflist = String.concat "" difflist in
94 let diff =
95 spf "INCORRECT: %s\n diff (actual vs expected) = \n%s"
96 actual difflist in
97 Hashtbl.add score hashkey (Common.Pb diff)
98 )
99
100 with exn ->
101
102 let s = spf "PROBLEM\n exn = %s\n" (Printexc.to_string exn) in
103 Hashtbl.add score hashkey (Common.Pb s)
104
105 (* Prints regression test statistics and information + updates score files.
106 * (perhaps split, but then also have to refactor Common.regression_testing_vs)
107 *
108 * <test_dir> The directory in which the regression test files are stored
109 * <score> The new test information
110 * Similar to coccinelle/testing.ml, but with less stuff.
111 *)
112 let print_update_regression test_dir score =
113
114 if Hashtbl.length score <= 0 then failwith "There are no tests results ...";
115
116 let expected_score_file = dbe2nm (test_dir, "SCORE_expected", score_ext) in
117 let best_of_both_file = dbe2nm (test_dir, "SCORE_best_of_both", score_ext) in
118 let actual_score_file = dbe2nm (test_dir, "SCORE_actual", score_ext) in
119
120 perr_nl "--------------------------------";
121 perr_nl "statistics";
122 perr_nl "--------------------------------";
123
124 let print_result (filename, result) =
125 perr (spf "%-40s: " filename);
126 perr (
127 match result with
128 | Common.Ok -> "CORRECT\n"
129 | Common.Pb s -> s
130 ) in
131
132 (* hash_to_list also sorts the entries by filename *)
133 List.iter print_result (Common.hash_to_list score);
134
135 perr_nl "--------------------------------";
136 perr_nl "regression testing information";
137 perr_nl "--------------------------------";
138 perr_nl ("regression file: "^ expected_score_file);
139
140 let expected_score =
141 if Sys.file_exists expected_score_file then
142 Common.load_score expected_score_file()
143 else
144 let s = Common.empty_score() in
145 let _ = Common.save_score s expected_score_file in
146 s
147 in
148
149 (* find and print changes in test results since last time *)
150 let new_bestscore = Common.regression_testing_vs score expected_score in
151
152 Common.save_score score actual_score_file;
153 Common.save_score new_bestscore best_of_both_file;
154 Common.print_total_score score;
155
156 let (good, total) = Common.total_scores score in
157 let (expected_good, expected_total) = Common.total_scores expected_score in
158
159 if good = expected_good then begin
160
161 perr_nl "Current score is equal to expected score; everything is fine"
162
163 end else
164 if good < expected_good then begin
165
166 perr_nl "Current score is lower than expected :(";
167 perr_nl (spf "(was expecting %d but got %d)\n" expected_good good);
168 perr_nl "If you think it's normal, then maybe you need to update the";
169 perr_nl (spf "score file %s, copying info from %s."
170 expected_score_file actual_score_file)
171
172 end else
173 (* if good > expected_good then *) begin
174
175 perr_nl "Current score is greater than expected :)";
176 perr_nl (spf "(was expecting %d but got %d)" expected_good good);
177 perr_nl "Generating new expected score file and saving old one";
178 Common.command2_y_or_no_exit_if_no
179 (spf "mv %s %s" expected_score_file (expected_score_file ^ ".save"));
180 Common.command2_y_or_no_exit_if_no
181 (spf "mv %s %s" best_of_both_file expected_score_file)
182
183 end
184
185
186 (* ------------------------------------------------------------------------- *)
187 (* ENTRY POINT *)
188
189 let regression_test ~test_dir =
190
191 (* sort expected result files by name *)
192 let test_files = dbe2nm (test_dir, "*", exp_ext) in
193 let e = Common.glob test_files in
194 let e = List.filter (fun f -> Common.filesize f > 0) e in
195 let expected_files = List.sort compare e in
196 if e = [] then failwith (
197 (spf "No test files with expected extension <.%s> found." exp_ext) ^
198 " Are you sure this is the right directory?"
199 );
200
201 (* populate score table *)
202 let actual_score = Common.empty_score() in
203 List.iter (compare_one actual_score) expected_files;
204
205 print_update_regression test_dir actual_score