Debian Codegraph @

external/boringssl/util/all_tests.go

raw
1 /* Copyright (c) 2015, Google Inc.
2 *
3 * Permission to use, copy, modify, and/or distribute this software for any
4 * purpose with or without fee is hereby granted, provided that the above
5 * copyright notice and this permission notice appear in all copies.
6 *
7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
14
15 package main
16
17 import (
18 "bytes"
19 "encoding/json"
20 "flag"
21 "fmt"
22 "os"
23 "os/exec"
24 "path"
25 "strconv"
26 "strings"
27 "sync"
28 "syscall"
29 "time"
30 )
31
32 // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
33
34 var (
35 useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
36 useCallgrind = flag.Bool("callgrind", false, "If true, run code under valgrind to generate callgrind traces.")
37 useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
38 buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.")
39 numWorkers = flag.Int("num-workers", 1, "Runs the given number of workers when testing.")
40 jsonOutput = flag.String("json-output", "", "The file to output JSON results to.")
41 mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
42 mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
43 )
44
45 type test []string
46
47 type result struct {
48 Test test
49 Passed bool
50 Error error
51 }
52
53 // testOutput is a representation of Chromium's JSON test result format. See
54 // https://www.chromium.org/developers/the-json-test-results-format
55 type testOutput struct {
56 Version int `json:"version"`
57 Interrupted bool `json:"interrupted"`
58 PathDelimiter string `json:"path_delimiter"`
59 SecondsSinceEpoch float64 `json:"seconds_since_epoch"`
60 NumFailuresByType map[string]int `json:"num_failures_by_type"`
61 Tests map[string]testResult `json:"tests"`
62 }
63
64 type testResult struct {
65 Actual string `json:"actual"`
66 Expected string `json:"expected"`
67 IsUnexpected bool `json:"is_unexpected"`
68 }
69
70 func newTestOutput() *testOutput {
71 return &testOutput{
72 Version: 3,
73 PathDelimiter: ".",
74 SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
75 NumFailuresByType: make(map[string]int),
76 Tests: make(map[string]testResult),
77 }
78 }
79
80 func (t *testOutput) addResult(name, result string) {
81 if _, found := t.Tests[name]; found {
82 panic(name)
83 }
84 t.Tests[name] = testResult{
85 Actual: result,
86 Expected: "PASS",
87 IsUnexpected: result != "PASS",
88 }
89 t.NumFailuresByType[result]++
90 }
91
92 func (t *testOutput) writeTo(name string) error {
93 file, err := os.Create(name)
94 if err != nil {
95 return err
96 }
97 defer file.Close()
98 out, err := json.MarshalIndent(t, "", " ")
99 if err != nil {
100 return err
101 }
102 _, err = file.Write(out)
103 return err
104 }
105
106 func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
107 valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"}
108 if dbAttach {
109 valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
110 }
111 valgrindArgs = append(valgrindArgs, path)
112 valgrindArgs = append(valgrindArgs, args...)
113
114 return exec.Command("valgrind", valgrindArgs...)
115 }
116
117 func callgrindOf(path string, args ...string) *exec.Cmd {
118 valgrindArgs := []string{"-q", "--tool=callgrind", "--dump-instr=yes", "--collect-jumps=yes", "--callgrind-out-file=" + *buildDir + "/callgrind/callgrind.out.%p"}
119 valgrindArgs = append(valgrindArgs, path)
120 valgrindArgs = append(valgrindArgs, args...)
121
122 return exec.Command("valgrind", valgrindArgs...)
123 }
124
125 func gdbOf(path string, args ...string) *exec.Cmd {
126 xtermArgs := []string{"-e", "gdb", "--args"}
127 xtermArgs = append(xtermArgs, path)
128 xtermArgs = append(xtermArgs, args...)
129
130 return exec.Command("xterm", xtermArgs...)
131 }
132
133 type moreMallocsError struct{}
134
135 func (moreMallocsError) Error() string {
136 return "child process did not exhaust all allocation calls"
137 }
138
139 var errMoreMallocs = moreMallocsError{}
140
141 func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
142 prog := path.Join(*buildDir, test[0])
143 args := test[1:]
144 var cmd *exec.Cmd
145 if *useValgrind {
146 cmd = valgrindOf(false, prog, args...)
147 } else if *useCallgrind {
148 cmd = callgrindOf(prog, args...)
149 } else if *useGDB {
150 cmd = gdbOf(prog, args...)
151 } else {
152 cmd = exec.Command(prog, args...)
153 }
154 var stdoutBuf bytes.Buffer
155 var stderrBuf bytes.Buffer
156 cmd.Stdout = &stdoutBuf
157 cmd.Stderr = &stderrBuf
158 if mallocNumToFail >= 0 {
159 cmd.Env = os.Environ()
160 cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10))
161 if *mallocTestDebug {
162 cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1")
163 }
164 cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1")
165 }
166
167 if err := cmd.Start(); err != nil {
168 return false, err
169 }
170 if err := cmd.Wait(); err != nil {
171 if exitError, ok := err.(*exec.ExitError); ok {
172 if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 {
173 return false, errMoreMallocs
174 }
175 }
176 fmt.Print(string(stderrBuf.Bytes()))
177 return false, err
178 }
179 fmt.Print(string(stderrBuf.Bytes()))
180
181 // Account for Windows line-endings.
182 stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
183
184 if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
185 (len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
186 return true, nil
187 }
188 return false, nil
189 }
190
191 func runTest(test test) (bool, error) {
192 if *mallocTest < 0 {
193 return runTestOnce(test, -1)
194 }
195
196 for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ {
197 if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs {
198 if err != nil {
199 err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err)
200 }
201 return passed, err
202 }
203 }
204 }
205
206 // shortTestName returns the short name of a test. Except for evp_test, it
207 // assumes that any argument which ends in .txt is a path to a data file and not
208 // relevant to the test's uniqueness.
209 func shortTestName(test test) string {
210 var args []string
211 for _, arg := range test {
212 if test[0] == "crypto/evp/evp_test" || !strings.HasSuffix(arg, ".txt") {
213 args = append(args, arg)
214 }
215 }
216 return strings.Join(args, " ")
217 }
218
219 // setWorkingDirectory walks up directories as needed until the current working
220 // directory is the top of a BoringSSL checkout.
221 func setWorkingDirectory() {
222 for i := 0; i < 64; i++ {
223 if _, err := os.Stat("BUILDING.md"); err == nil {
224 return
225 }
226 os.Chdir("..")
227 }
228
229 panic("Couldn't find BUILDING.md in a parent directory!")
230 }
231
232 func parseTestConfig(filename string) ([]test, error) {
233 in, err := os.Open(filename)
234 if err != nil {
235 return nil, err
236 }
237 defer in.Close()
238
239 decoder := json.NewDecoder(in)
240 var result []test
241 if err := decoder.Decode(&result); err != nil {
242 return nil, err
243 }
244 return result, nil
245 }
246
247 func worker(tests <-chan test, results chan<- result, done *sync.WaitGroup) {
248 defer done.Done()
249 for test := range tests {
250 passed, err := runTest(test)
251 results <- result{test, passed, err}
252 }
253 }
254
255 func main() {
256 flag.Parse()
257 setWorkingDirectory()
258
259 testCases, err := parseTestConfig("util/all_tests.json")
260 if err != nil {
261 fmt.Printf("Failed to parse input: %s\n", err)
262 os.Exit(1)
263 }
264
265 var wg sync.WaitGroup
266 tests := make(chan test, *numWorkers)
267 results := make(chan result, *numWorkers)
268
269 for i := 0; i < *numWorkers; i++ {
270 wg.Add(1)
271 go worker(tests, results, &wg)
272 }
273
274 go func() {
275 for _, test := range testCases {
276 tests <- test
277 }
278 close(tests)
279
280 wg.Wait()
281 close(results)
282 }()
283
284 testOutput := newTestOutput()
285 var failed []test
286 for testResult := range results {
287 test := testResult.Test
288
289 fmt.Printf("%s\n", strings.Join([]string(test), " "))
290 name := shortTestName(test)
291 if testResult.Error != nil {
292 fmt.Printf("%s failed to complete: %s\n", test[0], testResult.Error)
293 failed = append(failed, test)
294 testOutput.addResult(name, "CRASHED")
295 } else if !testResult.Passed {
296 fmt.Printf("%s failed to print PASS on the last line.\n", test[0])
297 failed = append(failed, test)
298 testOutput.addResult(name, "FAIL")
299 } else {
300 testOutput.addResult(name, "PASS")
301 }
302 }
303
304 if *jsonOutput != "" {
305 if err := testOutput.writeTo(*jsonOutput); err != nil {
306 fmt.Fprintf(os.Stderr, "Error: %s\n", err)
307 }
308 }
309
310 if len(failed) > 0 {
311 fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(testCases))
312 for _, test := range failed {
313 fmt.Printf("\t%s\n", strings.Join([]string(test), " "))
314 }
315 os.Exit(1)
316 }
317
318 fmt.Printf("\nAll tests passed!\n")
319 }