| 1 |
/* Copyright (c) 2015, Google Inc. |
| 2 |
* |
| 3 |
* Permission to use, copy, modify, and/or distribute this software for any |
| 4 |
* purpose with or without fee is hereby granted, provided that the above |
| 5 |
* copyright notice and this permission notice appear in all copies. |
| 6 |
* |
| 7 |
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
| 8 |
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
| 9 |
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY |
| 10 |
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
| 11 |
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION |
| 12 |
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN |
| 13 |
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ |
| 14 |
|
| 15 |
package main |
| 16 |
|
| 17 |
import ( |
| 18 |
"bytes" |
| 19 |
"encoding/json" |
| 20 |
"flag" |
| 21 |
"fmt" |
| 22 |
"os" |
| 23 |
"os/exec" |
| 24 |
"path" |
| 25 |
"strconv" |
| 26 |
"strings" |
| 27 |
"sync" |
| 28 |
"syscall" |
| 29 |
"time" |
| 30 |
) |
| 31 |
|
| 32 |
// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner. |
| 33 |
|
| 34 |
var ( |
| 35 |
useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") |
| 36 |
useCallgrind = flag.Bool("callgrind", false, "If true, run code under valgrind to generate callgrind traces.") |
| 37 |
useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") |
| 38 |
buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.") |
| 39 |
numWorkers = flag.Int("num-workers", 1, "Runs the given number of workers when testing.") |
| 40 |
jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") |
| 41 |
mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") |
| 42 |
mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") |
| 43 |
) |
| 44 |
|
| 45 |
type test []string |
| 46 |
|
| 47 |
type result struct { |
| 48 |
Test test |
| 49 |
Passed bool |
| 50 |
Error error |
| 51 |
} |
| 52 |
|
| 53 |
// testOutput is a representation of Chromium's JSON test result format. See |
| 54 |
// https://www.chromium.org/developers/the-json-test-results-format |
| 55 |
type testOutput struct { |
| 56 |
Version int `json:"version"` |
| 57 |
Interrupted bool `json:"interrupted"` |
| 58 |
PathDelimiter string `json:"path_delimiter"` |
| 59 |
SecondsSinceEpoch float64 `json:"seconds_since_epoch"` |
| 60 |
NumFailuresByType map[string]int `json:"num_failures_by_type"` |
| 61 |
Tests map[string]testResult `json:"tests"` |
| 62 |
} |
| 63 |
|
| 64 |
type testResult struct { |
| 65 |
Actual string `json:"actual"` |
| 66 |
Expected string `json:"expected"` |
| 67 |
IsUnexpected bool `json:"is_unexpected"` |
| 68 |
} |
| 69 |
|
| 70 |
func newTestOutput() *testOutput { |
| 71 |
return &testOutput{ |
| 72 |
Version: 3, |
| 73 |
PathDelimiter: ".", |
| 74 |
SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), |
| 75 |
NumFailuresByType: make(map[string]int), |
| 76 |
Tests: make(map[string]testResult), |
| 77 |
} |
| 78 |
} |
| 79 |
|
| 80 |
func (t *testOutput) addResult(name, result string) { |
| 81 |
if _, found := t.Tests[name]; found { |
| 82 |
panic(name) |
| 83 |
} |
| 84 |
t.Tests[name] = testResult{ |
| 85 |
Actual: result, |
| 86 |
Expected: "PASS", |
| 87 |
IsUnexpected: result != "PASS", |
| 88 |
} |
| 89 |
t.NumFailuresByType[result]++ |
| 90 |
} |
| 91 |
|
| 92 |
func (t *testOutput) writeTo(name string) error { |
| 93 |
file, err := os.Create(name) |
| 94 |
if err != nil { |
| 95 |
return err |
| 96 |
} |
| 97 |
defer file.Close() |
| 98 |
out, err := json.MarshalIndent(t, "", " ") |
| 99 |
if err != nil { |
| 100 |
return err |
| 101 |
} |
| 102 |
_, err = file.Write(out) |
| 103 |
return err |
| 104 |
} |
| 105 |
|
| 106 |
func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd { |
| 107 |
valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"} |
| 108 |
if dbAttach { |
| 109 |
valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p") |
| 110 |
} |
| 111 |
valgrindArgs = append(valgrindArgs, path) |
| 112 |
valgrindArgs = append(valgrindArgs, args...) |
| 113 |
|
| 114 |
return exec.Command("valgrind", valgrindArgs...) |
| 115 |
} |
| 116 |
|
| 117 |
func callgrindOf(path string, args ...string) *exec.Cmd { |
| 118 |
valgrindArgs := []string{"-q", "--tool=callgrind", "--dump-instr=yes", "--collect-jumps=yes", "--callgrind-out-file=" + *buildDir + "/callgrind/callgrind.out.%p"} |
| 119 |
valgrindArgs = append(valgrindArgs, path) |
| 120 |
valgrindArgs = append(valgrindArgs, args...) |
| 121 |
|
| 122 |
return exec.Command("valgrind", valgrindArgs...) |
| 123 |
} |
| 124 |
|
| 125 |
func gdbOf(path string, args ...string) *exec.Cmd { |
| 126 |
xtermArgs := []string{"-e", "gdb", "--args"} |
| 127 |
xtermArgs = append(xtermArgs, path) |
| 128 |
xtermArgs = append(xtermArgs, args...) |
| 129 |
|
| 130 |
return exec.Command("xterm", xtermArgs...) |
| 131 |
} |
| 132 |
|
| 133 |
type moreMallocsError struct{} |
| 134 |
|
| 135 |
func (moreMallocsError) Error() string { |
| 136 |
return "child process did not exhaust all allocation calls" |
| 137 |
} |
| 138 |
|
| 139 |
var errMoreMallocs = moreMallocsError{} |
| 140 |
|
| 141 |
func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) { |
| 142 |
prog := path.Join(*buildDir, test[0]) |
| 143 |
args := test[1:] |
| 144 |
var cmd *exec.Cmd |
| 145 |
if *useValgrind { |
| 146 |
cmd = valgrindOf(false, prog, args...) |
| 147 |
} else if *useCallgrind { |
| 148 |
cmd = callgrindOf(prog, args...) |
| 149 |
} else if *useGDB { |
| 150 |
cmd = gdbOf(prog, args...) |
| 151 |
} else { |
| 152 |
cmd = exec.Command(prog, args...) |
| 153 |
} |
| 154 |
var stdoutBuf bytes.Buffer |
| 155 |
var stderrBuf bytes.Buffer |
| 156 |
cmd.Stdout = &stdoutBuf |
| 157 |
cmd.Stderr = &stderrBuf |
| 158 |
if mallocNumToFail >= 0 { |
| 159 |
cmd.Env = os.Environ() |
| 160 |
cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10)) |
| 161 |
if *mallocTestDebug { |
| 162 |
cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1") |
| 163 |
} |
| 164 |
cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1") |
| 165 |
} |
| 166 |
|
| 167 |
if err := cmd.Start(); err != nil { |
| 168 |
return false, err |
| 169 |
} |
| 170 |
if err := cmd.Wait(); err != nil { |
| 171 |
if exitError, ok := err.(*exec.ExitError); ok { |
| 172 |
if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 { |
| 173 |
return false, errMoreMallocs |
| 174 |
} |
| 175 |
} |
| 176 |
fmt.Print(string(stderrBuf.Bytes())) |
| 177 |
return false, err |
| 178 |
} |
| 179 |
fmt.Print(string(stderrBuf.Bytes())) |
| 180 |
|
| 181 |
// Account for Windows line-endings. |
| 182 |
stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1) |
| 183 |
|
| 184 |
if bytes.HasSuffix(stdout, []byte("PASS\n")) && |
| 185 |
(len(stdout) == 5 || stdout[len(stdout)-6] == '\n') { |
| 186 |
return true, nil |
| 187 |
} |
| 188 |
return false, nil |
| 189 |
} |
| 190 |
|
| 191 |
func runTest(test test) (bool, error) { |
| 192 |
if *mallocTest < 0 { |
| 193 |
return runTestOnce(test, -1) |
| 194 |
} |
| 195 |
|
| 196 |
for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ { |
| 197 |
if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs { |
| 198 |
if err != nil { |
| 199 |
err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err) |
| 200 |
} |
| 201 |
return passed, err |
| 202 |
} |
| 203 |
} |
| 204 |
} |
| 205 |
|
| 206 |
// shortTestName returns the short name of a test. Except for evp_test, it |
| 207 |
// assumes that any argument which ends in .txt is a path to a data file and not |
| 208 |
// relevant to the test's uniqueness. |
| 209 |
func shortTestName(test test) string { |
| 210 |
var args []string |
| 211 |
for _, arg := range test { |
| 212 |
if test[0] == "crypto/evp/evp_test" || !strings.HasSuffix(arg, ".txt") { |
| 213 |
args = append(args, arg) |
| 214 |
} |
| 215 |
} |
| 216 |
return strings.Join(args, " ") |
| 217 |
} |
| 218 |
|
| 219 |
// setWorkingDirectory walks up directories as needed until the current working |
| 220 |
// directory is the top of a BoringSSL checkout. |
| 221 |
func setWorkingDirectory() { |
| 222 |
for i := 0; i < 64; i++ { |
| 223 |
if _, err := os.Stat("BUILDING.md"); err == nil { |
| 224 |
return |
| 225 |
} |
| 226 |
os.Chdir("..") |
| 227 |
} |
| 228 |
|
| 229 |
panic("Couldn't find BUILDING.md in a parent directory!") |
| 230 |
} |
| 231 |
|
| 232 |
func parseTestConfig(filename string) ([]test, error) { |
| 233 |
in, err := os.Open(filename) |
| 234 |
if err != nil { |
| 235 |
return nil, err |
| 236 |
} |
| 237 |
defer in.Close() |
| 238 |
|
| 239 |
decoder := json.NewDecoder(in) |
| 240 |
var result []test |
| 241 |
if err := decoder.Decode(&result); err != nil { |
| 242 |
return nil, err |
| 243 |
} |
| 244 |
return result, nil |
| 245 |
} |
| 246 |
|
| 247 |
func worker(tests <-chan test, results chan<- result, done *sync.WaitGroup) { |
| 248 |
defer done.Done() |
| 249 |
for test := range tests { |
| 250 |
passed, err := runTest(test) |
| 251 |
results <- result{test, passed, err} |
| 252 |
} |
| 253 |
} |
| 254 |
|
| 255 |
func main() { |
| 256 |
flag.Parse() |
| 257 |
setWorkingDirectory() |
| 258 |
|
| 259 |
testCases, err := parseTestConfig("util/all_tests.json") |
| 260 |
if err != nil { |
| 261 |
fmt.Printf("Failed to parse input: %s\n", err) |
| 262 |
os.Exit(1) |
| 263 |
} |
| 264 |
|
| 265 |
var wg sync.WaitGroup |
| 266 |
tests := make(chan test, *numWorkers) |
| 267 |
results := make(chan result, *numWorkers) |
| 268 |
|
| 269 |
for i := 0; i < *numWorkers; i++ { |
| 270 |
wg.Add(1) |
| 271 |
go worker(tests, results, &wg) |
| 272 |
} |
| 273 |
|
| 274 |
go func() { |
| 275 |
for _, test := range testCases { |
| 276 |
tests <- test |
| 277 |
} |
| 278 |
close(tests) |
| 279 |
|
| 280 |
wg.Wait() |
| 281 |
close(results) |
| 282 |
}() |
| 283 |
|
| 284 |
testOutput := newTestOutput() |
| 285 |
var failed []test |
| 286 |
for testResult := range results { |
| 287 |
test := testResult.Test |
| 288 |
|
| 289 |
fmt.Printf("%s\n", strings.Join([]string(test), " ")) |
| 290 |
name := shortTestName(test) |
| 291 |
if testResult.Error != nil { |
| 292 |
fmt.Printf("%s failed to complete: %s\n", test[0], testResult.Error) |
| 293 |
failed = append(failed, test) |
| 294 |
testOutput.addResult(name, "CRASHED") |
| 295 |
} else if !testResult.Passed { |
| 296 |
fmt.Printf("%s failed to print PASS on the last line.\n", test[0]) |
| 297 |
failed = append(failed, test) |
| 298 |
testOutput.addResult(name, "FAIL") |
| 299 |
} else { |
| 300 |
testOutput.addResult(name, "PASS") |
| 301 |
} |
| 302 |
} |
| 303 |
|
| 304 |
if *jsonOutput != "" { |
| 305 |
if err := testOutput.writeTo(*jsonOutput); err != nil { |
| 306 |
fmt.Fprintf(os.Stderr, "Error: %s\n", err) |
| 307 |
} |
| 308 |
} |
| 309 |
|
| 310 |
if len(failed) > 0 { |
| 311 |
fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(testCases)) |
| 312 |
for _, test := range failed { |
| 313 |
fmt.Printf("\t%s\n", strings.Join([]string(test), " ")) |
| 314 |
} |
| 315 |
os.Exit(1) |
| 316 |
} |
| 317 |
|
| 318 |
fmt.Printf("\nAll tests passed!\n") |
| 319 |
} |