Debian Codegraph @

extra/protobuf/protobuf-24.4/conformance/conformance_test_runner.cc

raw
1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // This file contains a program for running the test suite in a separate
32 // process. The other alternative is to run the suite in-process. See
33 // conformance.proto for pros/cons of these two options.
34 //
35 // This program will fork the process under test and communicate with it over
36 // its stdin/stdout:
37 //
38 // +--------+ pipe +----------+
39 // | tester | <------> | testee |
40 // | | | |
41 // | C++ | | any lang |
42 // +--------+ +----------+
43 //
44 // The tester contains all of the test cases and their expected output.
45 // The testee is a simple program written in the target language that reads
46 // each test case and attempts to produce acceptable output for it.
47 //
48 // Every test consists of a ConformanceRequest/ConformanceResponse
49 // request/reply pair. The protocol on the pipe is simply:
50 //
51 // 1. tester sends 4-byte length N (little endian)
52 // 2. tester sends N bytes representing a ConformanceRequest proto
53 // 3. testee sends 4-byte length M (little endian)
54 // 4. testee sends M bytes representing a ConformanceResponse proto
55
56 #include <errno.h>
57 #include <signal.h>
58 #include <sys/types.h>
59 #include <sys/wait.h>
60 #include <unistd.h>
61
62 #include <algorithm>
63 #include <cstdlib>
64 #include <fstream>
65 #include <future>
66 #include <vector>
67
68 #include "absl/log/absl_log.h"
69 #include "absl/strings/str_format.h"
70 #include "conformance/conformance.pb.h"
71 #include "conformance_test.h"
72
73 using conformance::ConformanceResponse;
74 using google::protobuf::ConformanceTestSuite;
75 using std::string;
76 using std::vector;
77
78 #define STRINGIFY(x) #x
79 #define TOSTRING(x) STRINGIFY(x)
80 #define CHECK_SYSCALL(call) \
81 if (call < 0) { \
82 perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
83 exit(1); \
84 }
85
86 namespace google {
87 namespace protobuf {
88
89 void ParseFailureList(const char *filename,
90 conformance::FailureSet *failure_list) {
91 std::ifstream infile(filename);
92
93 if (!infile.is_open()) {
94 fprintf(stderr, "Couldn't open failure list file: %s\n", filename);
95 exit(1);
96 }
97
98 for (string line; getline(infile, line);) {
99 // Remove whitespace.
100 line.erase(std::remove_if(line.begin(), line.end(), ::isspace), line.end());
101
102 // Remove comments.
103 line = line.substr(0, line.find("#"));
104
105 if (!line.empty()) {
106 failure_list->add_failure(line);
107 }
108 }
109 }
110
111 void UsageError() {
112 fprintf(stderr, "Usage: conformance-test-runner [options] <test-program>\n");
113 fprintf(stderr, "\n");
114 fprintf(stderr, "Options:\n");
115 fprintf(stderr,
116 " --failure_list <filename> Use to specify list of tests\n");
117 fprintf(stderr,
118 " that are expected to fail. File\n");
119 fprintf(stderr,
120 " should contain one test name per\n");
121 fprintf(stderr,
122 " line. Use '#' for comments.\n");
123 fprintf(stderr,
124 " --text_format_failure_list <filename> Use to specify list \n");
125 fprintf(stderr,
126 " of tests that are expected to \n");
127 fprintf(stderr, " fail in the \n");
128 fprintf(stderr,
129 " text_format_conformance_suite. \n");
130 fprintf(stderr,
131 " File should contain one test name \n");
132 fprintf(stderr,
133 " per line. Use '#' for comments.\n");
134
135 fprintf(stderr,
136 " --enforce_recommended Enforce that recommended test\n");
137 fprintf(stderr,
138 " cases are also passing. Specify\n");
139 fprintf(stderr,
140 " this flag if you want to be\n");
141 fprintf(stderr,
142 " strictly conforming to protobuf\n");
143 fprintf(stderr, " spec.\n");
144 fprintf(stderr,
145 " --output_dir <dirname> Directory to write\n"
146 " output files.\n");
147 exit(1);
148 }
149
150 void ForkPipeRunner::RunTest(const std::string &test_name,
151 const std::string &request,
152 std::string *response) {
153 if (child_pid_ < 0) {
154 SpawnTestProgram();
155 }
156 current_test_name_ = test_name;
157
158 uint32_t len = request.size();
159 CheckedWrite(write_fd_, &len, sizeof(uint32_t));
160 CheckedWrite(write_fd_, request.c_str(), request.size());
161
162 if (!TryRead(read_fd_, &len, sizeof(uint32_t))) {
163 // We failed to read from the child, assume a crash and try to reap.
164 ABSL_LOG(INFO) << "Trying to reap child, pid=" << child_pid_;
165
166 int status = 0;
167 waitpid(child_pid_, &status, WEXITED);
168
169 string error_msg;
170 conformance::ConformanceResponse response_obj;
171 if (WIFEXITED(status)) {
172 if (WEXITSTATUS(status) == 0) {
173 absl::StrAppendFormat(&error_msg,
174 "child timed out, killed by signal %d",
175 WTERMSIG(status));
176 response_obj.set_timeout_error(error_msg);
177 } else {
178 absl::StrAppendFormat(&error_msg, "child exited, status=%d",
179 WEXITSTATUS(status));
180 response_obj.set_runtime_error(error_msg);
181 }
182 } else if (WIFSIGNALED(status)) {
183 absl::StrAppendFormat(&error_msg, "child killed by signal %d",
184 WTERMSIG(status));
185 }
186 ABSL_LOG(INFO) << error_msg;
187 child_pid_ = -1;
188
189 response_obj.SerializeToString(response);
190 return;
191 }
192
193 response->resize(len);
194 CheckedRead(read_fd_, (void *)response->c_str(), len);
195 }
196
197 int ForkPipeRunner::Run(int argc, char *argv[],
198 const std::vector<ConformanceTestSuite *> &suites) {
199 if (suites.empty()) {
200 fprintf(stderr, "No test suites found.\n");
201 return EXIT_FAILURE;
202 }
203 bool all_ok = true;
204 for (ConformanceTestSuite *suite : suites) {
205 string program;
206 std::vector<string> program_args;
207 string failure_list_filename;
208 conformance::FailureSet failure_list;
209
210 bool performance = false;
211 for (int arg = 1; arg < argc; ++arg) {
212 if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
213 if (++arg == argc) UsageError();
214 failure_list_filename = argv[arg];
215 ParseFailureList(argv[arg], &failure_list);
216 } else if (strcmp(argv[arg], "--performance") == 0) {
217 performance = true;
218 suite->SetPerformance(true);
219 } else if (strcmp(argv[arg], "--verbose") == 0) {
220 suite->SetVerbose(true);
221 } else if (strcmp(argv[arg], "--enforce_recommended") == 0) {
222 suite->SetEnforceRecommended(true);
223 } else if (strcmp(argv[arg], "--output_dir") == 0) {
224 if (++arg == argc) UsageError();
225 suite->SetOutputDir(argv[arg]);
226 } else if (argv[arg][0] == '-') {
227 bool recognized_flag = false;
228 for (ConformanceTestSuite *suite : suites) {
229 if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
230 if (++arg == argc) UsageError();
231 recognized_flag = true;
232 }
233 }
234 if (!recognized_flag) {
235 fprintf(stderr, "Unknown option: %s\n", argv[arg]);
236 UsageError();
237 }
238 } else {
239 program += argv[arg++];
240 while (arg < argc) {
241 program_args.push_back(argv[arg]);
242 arg++;
243 }
244 }
245 }
246
247 ForkPipeRunner runner(program, program_args, performance);
248
249 std::string output;
250 all_ok = all_ok && suite->RunSuite(&runner, &output, failure_list_filename,
251 &failure_list);
252
253 fwrite(output.c_str(), 1, output.size(), stderr);
254 }
255 return all_ok ? EXIT_SUCCESS : EXIT_FAILURE;
256 }
257
258 // TODO(haberman): make this work on Windows, instead of using these
259 // UNIX-specific APIs.
260 //
261 // There is a platform-agnostic API in
262 // src/google/protobuf/compiler/subprocess.h
263 //
264 // However that API only supports sending a single message to the subprocess.
265 // We really want to be able to send messages and receive responses one at a
266 // time:
267 //
268 // 1. Spawning a new process for each test would take way too long for thousands
269 // of tests and subprocesses like java that can take 100ms or more to start
270 // up.
271 //
272 // 2. Sending all the tests in one big message and receiving all results in one
273 // big message would take away our visibility about which test(s) caused a
274 // crash or other fatal error. It would also give us only a single failure
275 // instead of all of them.
276 void ForkPipeRunner::SpawnTestProgram() {
277 int toproc_pipe_fd[2];
278 int fromproc_pipe_fd[2];
279 if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
280 perror("pipe");
281 exit(1);
282 }
283
284 pid_t pid = fork();
285 if (pid < 0) {
286 perror("fork");
287 exit(1);
288 }
289
290 if (pid) {
291 // Parent.
292 CHECK_SYSCALL(close(toproc_pipe_fd[0]));
293 CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
294 write_fd_ = toproc_pipe_fd[1];
295 read_fd_ = fromproc_pipe_fd[0];
296 child_pid_ = pid;
297 } else {
298 // Child.
299 CHECK_SYSCALL(close(STDIN_FILENO));
300 CHECK_SYSCALL(close(STDOUT_FILENO));
301 CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
302 CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));
303
304 CHECK_SYSCALL(close(toproc_pipe_fd[0]));
305 CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
306 CHECK_SYSCALL(close(toproc_pipe_fd[1]));
307 CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
308
309 std::unique_ptr<char[]> executable(new char[executable_.size() + 1]);
310 memcpy(executable.get(), executable_.c_str(), executable_.size());
311 executable[executable_.size()] = '\0';
312
313 std::vector<const char *> argv;
314 argv.push_back(executable.get());
315 ABSL_LOG(INFO) << argv[0];
316 for (size_t i = 0; i < executable_args_.size(); ++i) {
317 argv.push_back(executable_args_[i].c_str());
318 ABSL_LOG(INFO) << executable_args_[i];
319 }
320 argv.push_back(nullptr);
321 // Never returns.
322 CHECK_SYSCALL(execv(executable.get(), const_cast<char **>(argv.data())));
323 }
324 }
325
326 void ForkPipeRunner::CheckedWrite(int fd, const void *buf, size_t len) {
327 if (static_cast<size_t>(write(fd, buf, len)) != len) {
328 ABSL_LOG(FATAL) << current_test_name_
329 << ": error writing to test program: " << strerror(errno);
330 }
331 }
332
333 bool ForkPipeRunner::TryRead(int fd, void *buf, size_t len) {
334 size_t ofs = 0;
335 while (len > 0) {
336 std::future<ssize_t> future = std::async(
337 std::launch::async,
338 [](int fd, void *buf, size_t ofs, size_t len) {
339 return read(fd, (char *)buf + ofs, len);
340 },
341 fd, buf, ofs, len);
342 std::future_status status;
343 if (performance_) {
344 status = future.wait_for(std::chrono::seconds(5));
345 if (status == std::future_status::timeout) {
346 ABSL_LOG(ERROR) << current_test_name_ << ": timeout from test program";
347 kill(child_pid_, SIGQUIT);
348 // TODO(sandyzhang): Only log in flag-guarded mode, since reading output
349 // from SIGQUIT is slow and verbose.
350 std::vector<char> err;
351 err.resize(5000);
352 ssize_t err_bytes_read;
353 size_t err_ofs = 0;
354 do {
355 err_bytes_read =
356 read(fd, (void *)&err[err_ofs], err.size() - err_ofs);
357 err_ofs += err_bytes_read;
358 } while (err_bytes_read > 0 && err_ofs < err.size());
359 ABSL_LOG(ERROR) << "child_pid_=" << child_pid_ << " SIGQUIT: \n"
360 << &err[0];
361 return false;
362 }
363 } else {
364 future.wait();
365 }
366 ssize_t bytes_read = future.get();
367 if (bytes_read == 0) {
368 ABSL_LOG(ERROR) << current_test_name_
369 << ": unexpected EOF from test program";
370 return false;
371 } else if (bytes_read < 0) {
372 ABSL_LOG(ERROR) << current_test_name_
373 << ": error reading from test program: "
374 << strerror(errno);
375 return false;
376 }
377
378 len -= bytes_read;
379 ofs += bytes_read;
380 }
381
382 return true;
383 }
384
385 void ForkPipeRunner::CheckedRead(int fd, void *buf, size_t len) {
386 if (!TryRead(fd, buf, len)) {
387 ABSL_LOG(FATAL) << current_test_name_
388 << ": error reading from test program: " << strerror(errno);
389 }
390 }
391
392 } // namespace protobuf
393 } // namespace google