Debian Codegraph @

extra/protobuf/protobuf-24.4/conformance/conformance_test.cc

raw
1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 #include "conformance_test.h"
32
33 #include <stdarg.h>
34
35 #include <fstream>
36 #include <string>
37
38 #include "google/protobuf/message.h"
39 #include "google/protobuf/text_format.h"
40 #include "google/protobuf/util/field_comparator.h"
41 #include "google/protobuf/util/json_util.h"
42 #include "google/protobuf/util/message_differencer.h"
43 #include "absl/log/absl_log.h"
44 #include "absl/strings/str_cat.h"
45 #include "absl/strings/str_format.h"
46 #include "absl/strings/string_view.h"
47 #include "conformance/conformance.pb.h"
48 #include "conformance/conformance.pb.h"
49 #include "google/protobuf/descriptor_legacy.h"
50
51 using conformance::ConformanceRequest;
52 using conformance::ConformanceResponse;
53 using conformance::WireFormat;
54 using google::protobuf::TextFormat;
55 using google::protobuf::util::DefaultFieldComparator;
56 using google::protobuf::util::MessageDifferencer;
57 using std::string;
58
59 namespace {
60
61 static string ToOctString(const string& binary_string) {
62 string oct_string;
63 for (size_t i = 0; i < binary_string.size(); i++) {
64 uint8_t c = binary_string.at(i);
65 uint8_t high = c / 64;
66 uint8_t mid = (c % 64) / 8;
67 uint8_t low = c % 8;
68 oct_string.push_back('\\');
69 oct_string.push_back('0' + high);
70 oct_string.push_back('0' + mid);
71 oct_string.push_back('0' + low);
72 }
73 return oct_string;
74 }
75
76 template <typename SetT>
77 bool CheckSetEmpty(const SetT& set_to_check, absl::string_view write_to_file,
78 absl::string_view msg, absl::string_view output_dir,
79 std::string* output) {
80 if (set_to_check.empty()) return true;
81
82 absl::StrAppendFormat(output, "\n");
83 absl::StrAppendFormat(output, "%s\n\n", msg);
84 for (absl::string_view v : set_to_check) {
85 absl::StrAppendFormat(output, " %s\n", v);
86 }
87 absl::StrAppendFormat(output, "\n");
88
89 if (!write_to_file.empty()) {
90 std::string full_filename;
91 absl::string_view filename = write_to_file;
92 if (!output_dir.empty()) {
93 full_filename = std::string(output_dir);
94 if (*output_dir.rbegin() != '/') {
95 full_filename.push_back('/');
96 }
97 absl::StrAppend(&full_filename, write_to_file);
98 filename = full_filename;
99 }
100 std::ofstream os{std::string(filename)};
101 if (os) {
102 for (absl::string_view v : set_to_check) {
103 os << v << "\n";
104 }
105 } else {
106 absl::StrAppendFormat(output, "Failed to open file: %s\n", filename);
107 }
108 }
109
110 return false;
111 }
112
113 } // namespace
114
115 namespace google {
116 namespace protobuf {
117
118 ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
119 ConformanceLevel level, conformance::WireFormat input_format,
120 conformance::WireFormat output_format,
121 conformance::TestCategory test_category, const Message& prototype_message,
122 const string& test_name, const string& input)
123 : level_(level),
124 input_format_(input_format),
125 output_format_(output_format),
126 prototype_message_(prototype_message),
127 prototype_message_for_compare_(prototype_message.New()),
128 test_name_(test_name) {
129 switch (input_format) {
130 case conformance::PROTOBUF: {
131 request_.set_protobuf_payload(input);
132 break;
133 }
134
135 case conformance::JSON: {
136 request_.set_json_payload(input);
137 break;
138 }
139
140 case conformance::JSPB: {
141 request_.set_jspb_payload(input);
142 break;
143 }
144
145 case conformance::TEXT_FORMAT: {
146 request_.set_text_payload(input);
147 break;
148 }
149
150 default:
151 ABSL_LOG(FATAL) << "Unspecified input format";
152 }
153
154 request_.set_test_category(test_category);
155
156 request_.set_message_type(prototype_message.GetDescriptor()->full_name());
157 request_.set_requested_output_format(output_format);
158 }
159
160 std::unique_ptr<Message>
161 ConformanceTestSuite::ConformanceRequestSetting::NewTestMessage() const {
162 return std::unique_ptr<Message>(prototype_message_for_compare_->New());
163 }
164
165 string ConformanceTestSuite::ConformanceRequestSetting::GetTestName() const {
166 string rname;
167 switch (FileDescriptorLegacy(prototype_message_.GetDescriptor()->file())
168 .syntax()) {
169 case FileDescriptorLegacy::Syntax::SYNTAX_PROTO3:
170 rname = ".Proto3.";
171 break;
172 case FileDescriptorLegacy::Syntax::SYNTAX_PROTO2:
173 rname = ".Proto2.";
174 break;
175 default:
176 break;
177 }
178
179 return absl::StrCat(ConformanceLevelToString(level_), rname,
180 InputFormatString(input_format_), ".", test_name_, ".",
181 OutputFormatString(output_format_));
182 }
183
184 string
185 ConformanceTestSuite::ConformanceRequestSetting::ConformanceLevelToString(
186 ConformanceLevel level) const {
187 switch (level) {
188 case REQUIRED:
189 return "Required";
190 case RECOMMENDED:
191 return "Recommended";
192 }
193 ABSL_LOG(FATAL) << "Unknown value: " << level;
194 return "";
195 }
196
197 string ConformanceTestSuite::ConformanceRequestSetting::InputFormatString(
198 conformance::WireFormat format) const {
199 switch (format) {
200 case conformance::PROTOBUF:
201 return "ProtobufInput";
202 case conformance::JSON:
203 return "JsonInput";
204 case conformance::TEXT_FORMAT:
205 return "TextFormatInput";
206 default:
207 ABSL_LOG(FATAL) << "Unspecified output format";
208 }
209 return "";
210 }
211
212 string ConformanceTestSuite::ConformanceRequestSetting::OutputFormatString(
213 conformance::WireFormat format) const {
214 switch (format) {
215 case conformance::PROTOBUF:
216 return "ProtobufOutput";
217 case conformance::JSON:
218 return "JsonOutput";
219 case conformance::TEXT_FORMAT:
220 return "TextFormatOutput";
221 default:
222 ABSL_LOG(FATAL) << "Unspecified output format";
223 }
224 return "";
225 }
226
227 void ConformanceTestSuite::TruncateDebugPayload(string* payload) {
228 if (payload != nullptr && payload->size() > 200) {
229 payload->resize(200);
230 payload->append("...(truncated)");
231 }
232 }
233
234 const ConformanceRequest ConformanceTestSuite::TruncateRequest(
235 const ConformanceRequest& request) {
236 ConformanceRequest debug_request(request);
237 switch (debug_request.payload_case()) {
238 case ConformanceRequest::kProtobufPayload:
239 TruncateDebugPayload(debug_request.mutable_protobuf_payload());
240 break;
241 case ConformanceRequest::kJsonPayload:
242 TruncateDebugPayload(debug_request.mutable_json_payload());
243 break;
244 case ConformanceRequest::kTextPayload:
245 TruncateDebugPayload(debug_request.mutable_text_payload());
246 break;
247 case ConformanceRequest::kJspbPayload:
248 TruncateDebugPayload(debug_request.mutable_jspb_payload());
249 break;
250 default:
251 // Do nothing.
252 break;
253 }
254 return debug_request;
255 }
256
257 const ConformanceResponse ConformanceTestSuite::TruncateResponse(
258 const ConformanceResponse& response) {
259 ConformanceResponse debug_response(response);
260 switch (debug_response.result_case()) {
261 case ConformanceResponse::kProtobufPayload:
262 TruncateDebugPayload(debug_response.mutable_protobuf_payload());
263 break;
264 case ConformanceResponse::kJsonPayload:
265 TruncateDebugPayload(debug_response.mutable_json_payload());
266 break;
267 case ConformanceResponse::kTextPayload:
268 TruncateDebugPayload(debug_response.mutable_text_payload());
269 break;
270 case ConformanceResponse::kJspbPayload:
271 TruncateDebugPayload(debug_response.mutable_jspb_payload());
272 break;
273 default:
274 // Do nothing.
275 break;
276 }
277 return debug_response;
278 }
279
280 void ConformanceTestSuite::ReportSuccess(const string& test_name) {
281 if (expected_to_fail_.erase(test_name) != 0) {
282 absl::StrAppendFormat(
283 &output_,
284 "ERROR: test %s is in the failure list, but test succeeded. "
285 "Remove it from the failure list.\n",
286 test_name);
287 unexpected_succeeding_tests_.insert(test_name);
288 }
289 successes_++;
290 }
291
292 void ConformanceTestSuite::ReportFailure(const string& test_name,
293 ConformanceLevel level,
294 const ConformanceRequest& request,
295 const ConformanceResponse& response,
296 absl::string_view message) {
297 if (expected_to_fail_.erase(test_name) == 1) {
298 expected_failures_++;
299 if (!verbose_) return;
300 } else if (level == RECOMMENDED && !enforce_recommended_) {
301 absl::StrAppendFormat(&output_, "WARNING, test=%s: ", test_name);
302 } else {
303 absl::StrAppendFormat(&output_, "ERROR, test=%s: ", test_name);
304 unexpected_failing_tests_.insert(test_name);
305 }
306
307 absl::StrAppendFormat(&output_, "%s, request=%s, response=%s\n", message,
308 TruncateRequest(request).ShortDebugString(),
309 TruncateResponse(response).ShortDebugString());
310 }
311
312 void ConformanceTestSuite::ReportSkip(const string& test_name,
313 const ConformanceRequest& request,
314 const ConformanceResponse& response) {
315 if (verbose_) {
316 absl::StrAppendFormat(
317 &output_, "SKIPPED, test=%s request=%s, response=%s\n", test_name,
318 request.ShortDebugString(), response.ShortDebugString());
319 }
320 skipped_.insert(test_name);
321 }
322
323 void ConformanceTestSuite::RunValidInputTest(
324 const ConformanceRequestSetting& setting,
325 const string& equivalent_text_format) {
326 std::unique_ptr<Message> reference_message(setting.NewTestMessage());
327 ABSL_CHECK(TextFormat::ParseFromString(equivalent_text_format,
328 reference_message.get()))
329 << "Failed to parse data for test case: " << setting.GetTestName()
330 << ", data: " << equivalent_text_format;
331 const string equivalent_wire_format = reference_message->SerializeAsString();
332 RunValidBinaryInputTest(setting, equivalent_wire_format);
333 }
334
335 void ConformanceTestSuite::RunValidBinaryInputTest(
336 const ConformanceRequestSetting& setting,
337 const string& equivalent_wire_format, bool require_same_wire_format) {
338 const ConformanceRequest& request = setting.GetRequest();
339 ConformanceResponse response;
340 RunTest(setting.GetTestName(), request, &response);
341 VerifyResponse(setting, equivalent_wire_format, response, true,
342 require_same_wire_format);
343 }
344
345 void ConformanceTestSuite::VerifyResponse(
346 const ConformanceRequestSetting& setting,
347 const string& equivalent_wire_format, const ConformanceResponse& response,
348 bool need_report_success, bool require_same_wire_format) {
349 std::unique_ptr<Message> test_message(setting.NewTestMessage());
350 const ConformanceRequest& request = setting.GetRequest();
351 const string& test_name = setting.GetTestName();
352 ConformanceLevel level = setting.GetLevel();
353 std::unique_ptr<Message> reference_message = setting.NewTestMessage();
354
355 ABSL_CHECK(reference_message->ParseFromString(equivalent_wire_format))
356 << "Failed to parse wire data for test case: " << test_name;
357
358 switch (response.result_case()) {
359 case ConformanceResponse::RESULT_NOT_SET:
360 ReportFailure(test_name, level, request, response,
361 "Response didn't have any field in the Response.");
362 return;
363
364 case ConformanceResponse::kParseError:
365 case ConformanceResponse::kTimeoutError:
366 case ConformanceResponse::kRuntimeError:
367 case ConformanceResponse::kSerializeError:
368 ReportFailure(test_name, level, request, response,
369 "Failed to parse input or produce output.");
370 return;
371
372 case ConformanceResponse::kSkipped:
373 ReportSkip(test_name, request, response);
374 return;
375
376 default:
377 if (!ParseResponse(response, setting, test_message.get())) return;
378 }
379
380 MessageDifferencer differencer;
381 DefaultFieldComparator field_comparator;
382 field_comparator.set_treat_nan_as_equal(true);
383 differencer.set_field_comparator(&field_comparator);
384 string differences;
385 differencer.ReportDifferencesToString(&differences);
386
387 bool check = false;
388
389 if (require_same_wire_format) {
390 ABSL_DCHECK_EQ(response.result_case(),
391 ConformanceResponse::kProtobufPayload);
392 const string& protobuf_payload = response.protobuf_payload();
393 check = equivalent_wire_format == protobuf_payload;
394 differences = absl::StrCat("Expect: ", ToOctString(equivalent_wire_format),
395 ", but got: ", ToOctString(protobuf_payload));
396 } else {
397 check = differencer.Compare(*reference_message, *test_message);
398 }
399
400 if (check) {
401 if (need_report_success) {
402 ReportSuccess(test_name);
403 }
404 } else {
405 ReportFailure(
406 test_name, level, request, response,
407 absl::StrCat("Output was not equivalent to reference message: ",
408 differences));
409 }
410 }
411
412 void ConformanceTestSuite::RunTest(const string& test_name,
413 const ConformanceRequest& request,
414 ConformanceResponse* response) {
415 if (test_names_.insert(test_name).second == false) {
416 ABSL_LOG(FATAL) << "Duplicated test name: " << test_name;
417 }
418
419 string serialized_request;
420 string serialized_response;
421 request.SerializeToString(&serialized_request);
422
423 runner_->RunTest(test_name, serialized_request, &serialized_response);
424
425 if (!response->ParseFromString(serialized_response)) {
426 response->Clear();
427 response->set_runtime_error("response proto could not be parsed.");
428 }
429
430 if (verbose_) {
431 absl::StrAppendFormat(
432 &output_, "conformance test: name=%s, request=%s, response=%s\n",
433 test_name, TruncateRequest(request).ShortDebugString(),
434 TruncateResponse(*response).ShortDebugString());
435 }
436 }
437
438 string ConformanceTestSuite::WireFormatToString(WireFormat wire_format) {
439 switch (wire_format) {
440 case conformance::PROTOBUF:
441 return "PROTOBUF";
442 case conformance::JSON:
443 return "JSON";
444 case conformance::JSPB:
445 return "JSPB";
446 case conformance::TEXT_FORMAT:
447 return "TEXT_FORMAT";
448 case conformance::UNSPECIFIED:
449 return "UNSPECIFIED";
450 default:
451 ABSL_LOG(FATAL) << "unknown wire type: " << wire_format;
452 }
453 return "";
454 }
455
456 void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) {
457 expected_to_fail_.insert(test_name);
458 }
459
460 bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
461 std::string* output, const string& filename,
462 conformance::FailureSet* failure_list) {
463 runner_ = runner;
464 successes_ = 0;
465 expected_failures_ = 0;
466 skipped_.clear();
467 test_names_.clear();
468 unexpected_failing_tests_.clear();
469 unexpected_succeeding_tests_.clear();
470
471 output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n";
472
473 failure_list_filename_ = filename;
474 expected_to_fail_.clear();
475 for (const string& failure : failure_list->failure()) {
476 AddExpectedFailedTest(failure);
477 }
478 RunSuiteImpl();
479
480 bool ok = true;
481 if (!CheckSetEmpty(
482 expected_to_fail_, "nonexistent_tests.txt",
483 absl::StrCat("These tests were listed in the failure list, but they "
484 "don't exist. Remove them from the failure list by "
485 "running:\n"
486 " ./update_failure_list.py ",
487 failure_list_filename_,
488 " --remove nonexistent_tests.txt"),
489 output_dir_, &output_)) {
490 ok = false;
491 }
492 if (!CheckSetEmpty(
493 unexpected_failing_tests_, "failing_tests.txt",
494 absl::StrCat("These tests failed. If they can't be fixed right now, "
495 "you can add them to the failure list so the overall "
496 "suite can succeed. Add them to the failure list by "
497 "running:\n"
498 " ./update_failure_list.py ",
499 failure_list_filename_, " --add failing_tests.txt"),
500 output_dir_, &output_)) {
501 ok = false;
502 }
503 if (!CheckSetEmpty(
504 unexpected_succeeding_tests_, "succeeding_tests.txt",
505 absl::StrCat("These tests succeeded, even though they were listed in "
506 "the failure list. Remove them from the failure list "
507 "by running:\n"
508 " ./update_failure_list.py ",
509 failure_list_filename_,
510 " --remove succeeding_tests.txt"),
511 output_dir_, &output_)) {
512 ok = false;
513 }
514
515 if (verbose_) {
516 CheckSetEmpty(skipped_, "",
517 "These tests were skipped (probably because support for some "
518 "features is not implemented)",
519 output_dir_, &output_);
520 }
521
522 absl::StrAppendFormat(&output_,
523 "CONFORMANCE SUITE %s: %d successes, %zu skipped, "
524 "%d expected failures, %zu unexpected failures.\n",
525 ok ? "PASSED" : "FAILED", successes_, skipped_.size(),
526 expected_failures_, unexpected_failing_tests_.size());
527 absl::StrAppendFormat(&output_, "\n");
528
529 output->assign(output_);
530
531 return ok;
532 }
533
534 } // namespace protobuf
535 } // namespace google