| 1 |
// Protocol Buffers - Google's data interchange format |
| 2 |
// Copyright 2008 Google Inc. All rights reserved. |
| 3 |
// https://developers.google.com/protocol-buffers/ |
| 4 |
// |
| 5 |
// Redistribution and use in source and binary forms, with or without |
| 6 |
// modification, are permitted provided that the following conditions are |
| 7 |
// met: |
| 8 |
// |
| 9 |
// * Redistributions of source code must retain the above copyright |
| 10 |
// notice, this list of conditions and the following disclaimer. |
| 11 |
// * Redistributions in binary form must reproduce the above |
| 12 |
// copyright notice, this list of conditions and the following disclaimer |
| 13 |
// in the documentation and/or other materials provided with the |
| 14 |
// distribution. |
| 15 |
// * Neither the name of Google Inc. nor the names of its |
| 16 |
// contributors may be used to endorse or promote products derived from |
| 17 |
// this software without specific prior written permission. |
| 18 |
// |
| 19 |
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 |
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 |
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 |
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 |
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 |
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 |
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 |
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 |
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 |
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 |
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 |
|
| 31 |
|
| 32 |
// This file defines a protocol for running the conformance test suite |
| 33 |
// in-process. In other words, the suite itself will run in the same process as |
| 34 |
// the code under test. |
| 35 |
// |
| 36 |
// For pros and cons of this approach, please see conformance.proto. |
| 37 |
|
| 38 |
#ifndef CONFORMANCE_CONFORMANCE_TEST_H |
| 39 |
#define CONFORMANCE_CONFORMANCE_TEST_H |
| 40 |
|
| 41 |
#include <functional> |
| 42 |
#include <string> |
| 43 |
#include <vector> |
| 44 |
|
| 45 |
#include "google/protobuf/descriptor.h" |
| 46 |
#include "google/protobuf/util/type_resolver.h" |
| 47 |
#include "absl/container/btree_set.h" |
| 48 |
#include "absl/container/flat_hash_set.h" |
| 49 |
#include "conformance/conformance.pb.h" |
| 50 |
#include "google/protobuf/wire_format_lite.h" |
| 51 |
|
| 52 |
namespace conformance { |
| 53 |
class ConformanceRequest; |
| 54 |
class ConformanceResponse; |
| 55 |
} // namespace conformance |
| 56 |
|
| 57 |
namespace protobuf_test_messages { |
| 58 |
namespace proto3 { |
| 59 |
class TestAllTypesProto3; |
| 60 |
} // namespace proto3 |
| 61 |
} // namespace protobuf_test_messages |
| 62 |
|
| 63 |
namespace google { |
| 64 |
namespace protobuf { |
| 65 |
|
| 66 |
class ConformanceTestSuite; |
| 67 |
|
| 68 |
class ConformanceTestRunner { |
| 69 |
public: |
| 70 |
virtual ~ConformanceTestRunner() {} |
| 71 |
|
| 72 |
// Call to run a single conformance test. |
| 73 |
// |
| 74 |
// "input" is a serialized conformance.ConformanceRequest. |
| 75 |
// "output" should be set to a serialized conformance.ConformanceResponse. |
| 76 |
// |
| 77 |
// If there is any error in running the test itself, set "runtime_error" in |
| 78 |
// the response. |
| 79 |
virtual void RunTest(const std::string& test_name, |
| 80 |
const std::string& input, |
| 81 |
std::string* output) = 0; |
| 82 |
}; |
| 83 |
|
| 84 |
// Test runner that spawns the process being tested and communicates with it |
| 85 |
// over a pipe. |
| 86 |
class ForkPipeRunner : public ConformanceTestRunner { |
| 87 |
public: |
| 88 |
// Note: Run() doesn't take ownership of the pointers inside suites. |
| 89 |
static int Run(int argc, char *argv[], |
| 90 |
const std::vector<ConformanceTestSuite*>& suites); |
| 91 |
|
| 92 |
ForkPipeRunner(const std::string& executable, |
| 93 |
const std::vector<std::string>& executable_args, |
| 94 |
bool performance) |
| 95 |
: child_pid_(-1), |
| 96 |
executable_(executable), |
| 97 |
executable_args_(executable_args), |
| 98 |
performance_(performance) {} |
| 99 |
|
| 100 |
explicit ForkPipeRunner(const std::string& executable) |
| 101 |
: child_pid_(-1), executable_(executable) {} |
| 102 |
|
| 103 |
virtual ~ForkPipeRunner() {} |
| 104 |
|
| 105 |
void RunTest(const std::string& test_name, |
| 106 |
const std::string& request, |
| 107 |
std::string* response); |
| 108 |
|
| 109 |
private: |
| 110 |
void SpawnTestProgram(); |
| 111 |
|
| 112 |
void CheckedWrite(int fd, const void *buf, size_t len); |
| 113 |
bool TryRead(int fd, void *buf, size_t len); |
| 114 |
void CheckedRead(int fd, void *buf, size_t len); |
| 115 |
|
| 116 |
int write_fd_; |
| 117 |
int read_fd_; |
| 118 |
pid_t child_pid_; |
| 119 |
std::string executable_; |
| 120 |
const std::vector<std::string> executable_args_; |
| 121 |
bool performance_; |
| 122 |
std::string current_test_name_; |
| 123 |
}; |
| 124 |
|
| 125 |
// Class representing the test suite itself. To run it, implement your own |
| 126 |
// class derived from ConformanceTestRunner, class derived from |
| 127 |
// ConformanceTestSuite and then write code like: |
| 128 |
// |
| 129 |
// class MyConformanceTestSuite : public ConformanceTestSuite { |
| 130 |
// public: |
| 131 |
// void RunSuiteImpl() { |
| 132 |
// // INSERT ACTUAL TESTS. |
| 133 |
// } |
| 134 |
// }; |
| 135 |
// |
| 136 |
// class MyConformanceTestRunner : public ConformanceTestRunner { |
| 137 |
// public: |
| 138 |
// static int Run(int argc, char *argv[], |
| 139 |
// ConformanceTestSuite* suite); |
| 140 |
// |
| 141 |
// private: |
| 142 |
// virtual void RunTest(...) { |
| 143 |
// // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE. |
| 144 |
// } |
| 145 |
// }; |
| 146 |
// |
| 147 |
// int main() { |
| 148 |
// MyConformanceTestSuite suite; |
| 149 |
// MyConformanceTestRunner::Run(argc, argv, &suite); |
| 150 |
// } |
| 151 |
// |
| 152 |
class ConformanceTestSuite { |
| 153 |
public: |
| 154 |
ConformanceTestSuite() |
| 155 |
: verbose_(false), |
| 156 |
performance_(false), |
| 157 |
enforce_recommended_(false), |
| 158 |
failure_list_flag_name_("--failure_list") {} |
| 159 |
virtual ~ConformanceTestSuite() {} |
| 160 |
|
| 161 |
void SetPerformance(bool performance) { performance_ = performance; } |
| 162 |
void SetVerbose(bool verbose) { verbose_ = verbose; } |
| 163 |
|
| 164 |
// Whether to require the testee to pass RECOMMENDED tests. By default failing |
| 165 |
// a RECOMMENDED test case will not fail the entire suite but will only |
| 166 |
// generated a warning. If this flag is set to true, RECOMMENDED tests will |
| 167 |
// be treated the same way as REQUIRED tests and failing a RECOMMENDED test |
| 168 |
// case will cause the entire test suite to fail as well. An implementation |
| 169 |
// can enable this if it wants to be strictly conforming to protobuf spec. |
| 170 |
// See the comments about ConformanceLevel below to learn more about the |
| 171 |
// difference between REQUIRED and RECOMMENDED test cases. |
| 172 |
void SetEnforceRecommended(bool value) { |
| 173 |
enforce_recommended_ = value; |
| 174 |
} |
| 175 |
|
| 176 |
// Gets the flag name to the failure list file. |
| 177 |
// By default, this would return --failure_list |
| 178 |
std::string GetFailureListFlagName() { return failure_list_flag_name_; } |
| 179 |
|
| 180 |
void SetFailureListFlagName(const std::string& failure_list_flag_name) { |
| 181 |
failure_list_flag_name_ = failure_list_flag_name; |
| 182 |
} |
| 183 |
|
| 184 |
// Sets the path of the output directory. |
| 185 |
void SetOutputDir(const char* output_dir) { |
| 186 |
output_dir_ = output_dir; |
| 187 |
} |
| 188 |
|
| 189 |
// Run all the conformance tests against the given test runner. |
| 190 |
// Test output will be stored in "output". |
| 191 |
// |
| 192 |
// Returns true if the set of failing tests was exactly the same as the |
| 193 |
// failure list. |
| 194 |
// The filename here is *only* used to create/format useful error messages for |
| 195 |
// how to update the failure list. We do NOT read this file at all. |
| 196 |
bool RunSuite(ConformanceTestRunner* runner, std::string* output, |
| 197 |
const std::string& filename, |
| 198 |
conformance::FailureSet* failure_list); |
| 199 |
|
| 200 |
protected: |
| 201 |
// Test cases are classified into a few categories: |
| 202 |
// REQUIRED: the test case must be passed for an implementation to be |
| 203 |
// interoperable with other implementations. For example, a |
| 204 |
// parser implementation must accept both packed and unpacked |
| 205 |
// form of repeated primitive fields. |
| 206 |
// RECOMMENDED: the test case is not required for the implementation to |
| 207 |
// be interoperable with other implementations, but is |
| 208 |
// recommended for best performance and compatibility. For |
| 209 |
// example, a proto3 serializer should serialize repeated |
| 210 |
// primitive fields in packed form, but an implementation |
| 211 |
// failing to do so will still be able to communicate with |
| 212 |
// other implementations. |
| 213 |
enum ConformanceLevel { |
| 214 |
REQUIRED = 0, |
| 215 |
RECOMMENDED = 1, |
| 216 |
}; |
| 217 |
|
| 218 |
class ConformanceRequestSetting { |
| 219 |
public: |
| 220 |
ConformanceRequestSetting(ConformanceLevel level, |
| 221 |
conformance::WireFormat input_format, |
| 222 |
conformance::WireFormat output_format, |
| 223 |
conformance::TestCategory test_category, |
| 224 |
const Message& prototype_message, |
| 225 |
const std::string& test_name, |
| 226 |
const std::string& input); |
| 227 |
virtual ~ConformanceRequestSetting() {} |
| 228 |
|
| 229 |
std::unique_ptr<Message> NewTestMessage() const; |
| 230 |
|
| 231 |
std::string GetTestName() const; |
| 232 |
|
| 233 |
const conformance::ConformanceRequest& GetRequest() const { |
| 234 |
return request_; |
| 235 |
} |
| 236 |
|
| 237 |
const ConformanceLevel GetLevel() const { |
| 238 |
return level_; |
| 239 |
} |
| 240 |
|
| 241 |
std::string ConformanceLevelToString(ConformanceLevel level) const; |
| 242 |
|
| 243 |
void SetPrintUnknownFields(bool print_unknown_fields) { |
| 244 |
request_.set_print_unknown_fields(true); |
| 245 |
} |
| 246 |
|
| 247 |
void SetPrototypeMessageForCompare(const Message& message) { |
| 248 |
prototype_message_for_compare_.reset(message.New()); |
| 249 |
} |
| 250 |
|
| 251 |
protected: |
| 252 |
virtual std::string InputFormatString(conformance::WireFormat format) const; |
| 253 |
virtual std::string OutputFormatString( |
| 254 |
conformance::WireFormat format) const; |
| 255 |
conformance::ConformanceRequest request_; |
| 256 |
|
| 257 |
private: |
| 258 |
ConformanceLevel level_; |
| 259 |
::conformance::WireFormat input_format_; |
| 260 |
::conformance::WireFormat output_format_; |
| 261 |
const Message& prototype_message_; |
| 262 |
std::unique_ptr<Message> prototype_message_for_compare_; |
| 263 |
std::string test_name_; |
| 264 |
}; |
| 265 |
|
| 266 |
std::string WireFormatToString(conformance::WireFormat wire_format); |
| 267 |
|
| 268 |
// Parse payload in the response to the given message. Returns true on |
| 269 |
// success. |
| 270 |
virtual bool ParseResponse( |
| 271 |
const conformance::ConformanceResponse& response, |
| 272 |
const ConformanceRequestSetting& setting, |
| 273 |
Message* test_message) = 0; |
| 274 |
|
| 275 |
void VerifyResponse(const ConformanceRequestSetting& setting, |
| 276 |
const std::string& equivalent_wire_format, |
| 277 |
const conformance::ConformanceResponse& response, |
| 278 |
bool need_report_success, bool require_same_wire_format); |
| 279 |
|
| 280 |
void TruncateDebugPayload(std::string* payload); |
| 281 |
const conformance::ConformanceRequest TruncateRequest( |
| 282 |
const conformance::ConformanceRequest& request); |
| 283 |
const conformance::ConformanceResponse TruncateResponse( |
| 284 |
const conformance::ConformanceResponse& response); |
| 285 |
|
| 286 |
void ReportSuccess(const std::string& test_name); |
| 287 |
void ReportFailure(const std::string& test_name, ConformanceLevel level, |
| 288 |
const conformance::ConformanceRequest& request, |
| 289 |
const conformance::ConformanceResponse& response, |
| 290 |
absl::string_view message); |
| 291 |
void ReportSkip(const std::string& test_name, |
| 292 |
const conformance::ConformanceRequest& request, |
| 293 |
const conformance::ConformanceResponse& response); |
| 294 |
|
| 295 |
void RunValidInputTest(const ConformanceRequestSetting& setting, |
| 296 |
const std::string& equivalent_text_format); |
| 297 |
void RunValidBinaryInputTest(const ConformanceRequestSetting& setting, |
| 298 |
const std::string& equivalent_wire_format, |
| 299 |
bool require_same_wire_format = false); |
| 300 |
|
| 301 |
void RunTest(const std::string& test_name, |
| 302 |
const conformance::ConformanceRequest& request, |
| 303 |
conformance::ConformanceResponse* response); |
| 304 |
|
| 305 |
void AddExpectedFailedTest(const std::string& test_name); |
| 306 |
|
| 307 |
virtual void RunSuiteImpl() = 0; |
| 308 |
|
| 309 |
ConformanceTestRunner* runner_; |
| 310 |
int successes_; |
| 311 |
int expected_failures_; |
| 312 |
bool verbose_; |
| 313 |
bool performance_; |
| 314 |
bool enforce_recommended_; |
| 315 |
std::string output_; |
| 316 |
std::string output_dir_; |
| 317 |
std::string failure_list_flag_name_; |
| 318 |
std::string failure_list_filename_; |
| 319 |
|
| 320 |
// The set of test names that are expected to fail in this run, but haven't |
| 321 |
// failed yet. |
| 322 |
absl::btree_set<std::string> expected_to_fail_; |
| 323 |
|
| 324 |
// The set of test names that have been run. Used to ensure that there are no |
| 325 |
// duplicate names in the suite. |
| 326 |
absl::flat_hash_set<std::string> test_names_; |
| 327 |
|
| 328 |
// The set of tests that failed, but weren't expected to. |
| 329 |
absl::btree_set<std::string> unexpected_failing_tests_; |
| 330 |
|
| 331 |
// The set of tests that succeeded, but weren't expected to. |
| 332 |
absl::btree_set<std::string> unexpected_succeeding_tests_; |
| 333 |
|
| 334 |
// The set of tests that the testee opted out of; |
| 335 |
absl::btree_set<std::string> skipped_; |
| 336 |
}; |
| 337 |
|
| 338 |
} // namespace protobuf |
| 339 |
} // namespace google |
| 340 |
|
| 341 |
#endif // CONFORMANCE_CONFORMANCE_TEST_H |