Debian Codegraph @

extra/protobuf/protobuf-24.4/conformance/conformance_test.h

raw
1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31
32 // This file defines a protocol for running the conformance test suite
33 // in-process. In other words, the suite itself will run in the same process as
34 // the code under test.
35 //
36 // For pros and cons of this approach, please see conformance.proto.
37
38 #ifndef CONFORMANCE_CONFORMANCE_TEST_H
39 #define CONFORMANCE_CONFORMANCE_TEST_H
40
41 #include <functional>
42 #include <string>
43 #include <vector>
44
45 #include "google/protobuf/descriptor.h"
46 #include "google/protobuf/util/type_resolver.h"
47 #include "absl/container/btree_set.h"
48 #include "absl/container/flat_hash_set.h"
49 #include "conformance/conformance.pb.h"
50 #include "google/protobuf/wire_format_lite.h"
51
52 namespace conformance {
53 class ConformanceRequest;
54 class ConformanceResponse;
55 } // namespace conformance
56
57 namespace protobuf_test_messages {
58 namespace proto3 {
59 class TestAllTypesProto3;
60 } // namespace proto3
61 } // namespace protobuf_test_messages
62
63 namespace google {
64 namespace protobuf {
65
66 class ConformanceTestSuite;
67
68 class ConformanceTestRunner {
69 public:
70 virtual ~ConformanceTestRunner() {}
71
72 // Call to run a single conformance test.
73 //
74 // "input" is a serialized conformance.ConformanceRequest.
75 // "output" should be set to a serialized conformance.ConformanceResponse.
76 //
77 // If there is any error in running the test itself, set "runtime_error" in
78 // the response.
79 virtual void RunTest(const std::string& test_name,
80 const std::string& input,
81 std::string* output) = 0;
82 };
83
84 // Test runner that spawns the process being tested and communicates with it
85 // over a pipe.
86 class ForkPipeRunner : public ConformanceTestRunner {
87 public:
88 // Note: Run() doesn't take ownership of the pointers inside suites.
89 static int Run(int argc, char *argv[],
90 const std::vector<ConformanceTestSuite*>& suites);
91
92 ForkPipeRunner(const std::string& executable,
93 const std::vector<std::string>& executable_args,
94 bool performance)
95 : child_pid_(-1),
96 executable_(executable),
97 executable_args_(executable_args),
98 performance_(performance) {}
99
100 explicit ForkPipeRunner(const std::string& executable)
101 : child_pid_(-1), executable_(executable) {}
102
103 virtual ~ForkPipeRunner() {}
104
105 void RunTest(const std::string& test_name,
106 const std::string& request,
107 std::string* response);
108
109 private:
110 void SpawnTestProgram();
111
112 void CheckedWrite(int fd, const void *buf, size_t len);
113 bool TryRead(int fd, void *buf, size_t len);
114 void CheckedRead(int fd, void *buf, size_t len);
115
116 int write_fd_;
117 int read_fd_;
118 pid_t child_pid_;
119 std::string executable_;
120 const std::vector<std::string> executable_args_;
121 bool performance_;
122 std::string current_test_name_;
123 };
124
125 // Class representing the test suite itself. To run it, implement your own
126 // class derived from ConformanceTestRunner, class derived from
127 // ConformanceTestSuite and then write code like:
128 //
129 // class MyConformanceTestSuite : public ConformanceTestSuite {
130 // public:
131 // void RunSuiteImpl() {
132 // // INSERT ACTUAL TESTS.
133 // }
134 // };
135 //
136 // class MyConformanceTestRunner : public ConformanceTestRunner {
137 // public:
138 // static int Run(int argc, char *argv[],
139 // ConformanceTestSuite* suite);
140 //
141 // private:
142 // virtual void RunTest(...) {
143 // // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE.
144 // }
145 // };
146 //
147 // int main() {
148 // MyConformanceTestSuite suite;
149 // MyConformanceTestRunner::Run(argc, argv, &suite);
150 // }
151 //
152 class ConformanceTestSuite {
153 public:
154 ConformanceTestSuite()
155 : verbose_(false),
156 performance_(false),
157 enforce_recommended_(false),
158 failure_list_flag_name_("--failure_list") {}
159 virtual ~ConformanceTestSuite() {}
160
161 void SetPerformance(bool performance) { performance_ = performance; }
162 void SetVerbose(bool verbose) { verbose_ = verbose; }
163
164 // Whether to require the testee to pass RECOMMENDED tests. By default failing
165 // a RECOMMENDED test case will not fail the entire suite but will only
166 // generated a warning. If this flag is set to true, RECOMMENDED tests will
167 // be treated the same way as REQUIRED tests and failing a RECOMMENDED test
168 // case will cause the entire test suite to fail as well. An implementation
169 // can enable this if it wants to be strictly conforming to protobuf spec.
170 // See the comments about ConformanceLevel below to learn more about the
171 // difference between REQUIRED and RECOMMENDED test cases.
172 void SetEnforceRecommended(bool value) {
173 enforce_recommended_ = value;
174 }
175
176 // Gets the flag name to the failure list file.
177 // By default, this would return --failure_list
178 std::string GetFailureListFlagName() { return failure_list_flag_name_; }
179
180 void SetFailureListFlagName(const std::string& failure_list_flag_name) {
181 failure_list_flag_name_ = failure_list_flag_name;
182 }
183
184 // Sets the path of the output directory.
185 void SetOutputDir(const char* output_dir) {
186 output_dir_ = output_dir;
187 }
188
189 // Run all the conformance tests against the given test runner.
190 // Test output will be stored in "output".
191 //
192 // Returns true if the set of failing tests was exactly the same as the
193 // failure list.
194 // The filename here is *only* used to create/format useful error messages for
195 // how to update the failure list. We do NOT read this file at all.
196 bool RunSuite(ConformanceTestRunner* runner, std::string* output,
197 const std::string& filename,
198 conformance::FailureSet* failure_list);
199
200 protected:
201 // Test cases are classified into a few categories:
202 // REQUIRED: the test case must be passed for an implementation to be
203 // interoperable with other implementations. For example, a
204 // parser implementation must accept both packed and unpacked
205 // form of repeated primitive fields.
206 // RECOMMENDED: the test case is not required for the implementation to
207 // be interoperable with other implementations, but is
208 // recommended for best performance and compatibility. For
209 // example, a proto3 serializer should serialize repeated
210 // primitive fields in packed form, but an implementation
211 // failing to do so will still be able to communicate with
212 // other implementations.
213 enum ConformanceLevel {
214 REQUIRED = 0,
215 RECOMMENDED = 1,
216 };
217
218 class ConformanceRequestSetting {
219 public:
220 ConformanceRequestSetting(ConformanceLevel level,
221 conformance::WireFormat input_format,
222 conformance::WireFormat output_format,
223 conformance::TestCategory test_category,
224 const Message& prototype_message,
225 const std::string& test_name,
226 const std::string& input);
227 virtual ~ConformanceRequestSetting() {}
228
229 std::unique_ptr<Message> NewTestMessage() const;
230
231 std::string GetTestName() const;
232
233 const conformance::ConformanceRequest& GetRequest() const {
234 return request_;
235 }
236
237 const ConformanceLevel GetLevel() const {
238 return level_;
239 }
240
241 std::string ConformanceLevelToString(ConformanceLevel level) const;
242
243 void SetPrintUnknownFields(bool print_unknown_fields) {
244 request_.set_print_unknown_fields(true);
245 }
246
247 void SetPrototypeMessageForCompare(const Message& message) {
248 prototype_message_for_compare_.reset(message.New());
249 }
250
251 protected:
252 virtual std::string InputFormatString(conformance::WireFormat format) const;
253 virtual std::string OutputFormatString(
254 conformance::WireFormat format) const;
255 conformance::ConformanceRequest request_;
256
257 private:
258 ConformanceLevel level_;
259 ::conformance::WireFormat input_format_;
260 ::conformance::WireFormat output_format_;
261 const Message& prototype_message_;
262 std::unique_ptr<Message> prototype_message_for_compare_;
263 std::string test_name_;
264 };
265
266 std::string WireFormatToString(conformance::WireFormat wire_format);
267
268 // Parse payload in the response to the given message. Returns true on
269 // success.
270 virtual bool ParseResponse(
271 const conformance::ConformanceResponse& response,
272 const ConformanceRequestSetting& setting,
273 Message* test_message) = 0;
274
275 void VerifyResponse(const ConformanceRequestSetting& setting,
276 const std::string& equivalent_wire_format,
277 const conformance::ConformanceResponse& response,
278 bool need_report_success, bool require_same_wire_format);
279
280 void TruncateDebugPayload(std::string* payload);
281 const conformance::ConformanceRequest TruncateRequest(
282 const conformance::ConformanceRequest& request);
283 const conformance::ConformanceResponse TruncateResponse(
284 const conformance::ConformanceResponse& response);
285
286 void ReportSuccess(const std::string& test_name);
287 void ReportFailure(const std::string& test_name, ConformanceLevel level,
288 const conformance::ConformanceRequest& request,
289 const conformance::ConformanceResponse& response,
290 absl::string_view message);
291 void ReportSkip(const std::string& test_name,
292 const conformance::ConformanceRequest& request,
293 const conformance::ConformanceResponse& response);
294
295 void RunValidInputTest(const ConformanceRequestSetting& setting,
296 const std::string& equivalent_text_format);
297 void RunValidBinaryInputTest(const ConformanceRequestSetting& setting,
298 const std::string& equivalent_wire_format,
299 bool require_same_wire_format = false);
300
301 void RunTest(const std::string& test_name,
302 const conformance::ConformanceRequest& request,
303 conformance::ConformanceResponse* response);
304
305 void AddExpectedFailedTest(const std::string& test_name);
306
307 virtual void RunSuiteImpl() = 0;
308
309 ConformanceTestRunner* runner_;
310 int successes_;
311 int expected_failures_;
312 bool verbose_;
313 bool performance_;
314 bool enforce_recommended_;
315 std::string output_;
316 std::string output_dir_;
317 std::string failure_list_flag_name_;
318 std::string failure_list_filename_;
319
320 // The set of test names that are expected to fail in this run, but haven't
321 // failed yet.
322 absl::btree_set<std::string> expected_to_fail_;
323
324 // The set of test names that have been run. Used to ensure that there are no
325 // duplicate names in the suite.
326 absl::flat_hash_set<std::string> test_names_;
327
328 // The set of tests that failed, but weren't expected to.
329 absl::btree_set<std::string> unexpected_failing_tests_;
330
331 // The set of tests that succeeded, but weren't expected to.
332 absl::btree_set<std::string> unexpected_succeeding_tests_;
333
334 // The set of tests that the testee opted out of;
335 absl::btree_set<std::string> skipped_;
336 };
337
338 } // namespace protobuf
339 } // namespace google
340
341 #endif // CONFORMANCE_CONFORMANCE_TEST_H