Debian Codegraph @

lang/py3/avro/tests/test_schema.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20 """
21 Test the schema parsing logic.
22 """
23
24 import logging
25 import traceback
26 import unittest
27
28 from avro import schema
29
30
31 # ------------------------------------------------------------------------------
32
33
34 class ExampleSchema(object):
35 def __init__(self, schema_string, valid, name='', comment=''):
36 self._schema_string = schema_string
37 self._valid = valid
38 self._name = name or schema_string # default to schema_string for name
39 self.comment = comment
40
41 @property
42 def schema_string(self):
43 return self._schema_string
44
45 @property
46 def valid(self):
47 return self._valid
48
49 @property
50 def name(self):
51 return self._name
52
53
54 # ------------------------------------------------------------------------------
55 # Example Schemas
56
57
58 def MakePrimitiveExamples():
59 examples = []
60 for type in schema.PRIMITIVE_TYPES:
61 examples.append(ExampleSchema('"%s"' % type, valid=True))
62 examples.append(ExampleSchema('{"type": "%s"}' % type, valid=True))
63 return examples
64
65 PRIMITIVE_EXAMPLES = MakePrimitiveExamples() + [
66 ExampleSchema('"True"', valid=False),
67 ExampleSchema('True', valid=False),
68 ExampleSchema('{"no_type": "test"}', valid=False),
69 ExampleSchema('{"type": "panther"}', valid=False),
70 ]
71
72 FIXED_EXAMPLES = [
73 ExampleSchema('{"type": "fixed", "name": "Test", "size": 1}', valid=True),
74 ExampleSchema("""
75 {
76 "type": "fixed",
77 "name": "MyFixed",
78 "namespace": "org.apache.hadoop.avro",
79 "size": 1
80 }
81 """,
82 valid=True),
83 ExampleSchema("""
84 {
85 "type": "fixed",
86 "name": "Missing size"
87 }
88 """,
89 valid=False),
90 ExampleSchema("""
91 {
92 "type": "fixed",
93 "size": 314
94 }
95 """,
96 valid=False),
97 ]
98
99 ENUM_EXAMPLES = [
100 ExampleSchema(
101 '{"type": "enum", "name": "Test", "symbols": ["A", "B"]}',
102 valid=True),
103 ExampleSchema("""
104 {
105 "type": "enum",
106 "name": "Status",
107 "symbols": "Normal Caution Critical"
108 }
109 """,
110 valid=False),
111 ExampleSchema("""
112 {
113 "type": "enum",
114 "name": [0, 1, 1, 2, 3, 5, 8],
115 "symbols": ["Golden", "Mean"]
116 }
117 """,
118 valid=False),
119 ExampleSchema("""
120 {
121 "type": "enum",
122 "symbols": ["I", "will", "fail", "no", "name"]
123 }
124 """,
125 valid=False),
126 ExampleSchema("""
127 {
128 "type": "enum",
129 "name": "Test"
130 "symbols": ["AA", "AA"]
131 }
132 """,
133 valid=False),
134 ]
135
136 ARRAY_EXAMPLES = [
137 ExampleSchema('{"type": "array", "items": "long"}', valid=True),
138 ExampleSchema("""
139 {
140 "type": "array",
141 "items": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
142 }
143 """,
144 valid=True),
145 ]
146
147 MAP_EXAMPLES = [
148 ExampleSchema('{"type": "map", "values": "long"}', True),
149 ExampleSchema("""
150 {
151 "type": "map",
152 "values": {"type": "enum", "name": "Test", "symbols": ["A", "B"]}
153 }
154 """,
155 valid=True,
156 ),
157 ]
158
159 UNION_EXAMPLES = [
160 ExampleSchema('["string", "null", "long"]', valid=True),
161 ExampleSchema('["null", "null"]', valid=False),
162 ExampleSchema('["long", "long"]', valid=False),
163 ExampleSchema("""
164 [
165 {"type": "array", "items": "long"},
166 {"type": "array", "items": "string"}
167 ]
168 """,
169 valid=False,
170 ),
171 ]
172
173 RECORD_EXAMPLES = [
174 ExampleSchema("""
175 {
176 "type": "record",
177 "name": "Test",
178 "fields": [{"name": "f", "type": "long"}]
179 }
180 """,
181 valid=True,
182 ),
183 ExampleSchema("""
184 {
185 "type": "error",
186 "name": "Test",
187 "fields": [{"name": "f", "type": "long"}]
188 }
189 """,
190 valid=True,
191 ),
192 ExampleSchema("""
193 {
194 "type": "record",
195 "name": "Node",
196 "fields": [
197 {"name": "label", "type": "string"},
198 {"name": "children", "type": {"type": "array", "items": "Node"}}
199 ]
200 }
201 """,
202 valid=True,
203 ),
204 ExampleSchema("""
205 {
206 "type": "record",
207 "name": "Lisp",
208 "fields": [{
209 "name": "value",
210 "type": [
211 "null",
212 "string",
213 {
214 "type": "record",
215 "name": "Cons",
216 "fields": [{"name": "car", "type": "Lisp"},
217 {"name": "cdr", "type": "Lisp"}]
218 }
219 ]
220 }]
221 }
222 """,
223 valid=True,
224 ),
225 ExampleSchema("""
226 {
227 "type": "record",
228 "name": "HandshakeRequest",
229 "namespace": "org.apache.avro.ipc",
230 "fields": [
231 {
232 "name": "clientHash",
233 "type": {"type": "fixed", "name": "MD5", "size": 16}
234 },
235 {"name": "clientProtocol", "type": ["null", "string"]},
236 {"name": "serverHash", "type": "MD5"},
237 {
238 "name": "meta",
239 "type": ["null", {"type": "map", "values": "bytes"}]
240 }
241 ]
242 }
243 """,
244 valid=True,
245 ),
246 ExampleSchema("""
247 {
248 "type": "record",
249 "name": "HandshakeResponse",
250 "namespace": "org.apache.avro.ipc",
251 "fields": [
252 {
253 "name": "match",
254 "type": {
255 "type": "enum",
256 "name": "HandshakeMatch",
257 "symbols": ["BOTH", "CLIENT", "NONE"]
258 }
259 },
260 {"name": "serverProtocol", "type": ["null", "string"]},
261 {
262 "name": "serverHash",
263 "type": ["null", {"name": "MD5", "size": 16, "type": "fixed"}]
264 },
265 {
266 "name": "meta",
267 "type": ["null", {"type": "map", "values": "bytes"}]}]
268 }
269 """,
270 valid=True,
271 ),
272 ExampleSchema("""
273 {
274 "type": "record",
275 "name": "Interop",
276 "namespace": "org.apache.avro",
277 "fields": [
278 {"name": "intField", "type": "int"},
279 {"name": "longField", "type": "long"},
280 {"name": "stringField", "type": "string"},
281 {"name": "boolField", "type": "boolean"},
282 {"name": "floatField", "type": "float"},
283 {"name": "doubleField", "type": "double"},
284 {"name": "bytesField", "type": "bytes"},
285 {"name": "nullField", "type": "null"},
286 {"name": "arrayField", "type": {"type": "array", "items": "double"}},
287 {
288 "name": "mapField",
289 "type": {
290 "type": "map",
291 "values": {"name": "Foo",
292 "type": "record",
293 "fields": [{"name": "label", "type": "string"}]}
294 }
295 },
296 {
297 "name": "unionField",
298 "type": ["boolean", "double", {"type": "array", "items": "bytes"}]
299 },
300 {
301 "name": "enumField",
302 "type": {"type": "enum", "name": "Kind", "symbols": ["A", "B", "C"]}
303 },
304 {
305 "name": "fixedField",
306 "type": {"type": "fixed", "name": "MD5", "size": 16}
307 },
308 {
309 "name": "recordField",
310 "type": {"type": "record",
311 "name": "Node",
312 "fields": [{"name": "label", "type": "string"},
313 {"name": "children",
314 "type": {"type": "array",
315 "items": "Node"}}]}
316 }
317 ]
318 }
319 """,
320 valid=True,
321 ),
322 ExampleSchema("""
323 {
324 "type": "record",
325 "name": "ipAddr",
326 "fields": [{
327 "name": "addr",
328 "type": [
329 {"name": "IPv6", "type": "fixed", "size": 16},
330 {"name": "IPv4", "type": "fixed", "size": 4}
331 ]
332 }]
333 }
334 """,
335 valid=True,
336 ),
337 ExampleSchema("""
338 {
339 "type": "record",
340 "name": "Address",
341 "fields": [
342 {"type": "string"},
343 {"type": "string", "name": "City"}
344 ]
345 }
346 """,
347 valid=False,
348 ),
349 ExampleSchema("""
350 {
351 "type": "record",
352 "name": "Event",
353 "fields": [
354 {"name": "Sponsor"},
355 {"name": "City", "type": "string"}
356 ]
357 }
358 """,
359 valid=False,
360 ),
361 ExampleSchema("""
362 {
363 "type": "record",
364 "fields": "His vision, from the constantly passing bars,"
365 "name", "Rainer"
366 }
367 """,
368 valid=False,
369 ),
370 ExampleSchema("""
371 {
372 "name": ["Tom", "Jerry"],
373 "type": "record",
374 "fields": [{"name": "name", "type": "string"}]
375 }
376 """,
377 valid=False,
378 ),
379 ]
380
381 DOC_EXAMPLES = [
382 ExampleSchema("""
383 {
384 "type": "record",
385 "name": "TestDoc",
386 "doc": "Doc string",
387 "fields": [{"name": "name", "type": "string", "doc": "Doc String"}]
388 }
389 """,
390 valid=True,
391 ),
392 ExampleSchema("""
393 {"type": "enum", "name": "Test", "symbols": ["A", "B"], "doc": "Doc String"}
394 """,
395 valid=True,
396 ),
397 ]
398
399 OTHER_PROP_EXAMPLES = [
400 ExampleSchema("""
401 {
402 "type": "record",
403 "name": "TestRecord",
404 "cp_string": "string",
405 "cp_int": 1,
406 "cp_array": [ 1, 2, 3, 4],
407 "fields": [
408 {"name": "f1", "type": "string", "cp_object": {"a":1,"b":2}},
409 {"name": "f2", "type": "long", "cp_null": null}
410 ]
411 }
412 """,
413 valid=True,
414 ),
415 ExampleSchema(
416 '{"type": "map", "values": "long", "cp_boolean": true}',
417 valid=True,
418 ),
419 ExampleSchema("""
420 {
421 "type": "enum",
422 "name": "TestEnum",
423 "symbols": ["one", "two", "three"],
424 "cp_float": 1.0
425 }
426 """,
427 valid=True,
428 ),
429 ExampleSchema("""
430 {"type": "long", "date": "true"}
431 """,
432 valid=True,
433 ),
434 ]
435
436 EXAMPLES = PRIMITIVE_EXAMPLES
437 EXAMPLES += FIXED_EXAMPLES
438 EXAMPLES += ENUM_EXAMPLES
439 EXAMPLES += ARRAY_EXAMPLES
440 EXAMPLES += MAP_EXAMPLES
441 EXAMPLES += UNION_EXAMPLES
442 EXAMPLES += RECORD_EXAMPLES
443 EXAMPLES += DOC_EXAMPLES
444
445 VALID_EXAMPLES = [e for e in EXAMPLES if e.valid]
446
447
448 # ------------------------------------------------------------------------------
449
450
451 class TestSchema(unittest.TestCase):
452
453 def testCorrectRecursiveExtraction(self):
454 parsed = schema.Parse("""
455 {
456 "type": "record",
457 "name": "X",
458 "fields": [{
459 "name": "y",
460 "type": {
461 "type": "record",
462 "name": "Y",
463 "fields": [{"name": "Z", "type": "X"}, {"name": "W", "type": "X"}]
464 }
465 }]
466 }
467 """)
468 logging.debug('Parsed schema:\n%s', parsed)
469 logging.debug('Fields: %s', parsed.fields)
470 t = schema.Parse(str(parsed.fields[0].type))
471 # If we've made it this far, the subschema was reasonably stringified;
472 # it could be reparsed.
473 self.assertEqual("X", t.fields[0].type.name)
474
475 def testParse(self):
476 correct = 0
477 for iexample, example in enumerate(EXAMPLES):
478 logging.debug('Testing example #%d\n%s', iexample, example.schema_string)
479 try:
480 schema.Parse(example.schema_string)
481 if example.valid:
482 correct += 1
483 else:
484 self.fail('Invalid schema was parsed:\n%s' % example.schema_string)
485 except Exception as exn:
486 if example.valid:
487 self.fail(
488 'Valid schema failed to parse: %r\n%s'
489 % (example.schema_string, traceback.format_exc()))
490 else:
491 if logging.getLogger().getEffectiveLevel() <= 5:
492 logging.debug('Expected error:\n%s', traceback.format_exc())
493 else:
494 logging.debug('Expected error: %r', exn)
495 correct += 1
496
497 self.assertEqual(
498 correct,
499 len(EXAMPLES),
500 'Parse behavior correct on %d out of %d schemas.'
501 % (correct, len(EXAMPLES)),
502 )
503
504 def testValidCastToStringAfterParse(self):
505 """
506 Test that the string generated by an Avro Schema object
507 is, in fact, a valid Avro schema.
508 """
509 correct = 0
510 for example in VALID_EXAMPLES:
511 schema_data = schema.Parse(example.schema_string)
512 schema.Parse(str(schema_data))
513 correct += 1
514
515 fail_msg = "Cast to string success on %d out of %d schemas" % \
516 (correct, len(VALID_EXAMPLES))
517 self.assertEqual(correct, len(VALID_EXAMPLES), fail_msg)
518
519 def testEquivalenceAfterRoundTrip(self):
520 """
521 1. Given a string, parse it to get Avro schema "original".
522 2. Serialize "original" to a string and parse that string
523 to generate Avro schema "round trip".
524 3. Ensure "original" and "round trip" schemas are equivalent.
525 """
526 correct = 0
527 for example in VALID_EXAMPLES:
528 original_schema = schema.Parse(example.schema_string)
529 round_trip_schema = schema.Parse(str(original_schema))
530 if original_schema == round_trip_schema:
531 correct += 1
532 debug_msg = "%s: ROUND TRIP SUCCESS" % example.name
533 else:
534 debug_msg = "%s: ROUND TRIP FAILURE" % example.name
535 self.fail(
536 "Round trip failure: %s, %s, %s"
537 % (example.name, original_schema, str(original_schema)))
538
539 fail_msg = "Round trip success on %d out of %d schemas" % \
540 (correct, len(VALID_EXAMPLES))
541 self.assertEqual(correct, len(VALID_EXAMPLES), fail_msg)
542
543 def testFullname(self):
544 """The fullname is determined in one of the following ways:
545 * A name and namespace are both specified. For example,
546 one might use "name": "X", "namespace": "org.foo"
547 to indicate the fullname "org.foo.X".
548 * A fullname is specified. If the name specified contains
549 a dot, then it is assumed to be a fullname, and any
550 namespace also specified is ignored. For example,
551 use "name": "org.foo.X" to indicate the
552 fullname "org.foo.X".
553 * A name only is specified, i.e., a name that contains no
554 dots. In this case the namespace is taken from the most
555 tightly encosing schema or protocol. For example,
556 if "name": "X" is specified, and this occurs
557 within a field of the record definition
558 of "org.foo.Y", then the fullname is "org.foo.X".
559
560 References to previously defined names are as in the latter
561 two cases above: if they contain a dot they are a fullname, if
562 they do not contain a dot, the namespace is the namespace of
563 the enclosing definition.
564
565 Primitive type names have no namespace and their names may
566 not be defined in any namespace. A schema may only contain
567 multiple definitions of a fullname if the definitions are
568 equivalent.
569 """
570 # relative name and namespace specified
571 self.assertEqual(schema.Name('a', 'o.a.h').fullname, 'o.a.h.a')
572
573 # absolute name and namespace specified
574 self.assertEqual(schema.Name('.a', 'o.a.h').fullname, '.a')
575
576 # absolute name and namespace specified
577 fullname = schema.Name('a.b.c.d', 'o.a.h').fullname
578 self.assertEqual(fullname, 'a.b.c.d')
579
580 def testDocAttributes(self):
581 correct = 0
582 for example in DOC_EXAMPLES:
583 original_schema = schema.Parse(example.schema_string)
584 if original_schema.doc is not None:
585 correct += 1
586 if original_schema.type == 'record':
587 for f in original_schema.fields:
588 if f.doc is None:
589 self.fail(
590 "Failed to preserve 'doc' in fields: "
591 + example.schema_string)
592 self.assertEqual(correct,len(DOC_EXAMPLES))
593
594 def testOtherAttributes(self):
595 correct = 0
596 props = {}
597 for example in OTHER_PROP_EXAMPLES:
598 original_schema = schema.Parse(example.schema_string)
599 round_trip_schema = schema.Parse(str(original_schema))
600 self.assertEqual(original_schema.other_props,round_trip_schema.other_props)
601 if original_schema.type == "record":
602 field_props = 0
603 for f in original_schema.fields:
604 if f.other_props:
605 props.update(f.other_props)
606 field_props += 1
607 self.assertEqual(field_props,len(original_schema.fields))
608 if original_schema.other_props:
609 props.update(original_schema.other_props)
610 correct += 1
611 for k in props:
612 v = props[k]
613 if k == "cp_boolean":
614 self.assertEqual(type(v), bool)
615 elif k == "cp_int":
616 self.assertEqual(type(v), int)
617 elif k == "cp_object":
618 self.assertEqual(type(v), dict)
619 elif k == "cp_float":
620 self.assertEqual(type(v), float)
621 elif k == "cp_array":
622 self.assertEqual(type(v), list)
623 self.assertEqual(correct,len(OTHER_PROP_EXAMPLES))
624
625
626 # ------------------------------------------------------------------------------
627
628
629 if __name__ == '__main__':
630 raise Exception('Use run_tests.py')