Debian Codegraph @

lang/py3/avro/schema.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 """Representation of Avro schemas.
22
23 A schema may be one of:
24 - A record, mapping field names to field value data;
25 - An error, equivalent to a record;
26 - An enum, containing one of a small set of symbols;
27 - An array of values, all of the same schema;
28 - A map containing string/value pairs, each of a declared schema;
29 - A union of other schemas;
30 - A fixed sized binary object;
31 - A unicode string;
32 - A sequence of bytes;
33 - A 32-bit signed int;
34 - A 64-bit signed long;
35 - A 32-bit floating-point float;
36 - A 64-bit floating-point double;
37 - A boolean;
38 - Null.
39 """
40
41
42 import abc
43 import collections
44 import json
45 import logging
46 import re
47
48 logger = logging.getLogger(__name__)
49
50 # ------------------------------------------------------------------------------
51 # Constants
52
53 # Log level more verbose than DEBUG=10, INFO=20, etc.
54 DEBUG_VERBOSE=5
55
56
57 NULL = 'null'
58 BOOLEAN = 'boolean'
59 STRING = 'string'
60 BYTES = 'bytes'
61 INT = 'int'
62 LONG = 'long'
63 FLOAT = 'float'
64 DOUBLE = 'double'
65 FIXED = 'fixed'
66 ENUM = 'enum'
67 RECORD = 'record'
68 ERROR = 'error'
69 ARRAY = 'array'
70 MAP = 'map'
71 UNION = 'union'
72
73 # Request and error unions are part of Avro protocols:
74 REQUEST = 'request'
75 ERROR_UNION = 'error_union'
76
77 PRIMITIVE_TYPES = frozenset([
78 NULL,
79 BOOLEAN,
80 STRING,
81 BYTES,
82 INT,
83 LONG,
84 FLOAT,
85 DOUBLE,
86 ])
87
88 NAMED_TYPES = frozenset([
89 FIXED,
90 ENUM,
91 RECORD,
92 ERROR,
93 ])
94
95 VALID_TYPES = frozenset.union(
96 PRIMITIVE_TYPES,
97 NAMED_TYPES,
98 [
99 ARRAY,
100 MAP,
101 UNION,
102 REQUEST,
103 ERROR_UNION,
104 ],
105 )
106
107 SCHEMA_RESERVED_PROPS = frozenset([
108 'type',
109 'name',
110 'namespace',
111 'fields', # Record
112 'items', # Array
113 'size', # Fixed
114 'symbols', # Enum
115 'values', # Map
116 'doc',
117 ])
118
119 FIELD_RESERVED_PROPS = frozenset([
120 'default',
121 'name',
122 'doc',
123 'order',
124 'type',
125 ])
126
127 VALID_FIELD_SORT_ORDERS = frozenset([
128 'ascending',
129 'descending',
130 'ignore',
131 ])
132
133
134 # ------------------------------------------------------------------------------
135 # Exceptions
136
137
138 class Error(Exception):
139 """Base class for errors in this module."""
140 pass
141
142
143 class AvroException(Error):
144 """Generic Avro schema error."""
145 pass
146
147
148 class SchemaParseException(AvroException):
149 """Error while parsing a JSON schema descriptor."""
150 pass
151
152
153 # ------------------------------------------------------------------------------
154
155
156 class ImmutableDict(dict):
157 """Dictionary guaranteed immutable.
158
159 All mutations raise an exception.
160 Behaves exactly as a dict otherwise.
161 """
162
163 def __init__(self, items=None, **kwargs):
164 if items is not None:
165 super(ImmutableDict, self).__init__(items)
166 assert (len(kwargs) == 0)
167 else:
168 super(ImmutableDict, self).__init__(**kwargs)
169
170 def __setitem__(self, key, value):
171 raise Exception(
172 'Attempting to map key %r to value %r in ImmutableDict %r'
173 % (key, value, self))
174
175 def __delitem__(self, key):
176 raise Exception(
177 'Attempting to remove mapping for key %r in ImmutableDict %r'
178 % (key, self))
179
180 def clear(self):
181 raise Exception('Attempting to clear ImmutableDict %r' % self)
182
183 def update(self, items=None, **kwargs):
184 raise Exception(
185 'Attempting to update ImmutableDict %r with items=%r, kwargs=%r'
186 % (self, args, kwargs))
187
188 def pop(self, key, default=None):
189 raise Exception(
190 'Attempting to pop key %r from ImmutableDict %r' % (key, self))
191
192 def popitem(self):
193 raise Exception('Attempting to pop item from ImmutableDict %r' % self)
194
195
196 # ------------------------------------------------------------------------------
197
198
199 class Schema(object, metaclass=abc.ABCMeta):
200 """Abstract base class for all Schema classes."""
201
202 def __init__(self, type, other_props=None):
203 """Initializes a new schema object.
204
205 Args:
206 type: Type of the schema to initialize.
207 other_props: Optional dictionary of additional properties.
208 """
209 if type not in VALID_TYPES:
210 raise SchemaParseException('%r is not a valid Avro type.' % type)
211
212 # All properties of this schema, as a map: property name -> property value
213 self._props = {}
214
215 self._props['type'] = type
216 self._type = type
217
218 if other_props:
219 self._props.update(other_props)
220
221 @property
222 def name(self):
223 """Returns: the simple name of this schema."""
224 return self._props['name']
225
226 @property
227 def fullname(self):
228 """Returns: the fully qualified name of this schema."""
229 # By default, the full name is the simple name.
230 # Named schemas override this behavior to include the namespace.
231 return self.name
232
233 @property
234 def namespace(self):
235 """Returns: the namespace this schema belongs to, if any, or None."""
236 return self._props.get('namespace', None)
237
238 @property
239 def type(self):
240 """Returns: the type of this schema."""
241 return self._type
242
243 @property
244 def doc(self):
245 """Returns: the documentation associated to this schema, if any, or None."""
246 return self._props.get('doc', None)
247
248 @property
249 def props(self):
250 """Reports all the properties of this schema.
251
252 Includes all properties, reserved and non reserved.
253 JSON properties of this schema are directly generated from this dict.
254
255 Returns:
256 A read-only dictionary of properties associated to this schema.
257 """
258 return ImmutableDict(self._props)
259
260 @property
261 def other_props(self):
262 """Returns: the dictionary of non-reserved properties."""
263 return dict(FilterKeysOut(items=self._props, keys=SCHEMA_RESERVED_PROPS))
264
265 def __str__(self):
266 """Returns: the JSON representation of this schema."""
267 return json.dumps(self.to_json())
268
269 @abc.abstractmethod
270 def to_json(self, names):
271 """Converts the schema object into its AVRO specification representation.
272
273 Schema types that have names (records, enums, and fixed) must
274 be aware of not re-defining schemas that are already listed
275 in the parameter names.
276 """
277 raise Exception('Cannot run abstract method.')
278
279
280 # ------------------------------------------------------------------------------
281
282
283 _RE_NAME = re.compile(r'[A-Za-z_][A-Za-z0-9_]*')
284
285 _RE_FULL_NAME = re.compile(
286 r'^'
287 r'[.]?(?:[A-Za-z_][A-Za-z0-9_]*[.])*' # optional namespace
288 r'([A-Za-z_][A-Za-z0-9_]*)' # name
289 r'$'
290 )
291
292 class Name(object):
293 """Representation of an Avro name."""
294
295 def __init__(self, name, namespace=None):
296 """Parses an Avro name.
297
298 Args:
299 name: Avro name to parse (relative or absolute).
300 namespace: Optional explicit namespace if the name is relative.
301 """
302 # Normalize: namespace is always defined as a string, possibly empty.
303 if namespace is None: namespace = ''
304
305 if '.' in name:
306 # name is absolute, namespace is ignored:
307 self._fullname = name
308
309 match = _RE_FULL_NAME.match(self._fullname)
310 if match is None:
311 raise SchemaParseException(
312 'Invalid absolute schema name: %r.' % self._fullname)
313
314 self._name = match.group(1)
315 self._namespace = self._fullname[:-(len(self._name) + 1)]
316
317 else:
318 # name is relative, combine with explicit namespace:
319 self._name = name
320 self._namespace = namespace
321 self._fullname = '%s.%s' % (self._namespace, self._name)
322
323 # Validate the fullname:
324 if _RE_FULL_NAME.match(self._fullname) is None:
325 raise SchemaParseException(
326 'Invalid schema name %r infered from name %r and namespace %r.'
327 % (self._fullname, self._name, self._namespace))
328
329 def __eq__(self, other):
330 if not isinstance(other, Name):
331 return False
332 return (self.fullname == other.fullname)
333
334 @property
335 def simple_name(self):
336 """Returns: the simple name part of this name."""
337 return self._name
338
339 @property
340 def namespace(self):
341 """Returns: this name's namespace, possible the empty string."""
342 return self._namespace
343
344 @property
345 def fullname(self):
346 """Returns: the full name (always contains a period '.')."""
347 return self._fullname
348
349
350 # ------------------------------------------------------------------------------
351
352
353 class Names(object):
354 """Tracks Avro named schemas and default namespace during parsing."""
355
356 def __init__(self, default_namespace=None, names=None):
357 """Initializes a new name tracker.
358
359 Args:
360 default_namespace: Optional default namespace.
361 names: Optional initial mapping of known named schemas.
362 """
363 if names is None:
364 names = {}
365 self._names = names
366 self._default_namespace = default_namespace
367
368 @property
369 def names(self):
370 """Returns: the mapping of known named schemas."""
371 return self._names
372
373 @property
374 def default_namespace(self):
375 """Returns: the default namespace, if any, or None."""
376 return self._default_namespace
377
378 def NewWithDefaultNamespace(self, namespace):
379 """Creates a new name tracker from this tracker, but with a new default ns.
380
381 Args:
382 namespace: New default namespace to use.
383 Returns:
384 New name tracker with the specified default namespace.
385 """
386 return Names(names=self._names, default_namespace=namespace)
387
388 def GetName(self, name, namespace=None):
389 """Resolves the Avro name according to this name tracker's state.
390
391 Args:
392 name: Name to resolve (absolute or relative).
393 namespace: Optional explicit namespace.
394 Returns:
395 The specified name, resolved according to this tracker.
396 """
397 if namespace is None: namespace = self._default_namespace
398 return Name(name=name, namespace=namespace)
399
400 def has_name(self, name, namespace=None):
401 avro_name = self.GetName(name=name, namespace=namespace)
402 return avro_name.fullname in self._names
403
404 def get_name(self, name, namespace=None):
405 avro_name = self.GetName(name=name, namespace=namespace)
406 return self._names.get(avro_name.fullname, None)
407
408 def GetSchema(self, name, namespace=None):
409 """Resolves an Avro schema by name.
410
411 Args:
412 name: Name (relative or absolute) of the Avro schema to look up.
413 namespace: Optional explicit namespace.
414 Returns:
415 The schema with the specified name, if any, or None.
416 """
417 avro_name = self.GetName(name=name, namespace=namespace)
418 return self._names.get(avro_name.fullname, None)
419
420 def prune_namespace(self, properties):
421 """given a properties, return properties with namespace removed if
422 it matches the own default namespace
423 """
424 if self.default_namespace is None:
425 # I have no default -- no change
426 return properties
427 if 'namespace' not in properties:
428 # he has no namespace - no change
429 return properties
430 if properties['namespace'] != self.default_namespace:
431 # we're different - leave his stuff alone
432 return properties
433 # we each have a namespace and it's redundant. delete his.
434 prunable = properties.copy()
435 del(prunable['namespace'])
436 return prunable
437
438 def Register(self, schema):
439 """Registers a new named schema in this tracker.
440
441 Args:
442 schema: Named Avro schema to register in this tracker.
443 """
444 if schema.fullname in VALID_TYPES:
445 raise SchemaParseException(
446 '%s is a reserved type name.' % schema.fullname)
447 if schema.fullname in self.names:
448 raise SchemaParseException(
449 'Avro name %r already exists.' % schema.fullname)
450
451 logger.log(DEBUG_VERBOSE, 'Register new name for %r', schema.fullname)
452 self._names[schema.fullname] = schema
453
454
455 # ------------------------------------------------------------------------------
456
457
458 class NamedSchema(Schema):
459 """Abstract base class for named schemas.
460
461 Named schemas are enumerated in NAMED_TYPES.
462 """
463
464 def __init__(
465 self,
466 type,
467 name,
468 namespace=None,
469 names=None,
470 other_props=None,
471 ):
472 """Initializes a new named schema object.
473
474 Args:
475 type: Type of the named schema.
476 name: Name (absolute or relative) of the schema.
477 namespace: Optional explicit namespace if name is relative.
478 names: Tracker to resolve and register Avro names.
479 other_props: Optional map of additional properties of the schema.
480 """
481 assert (type in NAMED_TYPES), ('Invalid named type: %r' % type)
482 self._avro_name = names.GetName(name=name, namespace=namespace)
483
484 super(NamedSchema, self).__init__(type, other_props)
485
486 names.Register(self)
487
488 self._props['name'] = self.name
489 if self.namespace:
490 self._props['namespace'] = self.namespace
491
492 @property
493 def avro_name(self):
494 """Returns: the Name object describing this schema's name."""
495 return self._avro_name
496
497 @property
498 def name(self):
499 return self._avro_name.simple_name
500
501 @property
502 def namespace(self):
503 return self._avro_name.namespace
504
505 @property
506 def fullname(self):
507 return self._avro_name.fullname
508
509 def name_ref(self, names):
510 """Reports this schema name relative to the specified name tracker.
511
512 Args:
513 names: Avro name tracker to relativise this schema name against.
514 Returns:
515 This schema name, relativised against the specified name tracker.
516 """
517 if self.namespace == names.default_namespace:
518 return self.name
519 else:
520 return self.fullname
521
522
523 # ------------------------------------------------------------------------------
524
525
526 _NO_DEFAULT = object()
527
528
529 class Field(object):
530 """Representation of the schema of a field in a record."""
531
532 def __init__(
533 self,
534 type,
535 name,
536 index,
537 has_default,
538 default=_NO_DEFAULT,
539 order=None,
540 names=None,
541 doc=None,
542 other_props=None
543 ):
544 """Initializes a new Field object.
545
546 Args:
547 type: Avro schema of the field.
548 name: Name of the field.
549 index: 0-based position of the field.
550 has_default:
551 default:
552 order:
553 names:
554 doc:
555 other_props:
556 """
557 if (not isinstance(name, str)) or (len(name) == 0):
558 raise SchemaParseException('Invalid record field name: %r.' % name)
559 if (order is not None) and (order not in VALID_FIELD_SORT_ORDERS):
560 raise SchemaParseException('Invalid record field order: %r.' % order)
561
562 # All properties of this record field:
563 self._props = {}
564
565 self._has_default = has_default
566 if other_props:
567 self._props.update(other_props)
568
569 self._index = index
570 self._type = self._props['type'] = type
571 self._name = self._props['name'] = name
572
573 # TODO: check to ensure default is valid
574 if has_default:
575 self._props['default'] = default
576
577 if order is not None:
578 self._props['order'] = order
579
580 if doc is not None:
581 self._props['doc'] = doc
582
583 @property
584 def type(self):
585 """Returns: the schema of this field."""
586 return self._type
587
588 @property
589 def name(self):
590 """Returns: this field name."""
591 return self._name
592
593 @property
594 def index(self):
595 """Returns: the 0-based index of this field in the record."""
596 return self._index
597
598 @property
599 def default(self):
600 return self._props['default']
601
602 @property
603 def has_default(self):
604 return self._has_default
605
606 @property
607 def order(self):
608 return self._props.get('order', None)
609
610 @property
611 def doc(self):
612 return self._props.get('doc', None)
613
614 @property
615 def props(self):
616 return self._props
617
618 @property
619 def other_props(self):
620 return FilterKeysOut(items=self._props, keys=FIELD_RESERVED_PROPS)
621
622 def __str__(self):
623 return json.dumps(self.to_json())
624
625 def to_json(self, names=None):
626 if names is None:
627 names = Names()
628 to_dump = self.props.copy()
629 to_dump['type'] = self.type.to_json(names)
630 return to_dump
631
632 def __eq__(self, that):
633 to_cmp = json.loads(str(self))
634 return to_cmp == json.loads(str(that))
635
636
637 # ------------------------------------------------------------------------------
638 # Primitive Types
639
640
641 class PrimitiveSchema(Schema):
642 """Schema of a primitive Avro type.
643
644 Valid primitive types are defined in PRIMITIVE_TYPES.
645 """
646
647 def __init__(self, type, other_props=None):
648 """Initializes a new schema object for the specified primitive type.
649
650 Args:
651 type: Type of the schema to construct. Must be primitive.
652 """
653 if type not in PRIMITIVE_TYPES:
654 raise AvroException('%r is not a valid primitive type.' % type)
655 super(PrimitiveSchema, self).__init__(type, other_props=other_props)
656
657 @property
658 def name(self):
659 """Returns: the simple name of this schema."""
660 # The name of a primitive type is the type itself.
661 return self.type
662
663 def to_json(self, names=None):
664 if len(self.props) == 1:
665 return self.fullname
666 else:
667 return self.props
668
669 def __eq__(self, that):
670 return self.props == that.props
671
672
673 # ------------------------------------------------------------------------------
674 # Complex Types (non-recursive)
675
676
677 class FixedSchema(NamedSchema):
678 def __init__(
679 self,
680 name,
681 namespace,
682 size,
683 names=None,
684 other_props=None,
685 ):
686 # Ensure valid ctor args
687 if not isinstance(size, int):
688 fail_msg = 'Fixed Schema requires a valid integer for size property.'
689 raise AvroException(fail_msg)
690
691 super(FixedSchema, self).__init__(
692 type=FIXED,
693 name=name,
694 namespace=namespace,
695 names=names,
696 other_props=other_props,
697 )
698 self._props['size'] = size
699
700 @property
701 def size(self):
702 """Returns: the size of this fixed schema, in bytes."""
703 return self._props['size']
704
705 def to_json(self, names=None):
706 if names is None:
707 names = Names()
708 if self.fullname in names.names:
709 return self.name_ref(names)
710 else:
711 names.names[self.fullname] = self
712 return names.prune_namespace(self.props)
713
714 def __eq__(self, that):
715 return self.props == that.props
716
717
718 # ------------------------------------------------------------------------------
719
720
721 class EnumSchema(NamedSchema):
722 def __init__(
723 self,
724 name,
725 namespace,
726 symbols,
727 names=None,
728 doc=None,
729 other_props=None,
730 ):
731 """Initializes a new enumeration schema object.
732
733 Args:
734 name: Simple name of this enumeration.
735 namespace: Optional namespace.
736 symbols: Ordered list of symbols defined in this enumeration.
737 names:
738 doc:
739 other_props:
740 """
741 symbols = tuple(symbols)
742 symbol_set = frozenset(symbols)
743 if (len(symbol_set) != len(symbols)
744 or not all(map(lambda symbol: isinstance(symbol, str), symbols))):
745 raise AvroException(
746 'Invalid symbols for enum schema: %r.' % (symbols,))
747
748 super(EnumSchema, self).__init__(
749 type=ENUM,
750 name=name,
751 namespace=namespace,
752 names=names,
753 other_props=other_props,
754 )
755
756 self._props['symbols'] = symbols
757 if doc is not None:
758 self._props['doc'] = doc
759
760 @property
761 def symbols(self):
762 """Returns: the symbols defined in this enum."""
763 return self._props['symbols']
764
765 def to_json(self, names=None):
766 if names is None:
767 names = Names()
768 if self.fullname in names.names:
769 return self.name_ref(names)
770 else:
771 names.names[self.fullname] = self
772 return names.prune_namespace(self.props)
773
774 def __eq__(self, that):
775 return self.props == that.props
776
777
778 # ------------------------------------------------------------------------------
779 # Complex Types (recursive)
780
781
782 class ArraySchema(Schema):
783 """Schema of an array."""
784
785 def __init__(self, items, other_props=None):
786 """Initializes a new array schema object.
787
788 Args:
789 items: Avro schema of the array items.
790 other_props:
791 """
792 super(ArraySchema, self).__init__(
793 type=ARRAY,
794 other_props=other_props,
795 )
796 self._items_schema = items
797 self._props['items'] = items
798
799 @property
800 def items(self):
801 """Returns: the schema of the items in this array."""
802 return self._items_schema
803
804 def to_json(self, names=None):
805 if names is None:
806 names = Names()
807 to_dump = self.props.copy()
808 item_schema = self.items
809 to_dump['items'] = item_schema.to_json(names)
810 return to_dump
811
812 def __eq__(self, that):
813 to_cmp = json.loads(str(self))
814 return to_cmp == json.loads(str(that))
815
816
817 # ------------------------------------------------------------------------------
818
819
820 class MapSchema(Schema):
821 """Schema of a map."""
822
823 def __init__(self, values, other_props=None):
824 """Initializes a new map schema object.
825
826 Args:
827 values: Avro schema of the map values.
828 other_props:
829 """
830 super(MapSchema, self).__init__(
831 type=MAP,
832 other_props=other_props,
833 )
834 self._values_schema = values
835 self._props['values'] = values
836
837 @property
838 def values(self):
839 """Returns: the schema of the values in this map."""
840 return self._values_schema
841
842 def to_json(self, names=None):
843 if names is None:
844 names = Names()
845 to_dump = self.props.copy()
846 to_dump['values'] = self.values.to_json(names)
847 return to_dump
848
849 def __eq__(self, that):
850 to_cmp = json.loads(str(self))
851 return to_cmp == json.loads(str(that))
852
853
854 # ------------------------------------------------------------------------------
855
856
857 class UnionSchema(Schema):
858 """Schema of a union."""
859
860 def __init__(self, schemas):
861 """Initializes a new union schema object.
862
863 Args:
864 schemas: Ordered collection of schema branches in the union.
865 """
866 super(UnionSchema, self).__init__(type=UNION)
867 self._schemas = tuple(schemas)
868
869 # Validate the schema branches:
870
871 # All named schema names are unique:
872 named_branches = tuple(
873 filter(lambda schema: schema.type in NAMED_TYPES, self._schemas))
874 unique_names = frozenset(map(lambda schema: schema.fullname, named_branches))
875 if len(unique_names) != len(named_branches):
876 raise AvroException(
877 'Invalid union branches with duplicate schema name:%s'
878 % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas)))
879
880 # Types are unique within unnamed schemas, and union is not allowed:
881 unnamed_branches = tuple(
882 filter(lambda schema: schema.type not in NAMED_TYPES, self._schemas))
883 unique_types = frozenset(map(lambda schema: schema.type, unnamed_branches))
884 if UNION in unique_types:
885 raise AvroException(
886 'Invalid union branches contain other unions:%s'
887 % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas)))
888 if len(unique_types) != len(unnamed_branches):
889 raise AvroException(
890 'Invalid union branches with duplicate type:%s'
891 % ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas)))
892
893 @property
894 def schemas(self):
895 """Returns: the ordered list of schema branches in the union."""
896 return self._schemas
897
898 def to_json(self, names=None):
899 if names is None:
900 names = Names()
901 to_dump = []
902 for schema in self.schemas:
903 to_dump.append(schema.to_json(names))
904 return to_dump
905
906 def __eq__(self, that):
907 to_cmp = json.loads(str(self))
908 return to_cmp == json.loads(str(that))
909
910
911 # ------------------------------------------------------------------------------
912
913
914 class ErrorUnionSchema(UnionSchema):
915 """Schema representing the declared errors of a protocol message."""
916
917 def __init__(self, schemas):
918 """Initializes an error-union schema.
919
920 Args:
921 schema: collection of error schema.
922 """
923 # TODO: check that string isn't already listed explicitly as an error.
924 # Prepend "string" to handle system errors
925 schemas = [PrimitiveSchema(type=STRING)] + list(schemas)
926 super(ErrorUnionSchema, self).__init__(schemas=schemas)
927
928 def to_json(self, names=None):
929 if names is None:
930 names = Names()
931 to_dump = []
932 for schema in self.schemas:
933 # Don't print the system error schema
934 if schema.type == STRING: continue
935 to_dump.append(schema.to_json(names))
936 return to_dump
937
938
939 # ------------------------------------------------------------------------------
940
941
942 class RecordSchema(NamedSchema):
943 """Schema of a record."""
944
945 @staticmethod
946 def _MakeField(index, field_desc, names):
947 """Builds field schemas from a list of field JSON descriptors.
948
949 Args:
950 index: 0-based index of the field in the record.
951 field_desc: JSON descriptors of a record field.
952 names: Avro schema tracker.
953 Return:
954 The field schema.
955 """
956 field_schema = SchemaFromJSONData(
957 json_data=field_desc['type'],
958 names=names,
959 )
960 other_props = (
961 dict(FilterKeysOut(items=field_desc, keys=FIELD_RESERVED_PROPS)))
962 return Field(
963 type=field_schema,
964 name=field_desc['name'],
965 index=index,
966 has_default=('default' in field_desc),
967 default=field_desc.get('default', _NO_DEFAULT),
968 order=field_desc.get('order', None),
969 names=names,
970 doc=field_desc.get('doc', None),
971 other_props=other_props,
972 )
973
974 @staticmethod
975 def _MakeFieldList(field_desc_list, names):
976 """Builds field schemas from a list of field JSON descriptors.
977
978 Guarantees field name unicity.
979
980 Args:
981 field_desc_list: collection of field JSON descriptors.
982 names: Avro schema tracker.
983 Yields
984 Field schemas.
985 """
986 for index, field_desc in enumerate(field_desc_list):
987 yield RecordSchema._MakeField(index, field_desc, names)
988
989 @staticmethod
990 def _MakeFieldMap(fields):
991 """Builds the field map.
992
993 Guarantees field name unicity.
994
995 Args:
996 fields: iterable of field schema.
997 Returns:
998 A read-only map of field schemas, indexed by name.
999 """
1000 field_map = {}
1001 for field in fields:
1002 if field.name in field_map:
1003 raise SchemaParseException(
1004 'Duplicate field name %r in list %r.' % (field.name, field_desc_list))
1005 field_map[field.name] = field
1006 return ImmutableDict(field_map)
1007
1008 def __init__(
1009 self,
1010 name,
1011 namespace,
1012 fields=None,
1013 make_fields=None,
1014 names=None,
1015 record_type=RECORD,
1016 doc=None,
1017 other_props=None
1018 ):
1019 """Initializes a new record schema object.
1020
1021 Args:
1022 name: Name of the record (absolute or relative).
1023 namespace: Optional namespace the record belongs to, if name is relative.
1024 fields: collection of fields to add to this record.
1025 Exactly one of fields or make_fields must be specified.
1026 make_fields: function creating the fields that belong to the record.
1027 The function signature is: make_fields(names) -> ordered field list.
1028 Exactly one of fields or make_fields must be specified.
1029 names:
1030 record_type: Type of the record: one of RECORD, ERROR or REQUEST.
1031 Protocol requests are not named.
1032 doc:
1033 other_props:
1034 """
1035 if record_type == REQUEST:
1036 # Protocol requests are not named:
1037 super(NamedSchema, self).__init__(
1038 type=REQUEST,
1039 other_props=other_props,
1040 )
1041 elif record_type in [RECORD, ERROR]:
1042 # Register this record name in the tracker:
1043 super(RecordSchema, self).__init__(
1044 type=record_type,
1045 name=name,
1046 namespace=namespace,
1047 names=names,
1048 other_props=other_props,
1049 )
1050 else:
1051 raise SchemaParseException(
1052 'Invalid record type: %r.' % record_type)
1053
1054 if record_type in [RECORD, ERROR]:
1055 avro_name = names.GetName(name=name, namespace=namespace)
1056 nested_names = names.NewWithDefaultNamespace(namespace=avro_name.namespace)
1057 elif record_type == REQUEST:
1058 # Protocol request has no name: no need to change default namespace:
1059 nested_names = names
1060
1061 if fields is None:
1062 fields = make_fields(names=nested_names)
1063 else:
1064 assert (make_fields is None)
1065 self._fields = tuple(fields)
1066
1067 self._field_map = RecordSchema._MakeFieldMap(self._fields)
1068
1069 self._props['fields'] = fields
1070 if doc is not None:
1071 self._props['doc'] = doc
1072
1073 @property
1074 def fields(self):
1075 """Returns: the field schemas, as an ordered tuple."""
1076 return self._fields
1077
1078 @property
1079 def field_map(self):
1080 """Returns: a read-only map of the field schemas index by field names."""
1081 return self._field_map
1082
1083 def to_json(self, names=None):
1084 if names is None:
1085 names = Names()
1086 # Request records don't have names
1087 if self.type == REQUEST:
1088 return [f.to_json(names) for f in self.fields]
1089
1090 if self.fullname in names.names:
1091 return self.name_ref(names)
1092 else:
1093 names.names[self.fullname] = self
1094
1095 to_dump = names.prune_namespace(self.props.copy())
1096 to_dump['fields'] = [f.to_json(names) for f in self.fields]
1097 return to_dump
1098
1099 def __eq__(self, that):
1100 to_cmp = json.loads(str(self))
1101 return to_cmp == json.loads(str(that))
1102
1103
1104 # ------------------------------------------------------------------------------
1105 # Module functions
1106
1107
1108 def FilterKeysOut(items, keys):
1109 """Filters a collection of (key, value) items.
1110
1111 Exclude any item whose key belongs to keys.
1112
1113 Args:
1114 items: Dictionary of items to filter the keys out of.
1115 keys: Keys to filter out.
1116 Yields:
1117 Filtered items.
1118 """
1119 for key, value in items.items():
1120 if key in keys: continue
1121 yield (key, value)
1122
1123
1124 # ------------------------------------------------------------------------------
1125
1126
1127 def _SchemaFromJSONString(json_string, names):
1128 if json_string in PRIMITIVE_TYPES:
1129 return PrimitiveSchema(type=json_string)
1130 else:
1131 # Look for a known named schema:
1132 schema = names.GetSchema(name=json_string)
1133 if schema is None:
1134 raise SchemaParseException(
1135 'Unknown named schema %r, known names: %r.'
1136 % (json_string, sorted(names.names)))
1137 return schema
1138
1139
1140 def _SchemaFromJSONArray(json_array, names):
1141 def MakeSchema(desc):
1142 return SchemaFromJSONData(json_data=desc, names=names)
1143 return UnionSchema(map(MakeSchema, json_array))
1144
1145
1146 def _SchemaFromJSONObject(json_object, names):
1147 type = json_object.get('type')
1148 if type is None:
1149 raise SchemaParseException(
1150 'Avro schema JSON descriptor has no "type" property: %r' % json_object)
1151
1152 other_props = dict(
1153 FilterKeysOut(items=json_object, keys=SCHEMA_RESERVED_PROPS))
1154
1155 if type in PRIMITIVE_TYPES:
1156 # FIXME should not ignore other properties
1157 return PrimitiveSchema(type, other_props=other_props)
1158
1159 elif type in NAMED_TYPES:
1160 name = json_object.get('name')
1161 namespace = json_object.get('namespace', names.default_namespace)
1162 if type == FIXED:
1163 size = json_object.get('size')
1164 return FixedSchema(name, namespace, size, names, other_props)
1165 elif type == ENUM:
1166 symbols = json_object.get('symbols')
1167 doc = json_object.get('doc')
1168 return EnumSchema(name, namespace, symbols, names, doc, other_props)
1169
1170 elif type in [RECORD, ERROR]:
1171 field_desc_list = json_object.get('fields', ())
1172
1173 def MakeFields(names):
1174 return tuple(RecordSchema._MakeFieldList(field_desc_list, names))
1175
1176 return RecordSchema(
1177 name=name,
1178 namespace=namespace,
1179 make_fields=MakeFields,
1180 names=names,
1181 record_type=type,
1182 doc=json_object.get('doc'),
1183 other_props=other_props,
1184 )
1185 else:
1186 raise Exception('Internal error: unknown type %r.' % type)
1187
1188 elif type in VALID_TYPES:
1189 # Unnamed, non-primitive Avro type:
1190
1191 if type == ARRAY:
1192 items_desc = json_object.get('items')
1193 if items_desc is None:
1194 raise SchemaParseException(
1195 'Invalid array schema descriptor with no "items" : %r.'
1196 % json_object)
1197 return ArraySchema(
1198 items=SchemaFromJSONData(items_desc, names),
1199 other_props=other_props,
1200 )
1201
1202 elif type == MAP:
1203 values_desc = json_object.get('values')
1204 if values_desc is None:
1205 raise SchemaParseException(
1206 'Invalid map schema descriptor with no "values" : %r.'
1207 % json_object)
1208 return MapSchema(
1209 values=SchemaFromJSONData(values_desc, names=names),
1210 other_props=other_props,
1211 )
1212
1213 elif type == ERROR_UNION:
1214 error_desc_list = json_object.get('declared_errors')
1215 assert (error_desc_list is not None)
1216 error_schemas = map(
1217 lambda desc: SchemaFromJSONData(desc, names=names),
1218 error_desc_list)
1219 return ErrorUnionSchema(schemas=error_schemas)
1220
1221 else:
1222 raise Exception('Internal error: unknown type %r.' % type)
1223
1224 raise SchemaParseException(
1225 'Invalid JSON descriptor for an Avro schema: %r' % json_object)
1226
1227
1228 # Parsers for the JSON data types:
1229 _JSONDataParserTypeMap = {
1230 str: _SchemaFromJSONString,
1231 list: _SchemaFromJSONArray,
1232 dict: _SchemaFromJSONObject,
1233 }
1234
1235
1236 def SchemaFromJSONData(json_data, names=None):
1237 """Builds an Avro Schema from its JSON descriptor.
1238
1239 Args:
1240 json_data: JSON data representing the descriptor of the Avro schema.
1241 names: Optional tracker for Avro named schemas.
1242 Returns:
1243 The Avro schema parsed from the JSON descriptor.
1244 Raises:
1245 SchemaParseException: if the descriptor is invalid.
1246 """
1247 if names is None:
1248 names = Names()
1249
1250 # Select the appropriate parser based on the JSON data type:
1251 parser = _JSONDataParserTypeMap.get(type(json_data))
1252 if parser is None:
1253 raise SchemaParseException(
1254 'Invalid JSON descriptor for an Avro schema: %r.' % json_data)
1255 return parser(json_data, names=names)
1256
1257
1258 # ------------------------------------------------------------------------------
1259
1260
1261 def Parse(json_string):
1262 """Constructs a Schema from its JSON descriptor in text form.
1263
1264 Args:
1265 json_string: String representation of the JSON descriptor of the schema.
1266 Returns:
1267 The parsed schema.
1268 Raises:
1269 SchemaParseException: on JSON parsing error,
1270 or if the JSON descriptor is invalid.
1271 """
1272 try:
1273 json_data = json.loads(json_string)
1274 except Exception as exn:
1275 raise SchemaParseException(
1276 'Error parsing schema from JSON: %r. '
1277 'Error message: %r.'
1278 % (json_string, exn))
1279
1280 # Initialize the names object
1281 names = Names()
1282
1283 # construct the Avro Schema object
1284 return SchemaFromJSONData(json_data, names)