| 1 |
#!/usr/bin/env python3 |
| 2 |
# -*- mode: python -*- |
| 3 |
# -*- coding: utf-8 -*- |
| 4 |
|
| 5 |
# Licensed to the Apache Software Foundation (ASF) under one |
| 6 |
# or more contributor license agreements. See the NOTICE file |
| 7 |
# distributed with this work for additional information |
| 8 |
# regarding copyright ownership. The ASF licenses this file |
| 9 |
# to you under the Apache License, Version 2.0 (the |
| 10 |
# "License"); you may not use this file except in compliance |
| 11 |
# with the License. You may obtain a copy of the License at |
| 12 |
# |
| 13 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 14 |
# |
| 15 |
# Unless required by applicable law or agreed to in writing, software |
| 16 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 17 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 18 |
# See the License for the specific language governing permissions and |
| 19 |
# limitations under the License. |
| 20 |
|
| 21 |
"""Representation of Avro schemas. |
| 22 |
|
| 23 |
A schema may be one of: |
| 24 |
- A record, mapping field names to field value data; |
| 25 |
- An error, equivalent to a record; |
| 26 |
- An enum, containing one of a small set of symbols; |
| 27 |
- An array of values, all of the same schema; |
| 28 |
- A map containing string/value pairs, each of a declared schema; |
| 29 |
- A union of other schemas; |
| 30 |
- A fixed sized binary object; |
| 31 |
- A unicode string; |
| 32 |
- A sequence of bytes; |
| 33 |
- A 32-bit signed int; |
| 34 |
- A 64-bit signed long; |
| 35 |
- A 32-bit floating-point float; |
| 36 |
- A 64-bit floating-point double; |
| 37 |
- A boolean; |
| 38 |
- Null. |
| 39 |
""" |
| 40 |
|
| 41 |
|
| 42 |
import abc |
| 43 |
import collections |
| 44 |
import json |
| 45 |
import logging |
| 46 |
import re |
| 47 |
|
| 48 |
logger = logging.getLogger(__name__) |
| 49 |
|
| 50 |
# ------------------------------------------------------------------------------ |
| 51 |
# Constants |
| 52 |
|
| 53 |
# Log level more verbose than DEBUG=10, INFO=20, etc. |
| 54 |
DEBUG_VERBOSE=5 |
| 55 |
|
| 56 |
|
| 57 |
NULL = 'null' |
| 58 |
BOOLEAN = 'boolean' |
| 59 |
STRING = 'string' |
| 60 |
BYTES = 'bytes' |
| 61 |
INT = 'int' |
| 62 |
LONG = 'long' |
| 63 |
FLOAT = 'float' |
| 64 |
DOUBLE = 'double' |
| 65 |
FIXED = 'fixed' |
| 66 |
ENUM = 'enum' |
| 67 |
RECORD = 'record' |
| 68 |
ERROR = 'error' |
| 69 |
ARRAY = 'array' |
| 70 |
MAP = 'map' |
| 71 |
UNION = 'union' |
| 72 |
|
| 73 |
# Request and error unions are part of Avro protocols: |
| 74 |
REQUEST = 'request' |
| 75 |
ERROR_UNION = 'error_union' |
| 76 |
|
| 77 |
PRIMITIVE_TYPES = frozenset([ |
| 78 |
NULL, |
| 79 |
BOOLEAN, |
| 80 |
STRING, |
| 81 |
BYTES, |
| 82 |
INT, |
| 83 |
LONG, |
| 84 |
FLOAT, |
| 85 |
DOUBLE, |
| 86 |
]) |
| 87 |
|
| 88 |
NAMED_TYPES = frozenset([ |
| 89 |
FIXED, |
| 90 |
ENUM, |
| 91 |
RECORD, |
| 92 |
ERROR, |
| 93 |
]) |
| 94 |
|
| 95 |
VALID_TYPES = frozenset.union( |
| 96 |
PRIMITIVE_TYPES, |
| 97 |
NAMED_TYPES, |
| 98 |
[ |
| 99 |
ARRAY, |
| 100 |
MAP, |
| 101 |
UNION, |
| 102 |
REQUEST, |
| 103 |
ERROR_UNION, |
| 104 |
], |
| 105 |
) |
| 106 |
|
| 107 |
SCHEMA_RESERVED_PROPS = frozenset([ |
| 108 |
'type', |
| 109 |
'name', |
| 110 |
'namespace', |
| 111 |
'fields', # Record |
| 112 |
'items', # Array |
| 113 |
'size', # Fixed |
| 114 |
'symbols', # Enum |
| 115 |
'values', # Map |
| 116 |
'doc', |
| 117 |
]) |
| 118 |
|
| 119 |
FIELD_RESERVED_PROPS = frozenset([ |
| 120 |
'default', |
| 121 |
'name', |
| 122 |
'doc', |
| 123 |
'order', |
| 124 |
'type', |
| 125 |
]) |
| 126 |
|
| 127 |
VALID_FIELD_SORT_ORDERS = frozenset([ |
| 128 |
'ascending', |
| 129 |
'descending', |
| 130 |
'ignore', |
| 131 |
]) |
| 132 |
|
| 133 |
|
| 134 |
# ------------------------------------------------------------------------------ |
| 135 |
# Exceptions |
| 136 |
|
| 137 |
|
| 138 |
class Error(Exception): |
| 139 |
"""Base class for errors in this module.""" |
| 140 |
pass |
| 141 |
|
| 142 |
|
| 143 |
class AvroException(Error): |
| 144 |
"""Generic Avro schema error.""" |
| 145 |
pass |
| 146 |
|
| 147 |
|
| 148 |
class SchemaParseException(AvroException): |
| 149 |
"""Error while parsing a JSON schema descriptor.""" |
| 150 |
pass |
| 151 |
|
| 152 |
|
| 153 |
# ------------------------------------------------------------------------------ |
| 154 |
|
| 155 |
|
| 156 |
class ImmutableDict(dict): |
| 157 |
"""Dictionary guaranteed immutable. |
| 158 |
|
| 159 |
All mutations raise an exception. |
| 160 |
Behaves exactly as a dict otherwise. |
| 161 |
""" |
| 162 |
|
| 163 |
def __init__(self, items=None, **kwargs): |
| 164 |
if items is not None: |
| 165 |
super(ImmutableDict, self).__init__(items) |
| 166 |
assert (len(kwargs) == 0) |
| 167 |
else: |
| 168 |
super(ImmutableDict, self).__init__(**kwargs) |
| 169 |
|
| 170 |
def __setitem__(self, key, value): |
| 171 |
raise Exception( |
| 172 |
'Attempting to map key %r to value %r in ImmutableDict %r' |
| 173 |
% (key, value, self)) |
| 174 |
|
| 175 |
def __delitem__(self, key): |
| 176 |
raise Exception( |
| 177 |
'Attempting to remove mapping for key %r in ImmutableDict %r' |
| 178 |
% (key, self)) |
| 179 |
|
| 180 |
def clear(self): |
| 181 |
raise Exception('Attempting to clear ImmutableDict %r' % self) |
| 182 |
|
| 183 |
def update(self, items=None, **kwargs): |
| 184 |
raise Exception( |
| 185 |
'Attempting to update ImmutableDict %r with items=%r, kwargs=%r' |
| 186 |
% (self, args, kwargs)) |
| 187 |
|
| 188 |
def pop(self, key, default=None): |
| 189 |
raise Exception( |
| 190 |
'Attempting to pop key %r from ImmutableDict %r' % (key, self)) |
| 191 |
|
| 192 |
def popitem(self): |
| 193 |
raise Exception('Attempting to pop item from ImmutableDict %r' % self) |
| 194 |
|
| 195 |
|
| 196 |
# ------------------------------------------------------------------------------ |
| 197 |
|
| 198 |
|
| 199 |
class Schema(object, metaclass=abc.ABCMeta): |
| 200 |
"""Abstract base class for all Schema classes.""" |
| 201 |
|
| 202 |
def __init__(self, type, other_props=None): |
| 203 |
"""Initializes a new schema object. |
| 204 |
|
| 205 |
Args: |
| 206 |
type: Type of the schema to initialize. |
| 207 |
other_props: Optional dictionary of additional properties. |
| 208 |
""" |
| 209 |
if type not in VALID_TYPES: |
| 210 |
raise SchemaParseException('%r is not a valid Avro type.' % type) |
| 211 |
|
| 212 |
# All properties of this schema, as a map: property name -> property value |
| 213 |
self._props = {} |
| 214 |
|
| 215 |
self._props['type'] = type |
| 216 |
self._type = type |
| 217 |
|
| 218 |
if other_props: |
| 219 |
self._props.update(other_props) |
| 220 |
|
| 221 |
@property |
| 222 |
def name(self): |
| 223 |
"""Returns: the simple name of this schema.""" |
| 224 |
return self._props['name'] |
| 225 |
|
| 226 |
@property |
| 227 |
def fullname(self): |
| 228 |
"""Returns: the fully qualified name of this schema.""" |
| 229 |
# By default, the full name is the simple name. |
| 230 |
# Named schemas override this behavior to include the namespace. |
| 231 |
return self.name |
| 232 |
|
| 233 |
@property |
| 234 |
def namespace(self): |
| 235 |
"""Returns: the namespace this schema belongs to, if any, or None.""" |
| 236 |
return self._props.get('namespace', None) |
| 237 |
|
| 238 |
@property |
| 239 |
def type(self): |
| 240 |
"""Returns: the type of this schema.""" |
| 241 |
return self._type |
| 242 |
|
| 243 |
@property |
| 244 |
def doc(self): |
| 245 |
"""Returns: the documentation associated to this schema, if any, or None.""" |
| 246 |
return self._props.get('doc', None) |
| 247 |
|
| 248 |
@property |
| 249 |
def props(self): |
| 250 |
"""Reports all the properties of this schema. |
| 251 |
|
| 252 |
Includes all properties, reserved and non reserved. |
| 253 |
JSON properties of this schema are directly generated from this dict. |
| 254 |
|
| 255 |
Returns: |
| 256 |
A read-only dictionary of properties associated to this schema. |
| 257 |
""" |
| 258 |
return ImmutableDict(self._props) |
| 259 |
|
| 260 |
@property |
| 261 |
def other_props(self): |
| 262 |
"""Returns: the dictionary of non-reserved properties.""" |
| 263 |
return dict(FilterKeysOut(items=self._props, keys=SCHEMA_RESERVED_PROPS)) |
| 264 |
|
| 265 |
def __str__(self): |
| 266 |
"""Returns: the JSON representation of this schema.""" |
| 267 |
return json.dumps(self.to_json()) |
| 268 |
|
| 269 |
@abc.abstractmethod |
| 270 |
def to_json(self, names): |
| 271 |
"""Converts the schema object into its AVRO specification representation. |
| 272 |
|
| 273 |
Schema types that have names (records, enums, and fixed) must |
| 274 |
be aware of not re-defining schemas that are already listed |
| 275 |
in the parameter names. |
| 276 |
""" |
| 277 |
raise Exception('Cannot run abstract method.') |
| 278 |
|
| 279 |
|
| 280 |
# ------------------------------------------------------------------------------ |
| 281 |
|
| 282 |
|
| 283 |
_RE_NAME = re.compile(r'[A-Za-z_][A-Za-z0-9_]*') |
| 284 |
|
| 285 |
_RE_FULL_NAME = re.compile( |
| 286 |
r'^' |
| 287 |
r'[.]?(?:[A-Za-z_][A-Za-z0-9_]*[.])*' # optional namespace |
| 288 |
r'([A-Za-z_][A-Za-z0-9_]*)' # name |
| 289 |
r'$' |
| 290 |
) |
| 291 |
|
| 292 |
class Name(object): |
| 293 |
"""Representation of an Avro name.""" |
| 294 |
|
| 295 |
def __init__(self, name, namespace=None): |
| 296 |
"""Parses an Avro name. |
| 297 |
|
| 298 |
Args: |
| 299 |
name: Avro name to parse (relative or absolute). |
| 300 |
namespace: Optional explicit namespace if the name is relative. |
| 301 |
""" |
| 302 |
# Normalize: namespace is always defined as a string, possibly empty. |
| 303 |
if namespace is None: namespace = '' |
| 304 |
|
| 305 |
if '.' in name: |
| 306 |
# name is absolute, namespace is ignored: |
| 307 |
self._fullname = name |
| 308 |
|
| 309 |
match = _RE_FULL_NAME.match(self._fullname) |
| 310 |
if match is None: |
| 311 |
raise SchemaParseException( |
| 312 |
'Invalid absolute schema name: %r.' % self._fullname) |
| 313 |
|
| 314 |
self._name = match.group(1) |
| 315 |
self._namespace = self._fullname[:-(len(self._name) + 1)] |
| 316 |
|
| 317 |
else: |
| 318 |
# name is relative, combine with explicit namespace: |
| 319 |
self._name = name |
| 320 |
self._namespace = namespace |
| 321 |
self._fullname = '%s.%s' % (self._namespace, self._name) |
| 322 |
|
| 323 |
# Validate the fullname: |
| 324 |
if _RE_FULL_NAME.match(self._fullname) is None: |
| 325 |
raise SchemaParseException( |
| 326 |
'Invalid schema name %r infered from name %r and namespace %r.' |
| 327 |
% (self._fullname, self._name, self._namespace)) |
| 328 |
|
| 329 |
def __eq__(self, other): |
| 330 |
if not isinstance(other, Name): |
| 331 |
return False |
| 332 |
return (self.fullname == other.fullname) |
| 333 |
|
| 334 |
@property |
| 335 |
def simple_name(self): |
| 336 |
"""Returns: the simple name part of this name.""" |
| 337 |
return self._name |
| 338 |
|
| 339 |
@property |
| 340 |
def namespace(self): |
| 341 |
"""Returns: this name's namespace, possible the empty string.""" |
| 342 |
return self._namespace |
| 343 |
|
| 344 |
@property |
| 345 |
def fullname(self): |
| 346 |
"""Returns: the full name (always contains a period '.').""" |
| 347 |
return self._fullname |
| 348 |
|
| 349 |
|
| 350 |
# ------------------------------------------------------------------------------ |
| 351 |
|
| 352 |
|
| 353 |
class Names(object): |
| 354 |
"""Tracks Avro named schemas and default namespace during parsing.""" |
| 355 |
|
| 356 |
def __init__(self, default_namespace=None, names=None): |
| 357 |
"""Initializes a new name tracker. |
| 358 |
|
| 359 |
Args: |
| 360 |
default_namespace: Optional default namespace. |
| 361 |
names: Optional initial mapping of known named schemas. |
| 362 |
""" |
| 363 |
if names is None: |
| 364 |
names = {} |
| 365 |
self._names = names |
| 366 |
self._default_namespace = default_namespace |
| 367 |
|
| 368 |
@property |
| 369 |
def names(self): |
| 370 |
"""Returns: the mapping of known named schemas.""" |
| 371 |
return self._names |
| 372 |
|
| 373 |
@property |
| 374 |
def default_namespace(self): |
| 375 |
"""Returns: the default namespace, if any, or None.""" |
| 376 |
return self._default_namespace |
| 377 |
|
| 378 |
def NewWithDefaultNamespace(self, namespace): |
| 379 |
"""Creates a new name tracker from this tracker, but with a new default ns. |
| 380 |
|
| 381 |
Args: |
| 382 |
namespace: New default namespace to use. |
| 383 |
Returns: |
| 384 |
New name tracker with the specified default namespace. |
| 385 |
""" |
| 386 |
return Names(names=self._names, default_namespace=namespace) |
| 387 |
|
| 388 |
def GetName(self, name, namespace=None): |
| 389 |
"""Resolves the Avro name according to this name tracker's state. |
| 390 |
|
| 391 |
Args: |
| 392 |
name: Name to resolve (absolute or relative). |
| 393 |
namespace: Optional explicit namespace. |
| 394 |
Returns: |
| 395 |
The specified name, resolved according to this tracker. |
| 396 |
""" |
| 397 |
if namespace is None: namespace = self._default_namespace |
| 398 |
return Name(name=name, namespace=namespace) |
| 399 |
|
| 400 |
def has_name(self, name, namespace=None): |
| 401 |
avro_name = self.GetName(name=name, namespace=namespace) |
| 402 |
return avro_name.fullname in self._names |
| 403 |
|
| 404 |
def get_name(self, name, namespace=None): |
| 405 |
avro_name = self.GetName(name=name, namespace=namespace) |
| 406 |
return self._names.get(avro_name.fullname, None) |
| 407 |
|
| 408 |
def GetSchema(self, name, namespace=None): |
| 409 |
"""Resolves an Avro schema by name. |
| 410 |
|
| 411 |
Args: |
| 412 |
name: Name (relative or absolute) of the Avro schema to look up. |
| 413 |
namespace: Optional explicit namespace. |
| 414 |
Returns: |
| 415 |
The schema with the specified name, if any, or None. |
| 416 |
""" |
| 417 |
avro_name = self.GetName(name=name, namespace=namespace) |
| 418 |
return self._names.get(avro_name.fullname, None) |
| 419 |
|
| 420 |
def prune_namespace(self, properties): |
| 421 |
"""given a properties, return properties with namespace removed if |
| 422 |
it matches the own default namespace |
| 423 |
""" |
| 424 |
if self.default_namespace is None: |
| 425 |
# I have no default -- no change |
| 426 |
return properties |
| 427 |
if 'namespace' not in properties: |
| 428 |
# he has no namespace - no change |
| 429 |
return properties |
| 430 |
if properties['namespace'] != self.default_namespace: |
| 431 |
# we're different - leave his stuff alone |
| 432 |
return properties |
| 433 |
# we each have a namespace and it's redundant. delete his. |
| 434 |
prunable = properties.copy() |
| 435 |
del(prunable['namespace']) |
| 436 |
return prunable |
| 437 |
|
| 438 |
def Register(self, schema): |
| 439 |
"""Registers a new named schema in this tracker. |
| 440 |
|
| 441 |
Args: |
| 442 |
schema: Named Avro schema to register in this tracker. |
| 443 |
""" |
| 444 |
if schema.fullname in VALID_TYPES: |
| 445 |
raise SchemaParseException( |
| 446 |
'%s is a reserved type name.' % schema.fullname) |
| 447 |
if schema.fullname in self.names: |
| 448 |
raise SchemaParseException( |
| 449 |
'Avro name %r already exists.' % schema.fullname) |
| 450 |
|
| 451 |
logger.log(DEBUG_VERBOSE, 'Register new name for %r', schema.fullname) |
| 452 |
self._names[schema.fullname] = schema |
| 453 |
|
| 454 |
|
| 455 |
# ------------------------------------------------------------------------------ |
| 456 |
|
| 457 |
|
| 458 |
class NamedSchema(Schema): |
| 459 |
"""Abstract base class for named schemas. |
| 460 |
|
| 461 |
Named schemas are enumerated in NAMED_TYPES. |
| 462 |
""" |
| 463 |
|
| 464 |
def __init__( |
| 465 |
self, |
| 466 |
type, |
| 467 |
name, |
| 468 |
namespace=None, |
| 469 |
names=None, |
| 470 |
other_props=None, |
| 471 |
): |
| 472 |
"""Initializes a new named schema object. |
| 473 |
|
| 474 |
Args: |
| 475 |
type: Type of the named schema. |
| 476 |
name: Name (absolute or relative) of the schema. |
| 477 |
namespace: Optional explicit namespace if name is relative. |
| 478 |
names: Tracker to resolve and register Avro names. |
| 479 |
other_props: Optional map of additional properties of the schema. |
| 480 |
""" |
| 481 |
assert (type in NAMED_TYPES), ('Invalid named type: %r' % type) |
| 482 |
self._avro_name = names.GetName(name=name, namespace=namespace) |
| 483 |
|
| 484 |
super(NamedSchema, self).__init__(type, other_props) |
| 485 |
|
| 486 |
names.Register(self) |
| 487 |
|
| 488 |
self._props['name'] = self.name |
| 489 |
if self.namespace: |
| 490 |
self._props['namespace'] = self.namespace |
| 491 |
|
| 492 |
@property |
| 493 |
def avro_name(self): |
| 494 |
"""Returns: the Name object describing this schema's name.""" |
| 495 |
return self._avro_name |
| 496 |
|
| 497 |
@property |
| 498 |
def name(self): |
| 499 |
return self._avro_name.simple_name |
| 500 |
|
| 501 |
@property |
| 502 |
def namespace(self): |
| 503 |
return self._avro_name.namespace |
| 504 |
|
| 505 |
@property |
| 506 |
def fullname(self): |
| 507 |
return self._avro_name.fullname |
| 508 |
|
| 509 |
def name_ref(self, names): |
| 510 |
"""Reports this schema name relative to the specified name tracker. |
| 511 |
|
| 512 |
Args: |
| 513 |
names: Avro name tracker to relativise this schema name against. |
| 514 |
Returns: |
| 515 |
This schema name, relativised against the specified name tracker. |
| 516 |
""" |
| 517 |
if self.namespace == names.default_namespace: |
| 518 |
return self.name |
| 519 |
else: |
| 520 |
return self.fullname |
| 521 |
|
| 522 |
|
| 523 |
# ------------------------------------------------------------------------------ |
| 524 |
|
| 525 |
|
| 526 |
_NO_DEFAULT = object() |
| 527 |
|
| 528 |
|
| 529 |
class Field(object): |
| 530 |
"""Representation of the schema of a field in a record.""" |
| 531 |
|
| 532 |
def __init__( |
| 533 |
self, |
| 534 |
type, |
| 535 |
name, |
| 536 |
index, |
| 537 |
has_default, |
| 538 |
default=_NO_DEFAULT, |
| 539 |
order=None, |
| 540 |
names=None, |
| 541 |
doc=None, |
| 542 |
other_props=None |
| 543 |
): |
| 544 |
"""Initializes a new Field object. |
| 545 |
|
| 546 |
Args: |
| 547 |
type: Avro schema of the field. |
| 548 |
name: Name of the field. |
| 549 |
index: 0-based position of the field. |
| 550 |
has_default: |
| 551 |
default: |
| 552 |
order: |
| 553 |
names: |
| 554 |
doc: |
| 555 |
other_props: |
| 556 |
""" |
| 557 |
if (not isinstance(name, str)) or (len(name) == 0): |
| 558 |
raise SchemaParseException('Invalid record field name: %r.' % name) |
| 559 |
if (order is not None) and (order not in VALID_FIELD_SORT_ORDERS): |
| 560 |
raise SchemaParseException('Invalid record field order: %r.' % order) |
| 561 |
|
| 562 |
# All properties of this record field: |
| 563 |
self._props = {} |
| 564 |
|
| 565 |
self._has_default = has_default |
| 566 |
if other_props: |
| 567 |
self._props.update(other_props) |
| 568 |
|
| 569 |
self._index = index |
| 570 |
self._type = self._props['type'] = type |
| 571 |
self._name = self._props['name'] = name |
| 572 |
|
| 573 |
# TODO: check to ensure default is valid |
| 574 |
if has_default: |
| 575 |
self._props['default'] = default |
| 576 |
|
| 577 |
if order is not None: |
| 578 |
self._props['order'] = order |
| 579 |
|
| 580 |
if doc is not None: |
| 581 |
self._props['doc'] = doc |
| 582 |
|
| 583 |
@property |
| 584 |
def type(self): |
| 585 |
"""Returns: the schema of this field.""" |
| 586 |
return self._type |
| 587 |
|
| 588 |
@property |
| 589 |
def name(self): |
| 590 |
"""Returns: this field name.""" |
| 591 |
return self._name |
| 592 |
|
| 593 |
@property |
| 594 |
def index(self): |
| 595 |
"""Returns: the 0-based index of this field in the record.""" |
| 596 |
return self._index |
| 597 |
|
| 598 |
@property |
| 599 |
def default(self): |
| 600 |
return self._props['default'] |
| 601 |
|
| 602 |
@property |
| 603 |
def has_default(self): |
| 604 |
return self._has_default |
| 605 |
|
| 606 |
@property |
| 607 |
def order(self): |
| 608 |
return self._props.get('order', None) |
| 609 |
|
| 610 |
@property |
| 611 |
def doc(self): |
| 612 |
return self._props.get('doc', None) |
| 613 |
|
| 614 |
@property |
| 615 |
def props(self): |
| 616 |
return self._props |
| 617 |
|
| 618 |
@property |
| 619 |
def other_props(self): |
| 620 |
return FilterKeysOut(items=self._props, keys=FIELD_RESERVED_PROPS) |
| 621 |
|
| 622 |
def __str__(self): |
| 623 |
return json.dumps(self.to_json()) |
| 624 |
|
| 625 |
def to_json(self, names=None): |
| 626 |
if names is None: |
| 627 |
names = Names() |
| 628 |
to_dump = self.props.copy() |
| 629 |
to_dump['type'] = self.type.to_json(names) |
| 630 |
return to_dump |
| 631 |
|
| 632 |
def __eq__(self, that): |
| 633 |
to_cmp = json.loads(str(self)) |
| 634 |
return to_cmp == json.loads(str(that)) |
| 635 |
|
| 636 |
|
| 637 |
# ------------------------------------------------------------------------------ |
| 638 |
# Primitive Types |
| 639 |
|
| 640 |
|
| 641 |
class PrimitiveSchema(Schema): |
| 642 |
"""Schema of a primitive Avro type. |
| 643 |
|
| 644 |
Valid primitive types are defined in PRIMITIVE_TYPES. |
| 645 |
""" |
| 646 |
|
| 647 |
def __init__(self, type, other_props=None): |
| 648 |
"""Initializes a new schema object for the specified primitive type. |
| 649 |
|
| 650 |
Args: |
| 651 |
type: Type of the schema to construct. Must be primitive. |
| 652 |
""" |
| 653 |
if type not in PRIMITIVE_TYPES: |
| 654 |
raise AvroException('%r is not a valid primitive type.' % type) |
| 655 |
super(PrimitiveSchema, self).__init__(type, other_props=other_props) |
| 656 |
|
| 657 |
@property |
| 658 |
def name(self): |
| 659 |
"""Returns: the simple name of this schema.""" |
| 660 |
# The name of a primitive type is the type itself. |
| 661 |
return self.type |
| 662 |
|
| 663 |
def to_json(self, names=None): |
| 664 |
if len(self.props) == 1: |
| 665 |
return self.fullname |
| 666 |
else: |
| 667 |
return self.props |
| 668 |
|
| 669 |
def __eq__(self, that): |
| 670 |
return self.props == that.props |
| 671 |
|
| 672 |
|
| 673 |
# ------------------------------------------------------------------------------ |
| 674 |
# Complex Types (non-recursive) |
| 675 |
|
| 676 |
|
| 677 |
class FixedSchema(NamedSchema): |
| 678 |
def __init__( |
| 679 |
self, |
| 680 |
name, |
| 681 |
namespace, |
| 682 |
size, |
| 683 |
names=None, |
| 684 |
other_props=None, |
| 685 |
): |
| 686 |
# Ensure valid ctor args |
| 687 |
if not isinstance(size, int): |
| 688 |
fail_msg = 'Fixed Schema requires a valid integer for size property.' |
| 689 |
raise AvroException(fail_msg) |
| 690 |
|
| 691 |
super(FixedSchema, self).__init__( |
| 692 |
type=FIXED, |
| 693 |
name=name, |
| 694 |
namespace=namespace, |
| 695 |
names=names, |
| 696 |
other_props=other_props, |
| 697 |
) |
| 698 |
self._props['size'] = size |
| 699 |
|
| 700 |
@property |
| 701 |
def size(self): |
| 702 |
"""Returns: the size of this fixed schema, in bytes.""" |
| 703 |
return self._props['size'] |
| 704 |
|
| 705 |
def to_json(self, names=None): |
| 706 |
if names is None: |
| 707 |
names = Names() |
| 708 |
if self.fullname in names.names: |
| 709 |
return self.name_ref(names) |
| 710 |
else: |
| 711 |
names.names[self.fullname] = self |
| 712 |
return names.prune_namespace(self.props) |
| 713 |
|
| 714 |
def __eq__(self, that): |
| 715 |
return self.props == that.props |
| 716 |
|
| 717 |
|
| 718 |
# ------------------------------------------------------------------------------ |
| 719 |
|
| 720 |
|
| 721 |
class EnumSchema(NamedSchema): |
| 722 |
def __init__( |
| 723 |
self, |
| 724 |
name, |
| 725 |
namespace, |
| 726 |
symbols, |
| 727 |
names=None, |
| 728 |
doc=None, |
| 729 |
other_props=None, |
| 730 |
): |
| 731 |
"""Initializes a new enumeration schema object. |
| 732 |
|
| 733 |
Args: |
| 734 |
name: Simple name of this enumeration. |
| 735 |
namespace: Optional namespace. |
| 736 |
symbols: Ordered list of symbols defined in this enumeration. |
| 737 |
names: |
| 738 |
doc: |
| 739 |
other_props: |
| 740 |
""" |
| 741 |
symbols = tuple(symbols) |
| 742 |
symbol_set = frozenset(symbols) |
| 743 |
if (len(symbol_set) != len(symbols) |
| 744 |
or not all(map(lambda symbol: isinstance(symbol, str), symbols))): |
| 745 |
raise AvroException( |
| 746 |
'Invalid symbols for enum schema: %r.' % (symbols,)) |
| 747 |
|
| 748 |
super(EnumSchema, self).__init__( |
| 749 |
type=ENUM, |
| 750 |
name=name, |
| 751 |
namespace=namespace, |
| 752 |
names=names, |
| 753 |
other_props=other_props, |
| 754 |
) |
| 755 |
|
| 756 |
self._props['symbols'] = symbols |
| 757 |
if doc is not None: |
| 758 |
self._props['doc'] = doc |
| 759 |
|
| 760 |
@property |
| 761 |
def symbols(self): |
| 762 |
"""Returns: the symbols defined in this enum.""" |
| 763 |
return self._props['symbols'] |
| 764 |
|
| 765 |
def to_json(self, names=None): |
| 766 |
if names is None: |
| 767 |
names = Names() |
| 768 |
if self.fullname in names.names: |
| 769 |
return self.name_ref(names) |
| 770 |
else: |
| 771 |
names.names[self.fullname] = self |
| 772 |
return names.prune_namespace(self.props) |
| 773 |
|
| 774 |
def __eq__(self, that): |
| 775 |
return self.props == that.props |
| 776 |
|
| 777 |
|
| 778 |
# ------------------------------------------------------------------------------ |
| 779 |
# Complex Types (recursive) |
| 780 |
|
| 781 |
|
| 782 |
class ArraySchema(Schema): |
| 783 |
"""Schema of an array.""" |
| 784 |
|
| 785 |
def __init__(self, items, other_props=None): |
| 786 |
"""Initializes a new array schema object. |
| 787 |
|
| 788 |
Args: |
| 789 |
items: Avro schema of the array items. |
| 790 |
other_props: |
| 791 |
""" |
| 792 |
super(ArraySchema, self).__init__( |
| 793 |
type=ARRAY, |
| 794 |
other_props=other_props, |
| 795 |
) |
| 796 |
self._items_schema = items |
| 797 |
self._props['items'] = items |
| 798 |
|
| 799 |
@property |
| 800 |
def items(self): |
| 801 |
"""Returns: the schema of the items in this array.""" |
| 802 |
return self._items_schema |
| 803 |
|
| 804 |
def to_json(self, names=None): |
| 805 |
if names is None: |
| 806 |
names = Names() |
| 807 |
to_dump = self.props.copy() |
| 808 |
item_schema = self.items |
| 809 |
to_dump['items'] = item_schema.to_json(names) |
| 810 |
return to_dump |
| 811 |
|
| 812 |
def __eq__(self, that): |
| 813 |
to_cmp = json.loads(str(self)) |
| 814 |
return to_cmp == json.loads(str(that)) |
| 815 |
|
| 816 |
|
| 817 |
# ------------------------------------------------------------------------------ |
| 818 |
|
| 819 |
|
| 820 |
class MapSchema(Schema): |
| 821 |
"""Schema of a map.""" |
| 822 |
|
| 823 |
def __init__(self, values, other_props=None): |
| 824 |
"""Initializes a new map schema object. |
| 825 |
|
| 826 |
Args: |
| 827 |
values: Avro schema of the map values. |
| 828 |
other_props: |
| 829 |
""" |
| 830 |
super(MapSchema, self).__init__( |
| 831 |
type=MAP, |
| 832 |
other_props=other_props, |
| 833 |
) |
| 834 |
self._values_schema = values |
| 835 |
self._props['values'] = values |
| 836 |
|
| 837 |
@property |
| 838 |
def values(self): |
| 839 |
"""Returns: the schema of the values in this map.""" |
| 840 |
return self._values_schema |
| 841 |
|
| 842 |
def to_json(self, names=None): |
| 843 |
if names is None: |
| 844 |
names = Names() |
| 845 |
to_dump = self.props.copy() |
| 846 |
to_dump['values'] = self.values.to_json(names) |
| 847 |
return to_dump |
| 848 |
|
| 849 |
def __eq__(self, that): |
| 850 |
to_cmp = json.loads(str(self)) |
| 851 |
return to_cmp == json.loads(str(that)) |
| 852 |
|
| 853 |
|
| 854 |
# ------------------------------------------------------------------------------ |
| 855 |
|
| 856 |
|
| 857 |
class UnionSchema(Schema): |
| 858 |
"""Schema of a union.""" |
| 859 |
|
| 860 |
def __init__(self, schemas): |
| 861 |
"""Initializes a new union schema object. |
| 862 |
|
| 863 |
Args: |
| 864 |
schemas: Ordered collection of schema branches in the union. |
| 865 |
""" |
| 866 |
super(UnionSchema, self).__init__(type=UNION) |
| 867 |
self._schemas = tuple(schemas) |
| 868 |
|
| 869 |
# Validate the schema branches: |
| 870 |
|
| 871 |
# All named schema names are unique: |
| 872 |
named_branches = tuple( |
| 873 |
filter(lambda schema: schema.type in NAMED_TYPES, self._schemas)) |
| 874 |
unique_names = frozenset(map(lambda schema: schema.fullname, named_branches)) |
| 875 |
if len(unique_names) != len(named_branches): |
| 876 |
raise AvroException( |
| 877 |
'Invalid union branches with duplicate schema name:%s' |
| 878 |
% ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) |
| 879 |
|
| 880 |
# Types are unique within unnamed schemas, and union is not allowed: |
| 881 |
unnamed_branches = tuple( |
| 882 |
filter(lambda schema: schema.type not in NAMED_TYPES, self._schemas)) |
| 883 |
unique_types = frozenset(map(lambda schema: schema.type, unnamed_branches)) |
| 884 |
if UNION in unique_types: |
| 885 |
raise AvroException( |
| 886 |
'Invalid union branches contain other unions:%s' |
| 887 |
% ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) |
| 888 |
if len(unique_types) != len(unnamed_branches): |
| 889 |
raise AvroException( |
| 890 |
'Invalid union branches with duplicate type:%s' |
| 891 |
% ''.join(map(lambda schema: ('\n\t - %s' % schema), self._schemas))) |
| 892 |
|
| 893 |
@property |
| 894 |
def schemas(self): |
| 895 |
"""Returns: the ordered list of schema branches in the union.""" |
| 896 |
return self._schemas |
| 897 |
|
| 898 |
def to_json(self, names=None): |
| 899 |
if names is None: |
| 900 |
names = Names() |
| 901 |
to_dump = [] |
| 902 |
for schema in self.schemas: |
| 903 |
to_dump.append(schema.to_json(names)) |
| 904 |
return to_dump |
| 905 |
|
| 906 |
def __eq__(self, that): |
| 907 |
to_cmp = json.loads(str(self)) |
| 908 |
return to_cmp == json.loads(str(that)) |
| 909 |
|
| 910 |
|
| 911 |
# ------------------------------------------------------------------------------ |
| 912 |
|
| 913 |
|
| 914 |
class ErrorUnionSchema(UnionSchema): |
| 915 |
"""Schema representing the declared errors of a protocol message.""" |
| 916 |
|
| 917 |
def __init__(self, schemas): |
| 918 |
"""Initializes an error-union schema. |
| 919 |
|
| 920 |
Args: |
| 921 |
schema: collection of error schema. |
| 922 |
""" |
| 923 |
# TODO: check that string isn't already listed explicitly as an error. |
| 924 |
# Prepend "string" to handle system errors |
| 925 |
schemas = [PrimitiveSchema(type=STRING)] + list(schemas) |
| 926 |
super(ErrorUnionSchema, self).__init__(schemas=schemas) |
| 927 |
|
| 928 |
def to_json(self, names=None): |
| 929 |
if names is None: |
| 930 |
names = Names() |
| 931 |
to_dump = [] |
| 932 |
for schema in self.schemas: |
| 933 |
# Don't print the system error schema |
| 934 |
if schema.type == STRING: continue |
| 935 |
to_dump.append(schema.to_json(names)) |
| 936 |
return to_dump |
| 937 |
|
| 938 |
|
| 939 |
# ------------------------------------------------------------------------------ |
| 940 |
|
| 941 |
|
| 942 |
class RecordSchema(NamedSchema): |
| 943 |
"""Schema of a record.""" |
| 944 |
|
| 945 |
@staticmethod |
| 946 |
def _MakeField(index, field_desc, names): |
| 947 |
"""Builds field schemas from a list of field JSON descriptors. |
| 948 |
|
| 949 |
Args: |
| 950 |
index: 0-based index of the field in the record. |
| 951 |
field_desc: JSON descriptors of a record field. |
| 952 |
names: Avro schema tracker. |
| 953 |
Return: |
| 954 |
The field schema. |
| 955 |
""" |
| 956 |
field_schema = SchemaFromJSONData( |
| 957 |
json_data=field_desc['type'], |
| 958 |
names=names, |
| 959 |
) |
| 960 |
other_props = ( |
| 961 |
dict(FilterKeysOut(items=field_desc, keys=FIELD_RESERVED_PROPS))) |
| 962 |
return Field( |
| 963 |
type=field_schema, |
| 964 |
name=field_desc['name'], |
| 965 |
index=index, |
| 966 |
has_default=('default' in field_desc), |
| 967 |
default=field_desc.get('default', _NO_DEFAULT), |
| 968 |
order=field_desc.get('order', None), |
| 969 |
names=names, |
| 970 |
doc=field_desc.get('doc', None), |
| 971 |
other_props=other_props, |
| 972 |
) |
| 973 |
|
| 974 |
@staticmethod |
| 975 |
def _MakeFieldList(field_desc_list, names): |
| 976 |
"""Builds field schemas from a list of field JSON descriptors. |
| 977 |
|
| 978 |
Guarantees field name unicity. |
| 979 |
|
| 980 |
Args: |
| 981 |
field_desc_list: collection of field JSON descriptors. |
| 982 |
names: Avro schema tracker. |
| 983 |
Yields |
| 984 |
Field schemas. |
| 985 |
""" |
| 986 |
for index, field_desc in enumerate(field_desc_list): |
| 987 |
yield RecordSchema._MakeField(index, field_desc, names) |
| 988 |
|
| 989 |
@staticmethod |
| 990 |
def _MakeFieldMap(fields): |
| 991 |
"""Builds the field map. |
| 992 |
|
| 993 |
Guarantees field name unicity. |
| 994 |
|
| 995 |
Args: |
| 996 |
fields: iterable of field schema. |
| 997 |
Returns: |
| 998 |
A read-only map of field schemas, indexed by name. |
| 999 |
""" |
| 1000 |
field_map = {} |
| 1001 |
for field in fields: |
| 1002 |
if field.name in field_map: |
| 1003 |
raise SchemaParseException( |
| 1004 |
'Duplicate field name %r in list %r.' % (field.name, field_desc_list)) |
| 1005 |
field_map[field.name] = field |
| 1006 |
return ImmutableDict(field_map) |
| 1007 |
|
| 1008 |
def __init__( |
| 1009 |
self, |
| 1010 |
name, |
| 1011 |
namespace, |
| 1012 |
fields=None, |
| 1013 |
make_fields=None, |
| 1014 |
names=None, |
| 1015 |
record_type=RECORD, |
| 1016 |
doc=None, |
| 1017 |
other_props=None |
| 1018 |
): |
| 1019 |
"""Initializes a new record schema object. |
| 1020 |
|
| 1021 |
Args: |
| 1022 |
name: Name of the record (absolute or relative). |
| 1023 |
namespace: Optional namespace the record belongs to, if name is relative. |
| 1024 |
fields: collection of fields to add to this record. |
| 1025 |
Exactly one of fields or make_fields must be specified. |
| 1026 |
make_fields: function creating the fields that belong to the record. |
| 1027 |
The function signature is: make_fields(names) -> ordered field list. |
| 1028 |
Exactly one of fields or make_fields must be specified. |
| 1029 |
names: |
| 1030 |
record_type: Type of the record: one of RECORD, ERROR or REQUEST. |
| 1031 |
Protocol requests are not named. |
| 1032 |
doc: |
| 1033 |
other_props: |
| 1034 |
""" |
| 1035 |
if record_type == REQUEST: |
| 1036 |
# Protocol requests are not named: |
| 1037 |
super(NamedSchema, self).__init__( |
| 1038 |
type=REQUEST, |
| 1039 |
other_props=other_props, |
| 1040 |
) |
| 1041 |
elif record_type in [RECORD, ERROR]: |
| 1042 |
# Register this record name in the tracker: |
| 1043 |
super(RecordSchema, self).__init__( |
| 1044 |
type=record_type, |
| 1045 |
name=name, |
| 1046 |
namespace=namespace, |
| 1047 |
names=names, |
| 1048 |
other_props=other_props, |
| 1049 |
) |
| 1050 |
else: |
| 1051 |
raise SchemaParseException( |
| 1052 |
'Invalid record type: %r.' % record_type) |
| 1053 |
|
| 1054 |
if record_type in [RECORD, ERROR]: |
| 1055 |
avro_name = names.GetName(name=name, namespace=namespace) |
| 1056 |
nested_names = names.NewWithDefaultNamespace(namespace=avro_name.namespace) |
| 1057 |
elif record_type == REQUEST: |
| 1058 |
# Protocol request has no name: no need to change default namespace: |
| 1059 |
nested_names = names |
| 1060 |
|
| 1061 |
if fields is None: |
| 1062 |
fields = make_fields(names=nested_names) |
| 1063 |
else: |
| 1064 |
assert (make_fields is None) |
| 1065 |
self._fields = tuple(fields) |
| 1066 |
|
| 1067 |
self._field_map = RecordSchema._MakeFieldMap(self._fields) |
| 1068 |
|
| 1069 |
self._props['fields'] = fields |
| 1070 |
if doc is not None: |
| 1071 |
self._props['doc'] = doc |
| 1072 |
|
| 1073 |
@property |
| 1074 |
def fields(self): |
| 1075 |
"""Returns: the field schemas, as an ordered tuple.""" |
| 1076 |
return self._fields |
| 1077 |
|
| 1078 |
@property |
| 1079 |
def field_map(self): |
| 1080 |
"""Returns: a read-only map of the field schemas index by field names.""" |
| 1081 |
return self._field_map |
| 1082 |
|
| 1083 |
def to_json(self, names=None): |
| 1084 |
if names is None: |
| 1085 |
names = Names() |
| 1086 |
# Request records don't have names |
| 1087 |
if self.type == REQUEST: |
| 1088 |
return [f.to_json(names) for f in self.fields] |
| 1089 |
|
| 1090 |
if self.fullname in names.names: |
| 1091 |
return self.name_ref(names) |
| 1092 |
else: |
| 1093 |
names.names[self.fullname] = self |
| 1094 |
|
| 1095 |
to_dump = names.prune_namespace(self.props.copy()) |
| 1096 |
to_dump['fields'] = [f.to_json(names) for f in self.fields] |
| 1097 |
return to_dump |
| 1098 |
|
| 1099 |
def __eq__(self, that): |
| 1100 |
to_cmp = json.loads(str(self)) |
| 1101 |
return to_cmp == json.loads(str(that)) |
| 1102 |
|
| 1103 |
|
| 1104 |
# ------------------------------------------------------------------------------ |
| 1105 |
# Module functions |
| 1106 |
|
| 1107 |
|
| 1108 |
def FilterKeysOut(items, keys): |
| 1109 |
"""Filters a collection of (key, value) items. |
| 1110 |
|
| 1111 |
Exclude any item whose key belongs to keys. |
| 1112 |
|
| 1113 |
Args: |
| 1114 |
items: Dictionary of items to filter the keys out of. |
| 1115 |
keys: Keys to filter out. |
| 1116 |
Yields: |
| 1117 |
Filtered items. |
| 1118 |
""" |
| 1119 |
for key, value in items.items(): |
| 1120 |
if key in keys: continue |
| 1121 |
yield (key, value) |
| 1122 |
|
| 1123 |
|
| 1124 |
# ------------------------------------------------------------------------------ |
| 1125 |
|
| 1126 |
|
| 1127 |
def _SchemaFromJSONString(json_string, names): |
| 1128 |
if json_string in PRIMITIVE_TYPES: |
| 1129 |
return PrimitiveSchema(type=json_string) |
| 1130 |
else: |
| 1131 |
# Look for a known named schema: |
| 1132 |
schema = names.GetSchema(name=json_string) |
| 1133 |
if schema is None: |
| 1134 |
raise SchemaParseException( |
| 1135 |
'Unknown named schema %r, known names: %r.' |
| 1136 |
% (json_string, sorted(names.names))) |
| 1137 |
return schema |
| 1138 |
|
| 1139 |
|
| 1140 |
def _SchemaFromJSONArray(json_array, names): |
| 1141 |
def MakeSchema(desc): |
| 1142 |
return SchemaFromJSONData(json_data=desc, names=names) |
| 1143 |
return UnionSchema(map(MakeSchema, json_array)) |
| 1144 |
|
| 1145 |
|
| 1146 |
def _SchemaFromJSONObject(json_object, names): |
| 1147 |
type = json_object.get('type') |
| 1148 |
if type is None: |
| 1149 |
raise SchemaParseException( |
| 1150 |
'Avro schema JSON descriptor has no "type" property: %r' % json_object) |
| 1151 |
|
| 1152 |
other_props = dict( |
| 1153 |
FilterKeysOut(items=json_object, keys=SCHEMA_RESERVED_PROPS)) |
| 1154 |
|
| 1155 |
if type in PRIMITIVE_TYPES: |
| 1156 |
# FIXME should not ignore other properties |
| 1157 |
return PrimitiveSchema(type, other_props=other_props) |
| 1158 |
|
| 1159 |
elif type in NAMED_TYPES: |
| 1160 |
name = json_object.get('name') |
| 1161 |
namespace = json_object.get('namespace', names.default_namespace) |
| 1162 |
if type == FIXED: |
| 1163 |
size = json_object.get('size') |
| 1164 |
return FixedSchema(name, namespace, size, names, other_props) |
| 1165 |
elif type == ENUM: |
| 1166 |
symbols = json_object.get('symbols') |
| 1167 |
doc = json_object.get('doc') |
| 1168 |
return EnumSchema(name, namespace, symbols, names, doc, other_props) |
| 1169 |
|
| 1170 |
elif type in [RECORD, ERROR]: |
| 1171 |
field_desc_list = json_object.get('fields', ()) |
| 1172 |
|
| 1173 |
def MakeFields(names): |
| 1174 |
return tuple(RecordSchema._MakeFieldList(field_desc_list, names)) |
| 1175 |
|
| 1176 |
return RecordSchema( |
| 1177 |
name=name, |
| 1178 |
namespace=namespace, |
| 1179 |
make_fields=MakeFields, |
| 1180 |
names=names, |
| 1181 |
record_type=type, |
| 1182 |
doc=json_object.get('doc'), |
| 1183 |
other_props=other_props, |
| 1184 |
) |
| 1185 |
else: |
| 1186 |
raise Exception('Internal error: unknown type %r.' % type) |
| 1187 |
|
| 1188 |
elif type in VALID_TYPES: |
| 1189 |
# Unnamed, non-primitive Avro type: |
| 1190 |
|
| 1191 |
if type == ARRAY: |
| 1192 |
items_desc = json_object.get('items') |
| 1193 |
if items_desc is None: |
| 1194 |
raise SchemaParseException( |
| 1195 |
'Invalid array schema descriptor with no "items" : %r.' |
| 1196 |
% json_object) |
| 1197 |
return ArraySchema( |
| 1198 |
items=SchemaFromJSONData(items_desc, names), |
| 1199 |
other_props=other_props, |
| 1200 |
) |
| 1201 |
|
| 1202 |
elif type == MAP: |
| 1203 |
values_desc = json_object.get('values') |
| 1204 |
if values_desc is None: |
| 1205 |
raise SchemaParseException( |
| 1206 |
'Invalid map schema descriptor with no "values" : %r.' |
| 1207 |
% json_object) |
| 1208 |
return MapSchema( |
| 1209 |
values=SchemaFromJSONData(values_desc, names=names), |
| 1210 |
other_props=other_props, |
| 1211 |
) |
| 1212 |
|
| 1213 |
elif type == ERROR_UNION: |
| 1214 |
error_desc_list = json_object.get('declared_errors') |
| 1215 |
assert (error_desc_list is not None) |
| 1216 |
error_schemas = map( |
| 1217 |
lambda desc: SchemaFromJSONData(desc, names=names), |
| 1218 |
error_desc_list) |
| 1219 |
return ErrorUnionSchema(schemas=error_schemas) |
| 1220 |
|
| 1221 |
else: |
| 1222 |
raise Exception('Internal error: unknown type %r.' % type) |
| 1223 |
|
| 1224 |
raise SchemaParseException( |
| 1225 |
'Invalid JSON descriptor for an Avro schema: %r' % json_object) |
| 1226 |
|
| 1227 |
|
| 1228 |
# Parsers for the JSON data types: |
| 1229 |
_JSONDataParserTypeMap = { |
| 1230 |
str: _SchemaFromJSONString, |
| 1231 |
list: _SchemaFromJSONArray, |
| 1232 |
dict: _SchemaFromJSONObject, |
| 1233 |
} |
| 1234 |
|
| 1235 |
|
| 1236 |
def SchemaFromJSONData(json_data, names=None): |
| 1237 |
"""Builds an Avro Schema from its JSON descriptor. |
| 1238 |
|
| 1239 |
Args: |
| 1240 |
json_data: JSON data representing the descriptor of the Avro schema. |
| 1241 |
names: Optional tracker for Avro named schemas. |
| 1242 |
Returns: |
| 1243 |
The Avro schema parsed from the JSON descriptor. |
| 1244 |
Raises: |
| 1245 |
SchemaParseException: if the descriptor is invalid. |
| 1246 |
""" |
| 1247 |
if names is None: |
| 1248 |
names = Names() |
| 1249 |
|
| 1250 |
# Select the appropriate parser based on the JSON data type: |
| 1251 |
parser = _JSONDataParserTypeMap.get(type(json_data)) |
| 1252 |
if parser is None: |
| 1253 |
raise SchemaParseException( |
| 1254 |
'Invalid JSON descriptor for an Avro schema: %r.' % json_data) |
| 1255 |
return parser(json_data, names=names) |
| 1256 |
|
| 1257 |
|
| 1258 |
# ------------------------------------------------------------------------------ |
| 1259 |
|
| 1260 |
|
| 1261 |
def Parse(json_string): |
| 1262 |
"""Constructs a Schema from its JSON descriptor in text form. |
| 1263 |
|
| 1264 |
Args: |
| 1265 |
json_string: String representation of the JSON descriptor of the schema. |
| 1266 |
Returns: |
| 1267 |
The parsed schema. |
| 1268 |
Raises: |
| 1269 |
SchemaParseException: on JSON parsing error, |
| 1270 |
or if the JSON descriptor is invalid. |
| 1271 |
""" |
| 1272 |
try: |
| 1273 |
json_data = json.loads(json_string) |
| 1274 |
except Exception as exn: |
| 1275 |
raise SchemaParseException( |
| 1276 |
'Error parsing schema from JSON: %r. ' |
| 1277 |
'Error message: %r.' |
| 1278 |
% (json_string, exn)) |
| 1279 |
|
| 1280 |
# Initialize the names object |
| 1281 |
names = Names() |
| 1282 |
|
| 1283 |
# construct the Avro Schema object |
| 1284 |
return SchemaFromJSONData(json_data, names) |