| 1 |
#!/usr/bin/env python3 |
| 2 |
# -*- mode: python -*- |
| 3 |
# -*- coding: utf-8 -*- |
| 4 |
|
| 5 |
# Licensed to the Apache Software Foundation (ASF) under one |
| 6 |
# or more contributor license agreements. See the NOTICE file |
| 7 |
# distributed with this work for additional information |
| 8 |
# regarding copyright ownership. The ASF licenses this file |
| 9 |
# to you under the Apache License, Version 2.0 (the |
| 10 |
# "License"); you may not use this file except in compliance |
| 11 |
# with the License. You may obtain a copy of the License at |
| 12 |
# |
| 13 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 14 |
# |
| 15 |
# Unless required by applicable law or agreed to in writing, software |
| 16 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 17 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 18 |
# See the License for the specific language governing permissions and |
| 19 |
# limitations under the License. |
| 20 |
|
| 21 |
"""Input/output utilities. |
| 22 |
|
| 23 |
Includes: |
| 24 |
- i/o-specific constants |
| 25 |
- i/o-specific exceptions |
| 26 |
- schema validation |
| 27 |
- leaf value encoding and decoding |
| 28 |
- datum reader/writer stuff (?) |
| 29 |
|
| 30 |
Also includes a generic representation for data, which uses the |
| 31 |
following mapping: |
| 32 |
- Schema records are implemented as dict. |
| 33 |
- Schema arrays are implemented as list. |
| 34 |
- Schema maps are implemented as dict. |
| 35 |
- Schema strings are implemented as unicode. |
| 36 |
- Schema bytes are implemented as str. |
| 37 |
- Schema ints are implemented as int. |
| 38 |
- Schema longs are implemented as long. |
| 39 |
- Schema floats are implemented as float. |
| 40 |
- Schema doubles are implemented as float. |
| 41 |
- Schema booleans are implemented as bool. |
| 42 |
""" |
| 43 |
|
| 44 |
import binascii |
| 45 |
import json |
| 46 |
import logging |
| 47 |
import struct |
| 48 |
import sys |
| 49 |
|
| 50 |
from avro import schema |
| 51 |
|
| 52 |
logger = logging.getLogger(__name__) |
| 53 |
|
| 54 |
# ------------------------------------------------------------------------------ |
| 55 |
# Constants |
| 56 |
|
| 57 |
|
| 58 |
INT_MIN_VALUE = -(1 << 31) |
| 59 |
INT_MAX_VALUE = (1 << 31) - 1 |
| 60 |
LONG_MIN_VALUE = -(1 << 63) |
| 61 |
LONG_MAX_VALUE = (1 << 63) - 1 |
| 62 |
|
| 63 |
STRUCT_INT = struct.Struct('!I') # big-endian unsigned int |
| 64 |
STRUCT_LONG = struct.Struct('!Q') # big-endian unsigned long long |
| 65 |
STRUCT_FLOAT = struct.Struct('!f') # big-endian float |
| 66 |
STRUCT_DOUBLE = struct.Struct('!d') # big-endian double |
| 67 |
STRUCT_CRC32 = struct.Struct('>I') # big-endian unsigned int |
| 68 |
|
| 69 |
|
| 70 |
# ------------------------------------------------------------------------------ |
| 71 |
# Exceptions |
| 72 |
|
| 73 |
|
| 74 |
class AvroTypeException(schema.AvroException): |
| 75 |
"""Raised when datum is not an example of schema.""" |
| 76 |
def __init__(self, expected_schema, datum): |
| 77 |
pretty_expected = json.dumps(json.loads(str(expected_schema)), indent=2) |
| 78 |
fail_msg = "The datum %s is not an example of the schema %s"\ |
| 79 |
% (datum, pretty_expected) |
| 80 |
schema.AvroException.__init__(self, fail_msg) |
| 81 |
|
| 82 |
|
| 83 |
class SchemaResolutionException(schema.AvroException): |
| 84 |
def __init__(self, fail_msg, writer_schema=None, reader_schema=None): |
| 85 |
pretty_writers = json.dumps(json.loads(str(writer_schema)), indent=2) |
| 86 |
pretty_readers = json.dumps(json.loads(str(reader_schema)), indent=2) |
| 87 |
if writer_schema: fail_msg += "\nWriter's Schema: %s" % pretty_writers |
| 88 |
if reader_schema: fail_msg += "\nReader's Schema: %s" % pretty_readers |
| 89 |
schema.AvroException.__init__(self, fail_msg) |
| 90 |
|
| 91 |
|
| 92 |
# ------------------------------------------------------------------------------ |
| 93 |
# Validate |
| 94 |
|
| 95 |
|
| 96 |
def Validate(expected_schema, datum): |
| 97 |
"""Determines if a python datum is an instance of a schema. |
| 98 |
|
| 99 |
Args: |
| 100 |
expected_schema: Schema to validate against. |
| 101 |
datum: Datum to validate. |
| 102 |
Returns: |
| 103 |
True if the datum is an instance of the schema. |
| 104 |
""" |
| 105 |
schema_type = expected_schema.type |
| 106 |
if schema_type == 'null': |
| 107 |
return datum is None |
| 108 |
elif schema_type == 'boolean': |
| 109 |
return isinstance(datum, bool) |
| 110 |
elif schema_type == 'string': |
| 111 |
return isinstance(datum, str) |
| 112 |
elif schema_type == 'bytes': |
| 113 |
return isinstance(datum, bytes) |
| 114 |
elif schema_type == 'int': |
| 115 |
return (isinstance(datum, int) |
| 116 |
and (INT_MIN_VALUE <= datum <= INT_MAX_VALUE)) |
| 117 |
elif schema_type == 'long': |
| 118 |
return (isinstance(datum, int) |
| 119 |
and (LONG_MIN_VALUE <= datum <= LONG_MAX_VALUE)) |
| 120 |
elif schema_type in ['float', 'double']: |
| 121 |
return (isinstance(datum, int) or isinstance(datum, float)) |
| 122 |
elif schema_type == 'fixed': |
| 123 |
return isinstance(datum, bytes) and (len(datum) == expected_schema.size) |
| 124 |
elif schema_type == 'enum': |
| 125 |
return datum in expected_schema.symbols |
| 126 |
elif schema_type == 'array': |
| 127 |
return (isinstance(datum, list) |
| 128 |
and all(Validate(expected_schema.items, item) for item in datum)) |
| 129 |
elif schema_type == 'map': |
| 130 |
return (isinstance(datum, dict) |
| 131 |
and all(isinstance(key, str) for key in datum.keys()) |
| 132 |
and all(Validate(expected_schema.values, value) |
| 133 |
for value in datum.values())) |
| 134 |
elif schema_type in ['union', 'error_union']: |
| 135 |
return any(Validate(union_branch, datum) |
| 136 |
for union_branch in expected_schema.schemas) |
| 137 |
elif schema_type in ['record', 'error', 'request']: |
| 138 |
return (isinstance(datum, dict) |
| 139 |
and all(Validate(field.type, datum.get(field.name)) |
| 140 |
for field in expected_schema.fields)) |
| 141 |
else: |
| 142 |
raise AvroTypeException('Unknown Avro schema type: %r' % schema_type) |
| 143 |
|
| 144 |
|
| 145 |
# ------------------------------------------------------------------------------ |
| 146 |
# Decoder/Encoder |
| 147 |
|
| 148 |
|
| 149 |
class BinaryDecoder(object): |
| 150 |
"""Read leaf values.""" |
| 151 |
def __init__(self, reader): |
| 152 |
""" |
| 153 |
reader is a Python object on which we can call read, seek, and tell. |
| 154 |
""" |
| 155 |
self._reader = reader |
| 156 |
|
| 157 |
@property |
| 158 |
def reader(self): |
| 159 |
"""Reports the reader used by this decoder.""" |
| 160 |
return self._reader |
| 161 |
|
| 162 |
def read(self, n): |
| 163 |
"""Read n bytes. |
| 164 |
|
| 165 |
Args: |
| 166 |
n: Number of bytes to read. |
| 167 |
Returns: |
| 168 |
The next n bytes from the input. |
| 169 |
""" |
| 170 |
assert (n >= 0), n |
| 171 |
input_bytes = self.reader.read(n) |
| 172 |
assert (len(input_bytes) == n), input_bytes |
| 173 |
return input_bytes |
| 174 |
|
| 175 |
def read_null(self): |
| 176 |
""" |
| 177 |
null is written as zero bytes |
| 178 |
""" |
| 179 |
return None |
| 180 |
|
| 181 |
def read_boolean(self): |
| 182 |
""" |
| 183 |
a boolean is written as a single byte |
| 184 |
whose value is either 0 (false) or 1 (true). |
| 185 |
""" |
| 186 |
return ord(self.read(1)) == 1 |
| 187 |
|
| 188 |
def read_int(self): |
| 189 |
""" |
| 190 |
int and long values are written using variable-length, zig-zag coding. |
| 191 |
""" |
| 192 |
return self.read_long() |
| 193 |
|
| 194 |
def read_long(self): |
| 195 |
""" |
| 196 |
int and long values are written using variable-length, zig-zag coding. |
| 197 |
""" |
| 198 |
b = ord(self.read(1)) |
| 199 |
n = b & 0x7F |
| 200 |
shift = 7 |
| 201 |
while (b & 0x80) != 0: |
| 202 |
b = ord(self.read(1)) |
| 203 |
n |= (b & 0x7F) << shift |
| 204 |
shift += 7 |
| 205 |
datum = (n >> 1) ^ -(n & 1) |
| 206 |
return datum |
| 207 |
|
| 208 |
def read_float(self): |
| 209 |
""" |
| 210 |
A float is written as 4 bytes. |
| 211 |
The float is converted into a 32-bit integer using a method equivalent to |
| 212 |
Java's floatToIntBits and then encoded in little-endian format. |
| 213 |
""" |
| 214 |
bits = (((ord(self.read(1)) & 0xff)) | |
| 215 |
((ord(self.read(1)) & 0xff) << 8) | |
| 216 |
((ord(self.read(1)) & 0xff) << 16) | |
| 217 |
((ord(self.read(1)) & 0xff) << 24)) |
| 218 |
return STRUCT_FLOAT.unpack(STRUCT_INT.pack(bits))[0] |
| 219 |
|
| 220 |
def read_double(self): |
| 221 |
""" |
| 222 |
A double is written as 8 bytes. |
| 223 |
The double is converted into a 64-bit integer using a method equivalent to |
| 224 |
Java's doubleToLongBits and then encoded in little-endian format. |
| 225 |
""" |
| 226 |
bits = (((ord(self.read(1)) & 0xff)) | |
| 227 |
((ord(self.read(1)) & 0xff) << 8) | |
| 228 |
((ord(self.read(1)) & 0xff) << 16) | |
| 229 |
((ord(self.read(1)) & 0xff) << 24) | |
| 230 |
((ord(self.read(1)) & 0xff) << 32) | |
| 231 |
((ord(self.read(1)) & 0xff) << 40) | |
| 232 |
((ord(self.read(1)) & 0xff) << 48) | |
| 233 |
((ord(self.read(1)) & 0xff) << 56)) |
| 234 |
return STRUCT_DOUBLE.unpack(STRUCT_LONG.pack(bits))[0] |
| 235 |
|
| 236 |
def read_bytes(self): |
| 237 |
""" |
| 238 |
Bytes are encoded as a long followed by that many bytes of data. |
| 239 |
""" |
| 240 |
nbytes = self.read_long() |
| 241 |
assert (nbytes >= 0), nbytes |
| 242 |
return self.read(nbytes) |
| 243 |
|
| 244 |
def read_utf8(self): |
| 245 |
""" |
| 246 |
A string is encoded as a long followed by |
| 247 |
that many bytes of UTF-8 encoded character data. |
| 248 |
""" |
| 249 |
input_bytes = self.read_bytes() |
| 250 |
try: |
| 251 |
return input_bytes.decode('utf-8') |
| 252 |
except UnicodeDecodeError as exn: |
| 253 |
logger.error('Invalid UTF-8 input bytes: %r', input_bytes) |
| 254 |
raise exn |
| 255 |
|
| 256 |
def check_crc32(self, bytes): |
| 257 |
checksum = STRUCT_CRC32.unpack(self.read(4))[0]; |
| 258 |
if binascii.crc32(bytes) & 0xffffffff != checksum: |
| 259 |
raise schema.AvroException("Checksum failure") |
| 260 |
|
| 261 |
def skip_null(self): |
| 262 |
pass |
| 263 |
|
| 264 |
def skip_boolean(self): |
| 265 |
self.skip(1) |
| 266 |
|
| 267 |
def skip_int(self): |
| 268 |
self.skip_long() |
| 269 |
|
| 270 |
def skip_long(self): |
| 271 |
b = ord(self.read(1)) |
| 272 |
while (b & 0x80) != 0: |
| 273 |
b = ord(self.read(1)) |
| 274 |
|
| 275 |
def skip_float(self): |
| 276 |
self.skip(4) |
| 277 |
|
| 278 |
def skip_double(self): |
| 279 |
self.skip(8) |
| 280 |
|
| 281 |
def skip_bytes(self): |
| 282 |
self.skip(self.read_long()) |
| 283 |
|
| 284 |
def skip_utf8(self): |
| 285 |
self.skip_bytes() |
| 286 |
|
| 287 |
def skip(self, n): |
| 288 |
self.reader.seek(self.reader.tell() + n) |
| 289 |
|
| 290 |
|
| 291 |
# ------------------------------------------------------------------------------ |
| 292 |
|
| 293 |
|
| 294 |
class BinaryEncoder(object): |
| 295 |
"""Write leaf values.""" |
| 296 |
|
| 297 |
def __init__(self, writer): |
| 298 |
""" |
| 299 |
writer is a Python object on which we can call write. |
| 300 |
""" |
| 301 |
self._writer = writer |
| 302 |
|
| 303 |
@property |
| 304 |
def writer(self): |
| 305 |
"""Reports the writer used by this encoder.""" |
| 306 |
return self._writer |
| 307 |
|
| 308 |
|
| 309 |
def write(self, datum): |
| 310 |
"""Write a sequence of bytes. |
| 311 |
|
| 312 |
Args: |
| 313 |
datum: Byte array, as a Python bytes. |
| 314 |
""" |
| 315 |
assert isinstance(datum, bytes), ('Expecting bytes, got %r' % datum) |
| 316 |
self.writer.write(datum) |
| 317 |
|
| 318 |
def WriteByte(self, byte): |
| 319 |
self.writer.write(bytes((byte,))) |
| 320 |
|
| 321 |
def write_null(self, datum): |
| 322 |
""" |
| 323 |
null is written as zero bytes |
| 324 |
""" |
| 325 |
pass |
| 326 |
|
| 327 |
def write_boolean(self, datum): |
| 328 |
""" |
| 329 |
a boolean is written as a single byte |
| 330 |
whose value is either 0 (false) or 1 (true). |
| 331 |
""" |
| 332 |
# Python maps True to 1 and False to 0. |
| 333 |
self.WriteByte(int(bool(datum))) |
| 334 |
|
| 335 |
def write_int(self, datum): |
| 336 |
""" |
| 337 |
int and long values are written using variable-length, zig-zag coding. |
| 338 |
""" |
| 339 |
self.write_long(datum); |
| 340 |
|
| 341 |
def write_long(self, datum): |
| 342 |
""" |
| 343 |
int and long values are written using variable-length, zig-zag coding. |
| 344 |
""" |
| 345 |
datum = (datum << 1) ^ (datum >> 63) |
| 346 |
while (datum & ~0x7F) != 0: |
| 347 |
self.WriteByte((datum & 0x7f) | 0x80) |
| 348 |
datum >>= 7 |
| 349 |
self.WriteByte(datum) |
| 350 |
|
| 351 |
def write_float(self, datum): |
| 352 |
""" |
| 353 |
A float is written as 4 bytes. |
| 354 |
The float is converted into a 32-bit integer using a method equivalent to |
| 355 |
Java's floatToIntBits and then encoded in little-endian format. |
| 356 |
""" |
| 357 |
bits = STRUCT_INT.unpack(STRUCT_FLOAT.pack(datum))[0] |
| 358 |
self.WriteByte((bits) & 0xFF) |
| 359 |
self.WriteByte((bits >> 8) & 0xFF) |
| 360 |
self.WriteByte((bits >> 16) & 0xFF) |
| 361 |
self.WriteByte((bits >> 24) & 0xFF) |
| 362 |
|
| 363 |
def write_double(self, datum): |
| 364 |
""" |
| 365 |
A double is written as 8 bytes. |
| 366 |
The double is converted into a 64-bit integer using a method equivalent to |
| 367 |
Java's doubleToLongBits and then encoded in little-endian format. |
| 368 |
""" |
| 369 |
bits = STRUCT_LONG.unpack(STRUCT_DOUBLE.pack(datum))[0] |
| 370 |
self.WriteByte((bits) & 0xFF) |
| 371 |
self.WriteByte((bits >> 8) & 0xFF) |
| 372 |
self.WriteByte((bits >> 16) & 0xFF) |
| 373 |
self.WriteByte((bits >> 24) & 0xFF) |
| 374 |
self.WriteByte((bits >> 32) & 0xFF) |
| 375 |
self.WriteByte((bits >> 40) & 0xFF) |
| 376 |
self.WriteByte((bits >> 48) & 0xFF) |
| 377 |
self.WriteByte((bits >> 56) & 0xFF) |
| 378 |
|
| 379 |
def write_bytes(self, datum): |
| 380 |
""" |
| 381 |
Bytes are encoded as a long followed by that many bytes of data. |
| 382 |
""" |
| 383 |
self.write_long(len(datum)) |
| 384 |
self.write(datum) |
| 385 |
|
| 386 |
def write_utf8(self, datum): |
| 387 |
""" |
| 388 |
A string is encoded as a long followed by |
| 389 |
that many bytes of UTF-8 encoded character data. |
| 390 |
""" |
| 391 |
datum = datum.encode("utf-8") |
| 392 |
self.write_bytes(datum) |
| 393 |
|
| 394 |
def write_crc32(self, bytes): |
| 395 |
""" |
| 396 |
A 4-byte, big-endian CRC32 checksum |
| 397 |
""" |
| 398 |
self.write(STRUCT_CRC32.pack(binascii.crc32(bytes) & 0xffffffff)); |
| 399 |
|
| 400 |
|
| 401 |
# ------------------------------------------------------------------------------ |
| 402 |
# DatumReader/Writer |
| 403 |
|
| 404 |
|
| 405 |
class DatumReader(object): |
| 406 |
"""Deserialize Avro-encoded data into a Python data structure.""" |
| 407 |
@staticmethod |
| 408 |
def check_props(schema_one, schema_two, prop_list): |
| 409 |
for prop in prop_list: |
| 410 |
if getattr(schema_one, prop) != getattr(schema_two, prop): |
| 411 |
return False |
| 412 |
return True |
| 413 |
|
| 414 |
@staticmethod |
| 415 |
def match_schemas(writer_schema, reader_schema): |
| 416 |
w_type = writer_schema.type |
| 417 |
r_type = reader_schema.type |
| 418 |
if 'union' in [w_type, r_type] or 'error_union' in [w_type, r_type]: |
| 419 |
return True |
| 420 |
elif (w_type in schema.PRIMITIVE_TYPES and r_type in schema.PRIMITIVE_TYPES |
| 421 |
and w_type == r_type): |
| 422 |
return True |
| 423 |
elif (w_type == r_type == 'record' and |
| 424 |
DatumReader.check_props(writer_schema, reader_schema, |
| 425 |
['fullname'])): |
| 426 |
return True |
| 427 |
elif (w_type == r_type == 'error' and |
| 428 |
DatumReader.check_props(writer_schema, reader_schema, |
| 429 |
['fullname'])): |
| 430 |
return True |
| 431 |
elif (w_type == r_type == 'request'): |
| 432 |
return True |
| 433 |
elif (w_type == r_type == 'fixed' and |
| 434 |
DatumReader.check_props(writer_schema, reader_schema, |
| 435 |
['fullname', 'size'])): |
| 436 |
return True |
| 437 |
elif (w_type == r_type == 'enum' and |
| 438 |
DatumReader.check_props(writer_schema, reader_schema, |
| 439 |
['fullname'])): |
| 440 |
return True |
| 441 |
elif (w_type == r_type == 'map' and |
| 442 |
DatumReader.check_props(writer_schema.values, |
| 443 |
reader_schema.values, ['type'])): |
| 444 |
return True |
| 445 |
elif (w_type == r_type == 'array' and |
| 446 |
DatumReader.check_props(writer_schema.items, |
| 447 |
reader_schema.items, ['type'])): |
| 448 |
return True |
| 449 |
|
| 450 |
# Handle schema promotion |
| 451 |
if w_type == 'int' and r_type in ['long', 'float', 'double']: |
| 452 |
return True |
| 453 |
elif w_type == 'long' and r_type in ['float', 'double']: |
| 454 |
return True |
| 455 |
elif w_type == 'float' and r_type == 'double': |
| 456 |
return True |
| 457 |
return False |
| 458 |
|
| 459 |
def __init__(self, writer_schema=None, reader_schema=None): |
| 460 |
""" |
| 461 |
As defined in the Avro specification, we call the schema encoded |
| 462 |
in the data the "writer's schema", and the schema expected by the |
| 463 |
reader the "reader's schema". |
| 464 |
""" |
| 465 |
self._writer_schema = writer_schema |
| 466 |
self._reader_schema = reader_schema |
| 467 |
|
| 468 |
# read/write properties |
| 469 |
def set_writer_schema(self, writer_schema): |
| 470 |
self._writer_schema = writer_schema |
| 471 |
writer_schema = property(lambda self: self._writer_schema, |
| 472 |
set_writer_schema) |
| 473 |
def set_reader_schema(self, reader_schema): |
| 474 |
self._reader_schema = reader_schema |
| 475 |
reader_schema = property(lambda self: self._reader_schema, |
| 476 |
set_reader_schema) |
| 477 |
|
| 478 |
def read(self, decoder): |
| 479 |
if self.reader_schema is None: |
| 480 |
self.reader_schema = self.writer_schema |
| 481 |
return self.read_data(self.writer_schema, self.reader_schema, decoder) |
| 482 |
|
| 483 |
def read_data(self, writer_schema, reader_schema, decoder): |
| 484 |
# schema matching |
| 485 |
if not DatumReader.match_schemas(writer_schema, reader_schema): |
| 486 |
fail_msg = 'Schemas do not match.' |
| 487 |
raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) |
| 488 |
|
| 489 |
# schema resolution: reader's schema is a union, writer's schema is not |
| 490 |
if (writer_schema.type not in ['union', 'error_union'] |
| 491 |
and reader_schema.type in ['union', 'error_union']): |
| 492 |
for s in reader_schema.schemas: |
| 493 |
if DatumReader.match_schemas(writer_schema, s): |
| 494 |
return self.read_data(writer_schema, s, decoder) |
| 495 |
fail_msg = 'Schemas do not match.' |
| 496 |
raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) |
| 497 |
|
| 498 |
# function dispatch for reading data based on type of writer's schema |
| 499 |
if writer_schema.type == 'null': |
| 500 |
return decoder.read_null() |
| 501 |
elif writer_schema.type == 'boolean': |
| 502 |
return decoder.read_boolean() |
| 503 |
elif writer_schema.type == 'string': |
| 504 |
return decoder.read_utf8() |
| 505 |
elif writer_schema.type == 'int': |
| 506 |
return decoder.read_int() |
| 507 |
elif writer_schema.type == 'long': |
| 508 |
return decoder.read_long() |
| 509 |
elif writer_schema.type == 'float': |
| 510 |
return decoder.read_float() |
| 511 |
elif writer_schema.type == 'double': |
| 512 |
return decoder.read_double() |
| 513 |
elif writer_schema.type == 'bytes': |
| 514 |
return decoder.read_bytes() |
| 515 |
elif writer_schema.type == 'fixed': |
| 516 |
return self.read_fixed(writer_schema, reader_schema, decoder) |
| 517 |
elif writer_schema.type == 'enum': |
| 518 |
return self.read_enum(writer_schema, reader_schema, decoder) |
| 519 |
elif writer_schema.type == 'array': |
| 520 |
return self.read_array(writer_schema, reader_schema, decoder) |
| 521 |
elif writer_schema.type == 'map': |
| 522 |
return self.read_map(writer_schema, reader_schema, decoder) |
| 523 |
elif writer_schema.type in ['union', 'error_union']: |
| 524 |
return self.read_union(writer_schema, reader_schema, decoder) |
| 525 |
elif writer_schema.type in ['record', 'error', 'request']: |
| 526 |
return self.read_record(writer_schema, reader_schema, decoder) |
| 527 |
else: |
| 528 |
fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type |
| 529 |
raise schema.AvroException(fail_msg) |
| 530 |
|
| 531 |
def skip_data(self, writer_schema, decoder): |
| 532 |
if writer_schema.type == 'null': |
| 533 |
return decoder.skip_null() |
| 534 |
elif writer_schema.type == 'boolean': |
| 535 |
return decoder.skip_boolean() |
| 536 |
elif writer_schema.type == 'string': |
| 537 |
return decoder.skip_utf8() |
| 538 |
elif writer_schema.type == 'int': |
| 539 |
return decoder.skip_int() |
| 540 |
elif writer_schema.type == 'long': |
| 541 |
return decoder.skip_long() |
| 542 |
elif writer_schema.type == 'float': |
| 543 |
return decoder.skip_float() |
| 544 |
elif writer_schema.type == 'double': |
| 545 |
return decoder.skip_double() |
| 546 |
elif writer_schema.type == 'bytes': |
| 547 |
return decoder.skip_bytes() |
| 548 |
elif writer_schema.type == 'fixed': |
| 549 |
return self.skip_fixed(writer_schema, decoder) |
| 550 |
elif writer_schema.type == 'enum': |
| 551 |
return self.skip_enum(writer_schema, decoder) |
| 552 |
elif writer_schema.type == 'array': |
| 553 |
return self.skip_array(writer_schema, decoder) |
| 554 |
elif writer_schema.type == 'map': |
| 555 |
return self.skip_map(writer_schema, decoder) |
| 556 |
elif writer_schema.type in ['union', 'error_union']: |
| 557 |
return self.skip_union(writer_schema, decoder) |
| 558 |
elif writer_schema.type in ['record', 'error', 'request']: |
| 559 |
return self.skip_record(writer_schema, decoder) |
| 560 |
else: |
| 561 |
fail_msg = "Unknown schema type: %s" % writer_schema.type |
| 562 |
raise schema.AvroException(fail_msg) |
| 563 |
|
| 564 |
def read_fixed(self, writer_schema, reader_schema, decoder): |
| 565 |
""" |
| 566 |
Fixed instances are encoded using the number of bytes declared |
| 567 |
in the schema. |
| 568 |
""" |
| 569 |
return decoder.read(writer_schema.size) |
| 570 |
|
| 571 |
def skip_fixed(self, writer_schema, decoder): |
| 572 |
return decoder.skip(writer_schema.size) |
| 573 |
|
| 574 |
def read_enum(self, writer_schema, reader_schema, decoder): |
| 575 |
""" |
| 576 |
An enum is encoded by a int, representing the zero-based position |
| 577 |
of the symbol in the schema. |
| 578 |
""" |
| 579 |
# read data |
| 580 |
index_of_symbol = decoder.read_int() |
| 581 |
if index_of_symbol >= len(writer_schema.symbols): |
| 582 |
fail_msg = "Can't access enum index %d for enum with %d symbols"\ |
| 583 |
% (index_of_symbol, len(writer_schema.symbols)) |
| 584 |
raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) |
| 585 |
read_symbol = writer_schema.symbols[index_of_symbol] |
| 586 |
|
| 587 |
# schema resolution |
| 588 |
if read_symbol not in reader_schema.symbols: |
| 589 |
fail_msg = "Symbol %s not present in Reader's Schema" % read_symbol |
| 590 |
raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) |
| 591 |
|
| 592 |
return read_symbol |
| 593 |
|
| 594 |
def skip_enum(self, writer_schema, decoder): |
| 595 |
return decoder.skip_int() |
| 596 |
|
| 597 |
def read_array(self, writer_schema, reader_schema, decoder): |
| 598 |
""" |
| 599 |
Arrays are encoded as a series of blocks. |
| 600 |
|
| 601 |
Each block consists of a long count value, |
| 602 |
followed by that many array items. |
| 603 |
A block with count zero indicates the end of the array. |
| 604 |
Each item is encoded per the array's item schema. |
| 605 |
|
| 606 |
If a block's count is negative, |
| 607 |
then the count is followed immediately by a long block size, |
| 608 |
indicating the number of bytes in the block. |
| 609 |
The actual count in this case |
| 610 |
is the absolute value of the count written. |
| 611 |
""" |
| 612 |
read_items = [] |
| 613 |
block_count = decoder.read_long() |
| 614 |
while block_count != 0: |
| 615 |
if block_count < 0: |
| 616 |
block_count = -block_count |
| 617 |
block_size = decoder.read_long() |
| 618 |
for i in range(block_count): |
| 619 |
read_items.append(self.read_data(writer_schema.items, |
| 620 |
reader_schema.items, decoder)) |
| 621 |
block_count = decoder.read_long() |
| 622 |
return read_items |
| 623 |
|
| 624 |
def skip_array(self, writer_schema, decoder): |
| 625 |
block_count = decoder.read_long() |
| 626 |
while block_count != 0: |
| 627 |
if block_count < 0: |
| 628 |
block_size = decoder.read_long() |
| 629 |
decoder.skip(block_size) |
| 630 |
else: |
| 631 |
for i in range(block_count): |
| 632 |
self.skip_data(writer_schema.items, decoder) |
| 633 |
block_count = decoder.read_long() |
| 634 |
|
| 635 |
def read_map(self, writer_schema, reader_schema, decoder): |
| 636 |
""" |
| 637 |
Maps are encoded as a series of blocks. |
| 638 |
|
| 639 |
Each block consists of a long count value, |
| 640 |
followed by that many key/value pairs. |
| 641 |
A block with count zero indicates the end of the map. |
| 642 |
Each item is encoded per the map's value schema. |
| 643 |
|
| 644 |
If a block's count is negative, |
| 645 |
then the count is followed immediately by a long block size, |
| 646 |
indicating the number of bytes in the block. |
| 647 |
The actual count in this case |
| 648 |
is the absolute value of the count written. |
| 649 |
""" |
| 650 |
read_items = {} |
| 651 |
block_count = decoder.read_long() |
| 652 |
while block_count != 0: |
| 653 |
if block_count < 0: |
| 654 |
block_count = -block_count |
| 655 |
block_size = decoder.read_long() |
| 656 |
for i in range(block_count): |
| 657 |
key = decoder.read_utf8() |
| 658 |
read_items[key] = self.read_data(writer_schema.values, |
| 659 |
reader_schema.values, decoder) |
| 660 |
block_count = decoder.read_long() |
| 661 |
return read_items |
| 662 |
|
| 663 |
def skip_map(self, writer_schema, decoder): |
| 664 |
block_count = decoder.read_long() |
| 665 |
while block_count != 0: |
| 666 |
if block_count < 0: |
| 667 |
block_size = decoder.read_long() |
| 668 |
decoder.skip(block_size) |
| 669 |
else: |
| 670 |
for i in range(block_count): |
| 671 |
decoder.skip_utf8() |
| 672 |
self.skip_data(writer_schema.values, decoder) |
| 673 |
block_count = decoder.read_long() |
| 674 |
|
| 675 |
def read_union(self, writer_schema, reader_schema, decoder): |
| 676 |
""" |
| 677 |
A union is encoded by first writing a long value indicating |
| 678 |
the zero-based position within the union of the schema of its value. |
| 679 |
The value is then encoded per the indicated schema within the union. |
| 680 |
""" |
| 681 |
# schema resolution |
| 682 |
index_of_schema = int(decoder.read_long()) |
| 683 |
if index_of_schema >= len(writer_schema.schemas): |
| 684 |
fail_msg = "Can't access branch index %d for union with %d branches"\ |
| 685 |
% (index_of_schema, len(writer_schema.schemas)) |
| 686 |
raise SchemaResolutionException(fail_msg, writer_schema, reader_schema) |
| 687 |
selected_writer_schema = writer_schema.schemas[index_of_schema] |
| 688 |
|
| 689 |
# read data |
| 690 |
return self.read_data(selected_writer_schema, reader_schema, decoder) |
| 691 |
|
| 692 |
def skip_union(self, writer_schema, decoder): |
| 693 |
index_of_schema = int(decoder.read_long()) |
| 694 |
if index_of_schema >= len(writer_schema.schemas): |
| 695 |
fail_msg = "Can't access branch index %d for union with %d branches"\ |
| 696 |
% (index_of_schema, len(writer_schema.schemas)) |
| 697 |
raise SchemaResolutionException(fail_msg, writer_schema) |
| 698 |
return self.skip_data(writer_schema.schemas[index_of_schema], decoder) |
| 699 |
|
| 700 |
def read_record(self, writer_schema, reader_schema, decoder): |
| 701 |
""" |
| 702 |
A record is encoded by encoding the values of its fields |
| 703 |
in the order that they are declared. In other words, a record |
| 704 |
is encoded as just the concatenation of the encodings of its fields. |
| 705 |
Field values are encoded per their schema. |
| 706 |
|
| 707 |
Schema Resolution: |
| 708 |
* the ordering of fields may be different: fields are matched by name. |
| 709 |
* schemas for fields with the same name in both records are resolved |
| 710 |
recursively. |
| 711 |
* if the writer's record contains a field with a name not present in the |
| 712 |
reader's record, the writer's value for that field is ignored. |
| 713 |
* if the reader's record schema has a field that contains a default value, |
| 714 |
and writer's schema does not have a field with the same name, then the |
| 715 |
reader should use the default value from its field. |
| 716 |
* if the reader's record schema has a field with no default value, and |
| 717 |
writer's schema does not have a field with the same name, then the |
| 718 |
field's value is unset. |
| 719 |
""" |
| 720 |
# schema resolution |
| 721 |
readers_fields_dict = reader_schema.field_map |
| 722 |
read_record = {} |
| 723 |
for field in writer_schema.fields: |
| 724 |
readers_field = readers_fields_dict.get(field.name) |
| 725 |
if readers_field is not None: |
| 726 |
field_val = self.read_data(field.type, readers_field.type, decoder) |
| 727 |
read_record[field.name] = field_val |
| 728 |
else: |
| 729 |
self.skip_data(field.type, decoder) |
| 730 |
|
| 731 |
# fill in default values |
| 732 |
if len(readers_fields_dict) > len(read_record): |
| 733 |
writers_fields_dict = writer_schema.field_map |
| 734 |
for field_name, field in readers_fields_dict.items(): |
| 735 |
if field_name not in writers_fields_dict: |
| 736 |
if field.has_default: |
| 737 |
field_val = self._read_default_value(field.type, field.default) |
| 738 |
read_record[field.name] = field_val |
| 739 |
else: |
| 740 |
fail_msg = 'No default value for field %s' % field_name |
| 741 |
raise SchemaResolutionException(fail_msg, writer_schema, |
| 742 |
reader_schema) |
| 743 |
return read_record |
| 744 |
|
| 745 |
def skip_record(self, writer_schema, decoder): |
| 746 |
for field in writer_schema.fields: |
| 747 |
self.skip_data(field.type, decoder) |
| 748 |
|
| 749 |
def _read_default_value(self, field_schema, default_value): |
| 750 |
""" |
| 751 |
Basically a JSON Decoder? |
| 752 |
""" |
| 753 |
if field_schema.type == 'null': |
| 754 |
return None |
| 755 |
elif field_schema.type == 'boolean': |
| 756 |
return bool(default_value) |
| 757 |
elif field_schema.type == 'int': |
| 758 |
return int(default_value) |
| 759 |
elif field_schema.type == 'long': |
| 760 |
return int(default_value) |
| 761 |
elif field_schema.type in ['float', 'double']: |
| 762 |
return float(default_value) |
| 763 |
elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']: |
| 764 |
return default_value |
| 765 |
elif field_schema.type == 'array': |
| 766 |
read_array = [] |
| 767 |
for json_val in default_value: |
| 768 |
item_val = self._read_default_value(field_schema.items, json_val) |
| 769 |
read_array.append(item_val) |
| 770 |
return read_array |
| 771 |
elif field_schema.type == 'map': |
| 772 |
read_map = {} |
| 773 |
for key, json_val in default_value.items(): |
| 774 |
map_val = self._read_default_value(field_schema.values, json_val) |
| 775 |
read_map[key] = map_val |
| 776 |
return read_map |
| 777 |
elif field_schema.type in ['union', 'error_union']: |
| 778 |
return self._read_default_value(field_schema.schemas[0], default_value) |
| 779 |
elif field_schema.type == 'record': |
| 780 |
read_record = {} |
| 781 |
for field in field_schema.fields: |
| 782 |
json_val = default_value.get(field.name) |
| 783 |
if json_val is None: json_val = field.default |
| 784 |
field_val = self._read_default_value(field.type, json_val) |
| 785 |
read_record[field.name] = field_val |
| 786 |
return read_record |
| 787 |
else: |
| 788 |
fail_msg = 'Unknown type: %s' % field_schema.type |
| 789 |
raise schema.AvroException(fail_msg) |
| 790 |
|
| 791 |
|
| 792 |
# ------------------------------------------------------------------------------ |
| 793 |
|
| 794 |
|
| 795 |
class DatumWriter(object): |
| 796 |
"""DatumWriter for generic python objects.""" |
| 797 |
def __init__(self, writer_schema=None): |
| 798 |
self._writer_schema = writer_schema |
| 799 |
|
| 800 |
# read/write properties |
| 801 |
def set_writer_schema(self, writer_schema): |
| 802 |
self._writer_schema = writer_schema |
| 803 |
writer_schema = property(lambda self: self._writer_schema, |
| 804 |
set_writer_schema) |
| 805 |
|
| 806 |
def write(self, datum, encoder): |
| 807 |
# validate datum |
| 808 |
if not Validate(self.writer_schema, datum): |
| 809 |
raise AvroTypeException(self.writer_schema, datum) |
| 810 |
|
| 811 |
self.write_data(self.writer_schema, datum, encoder) |
| 812 |
|
| 813 |
def write_data(self, writer_schema, datum, encoder): |
| 814 |
# function dispatch to write datum |
| 815 |
if writer_schema.type == 'null': |
| 816 |
encoder.write_null(datum) |
| 817 |
elif writer_schema.type == 'boolean': |
| 818 |
encoder.write_boolean(datum) |
| 819 |
elif writer_schema.type == 'string': |
| 820 |
encoder.write_utf8(datum) |
| 821 |
elif writer_schema.type == 'int': |
| 822 |
encoder.write_int(datum) |
| 823 |
elif writer_schema.type == 'long': |
| 824 |
encoder.write_long(datum) |
| 825 |
elif writer_schema.type == 'float': |
| 826 |
encoder.write_float(datum) |
| 827 |
elif writer_schema.type == 'double': |
| 828 |
encoder.write_double(datum) |
| 829 |
elif writer_schema.type == 'bytes': |
| 830 |
encoder.write_bytes(datum) |
| 831 |
elif writer_schema.type == 'fixed': |
| 832 |
self.write_fixed(writer_schema, datum, encoder) |
| 833 |
elif writer_schema.type == 'enum': |
| 834 |
self.write_enum(writer_schema, datum, encoder) |
| 835 |
elif writer_schema.type == 'array': |
| 836 |
self.write_array(writer_schema, datum, encoder) |
| 837 |
elif writer_schema.type == 'map': |
| 838 |
self.write_map(writer_schema, datum, encoder) |
| 839 |
elif writer_schema.type in ['union', 'error_union']: |
| 840 |
self.write_union(writer_schema, datum, encoder) |
| 841 |
elif writer_schema.type in ['record', 'error', 'request']: |
| 842 |
self.write_record(writer_schema, datum, encoder) |
| 843 |
else: |
| 844 |
fail_msg = 'Unknown type: %s' % writer_schema.type |
| 845 |
raise schema.AvroException(fail_msg) |
| 846 |
|
| 847 |
def write_fixed(self, writer_schema, datum, encoder): |
| 848 |
""" |
| 849 |
Fixed instances are encoded using the number of bytes declared |
| 850 |
in the schema. |
| 851 |
""" |
| 852 |
encoder.write(datum) |
| 853 |
|
| 854 |
def write_enum(self, writer_schema, datum, encoder): |
| 855 |
""" |
| 856 |
An enum is encoded by a int, representing the zero-based position |
| 857 |
of the symbol in the schema. |
| 858 |
""" |
| 859 |
index_of_datum = writer_schema.symbols.index(datum) |
| 860 |
encoder.write_int(index_of_datum) |
| 861 |
|
| 862 |
def write_array(self, writer_schema, datum, encoder): |
| 863 |
""" |
| 864 |
Arrays are encoded as a series of blocks. |
| 865 |
|
| 866 |
Each block consists of a long count value, |
| 867 |
followed by that many array items. |
| 868 |
A block with count zero indicates the end of the array. |
| 869 |
Each item is encoded per the array's item schema. |
| 870 |
|
| 871 |
If a block's count is negative, |
| 872 |
then the count is followed immediately by a long block size, |
| 873 |
indicating the number of bytes in the block. |
| 874 |
The actual count in this case |
| 875 |
is the absolute value of the count written. |
| 876 |
""" |
| 877 |
if len(datum) > 0: |
| 878 |
encoder.write_long(len(datum)) |
| 879 |
for item in datum: |
| 880 |
self.write_data(writer_schema.items, item, encoder) |
| 881 |
encoder.write_long(0) |
| 882 |
|
| 883 |
def write_map(self, writer_schema, datum, encoder): |
| 884 |
""" |
| 885 |
Maps are encoded as a series of blocks. |
| 886 |
|
| 887 |
Each block consists of a long count value, |
| 888 |
followed by that many key/value pairs. |
| 889 |
A block with count zero indicates the end of the map. |
| 890 |
Each item is encoded per the map's value schema. |
| 891 |
|
| 892 |
If a block's count is negative, |
| 893 |
then the count is followed immediately by a long block size, |
| 894 |
indicating the number of bytes in the block. |
| 895 |
The actual count in this case |
| 896 |
is the absolute value of the count written. |
| 897 |
""" |
| 898 |
if len(datum) > 0: |
| 899 |
encoder.write_long(len(datum)) |
| 900 |
for key, val in datum.items(): |
| 901 |
encoder.write_utf8(key) |
| 902 |
self.write_data(writer_schema.values, val, encoder) |
| 903 |
encoder.write_long(0) |
| 904 |
|
| 905 |
def write_union(self, writer_schema, datum, encoder): |
| 906 |
""" |
| 907 |
A union is encoded by first writing a long value indicating |
| 908 |
the zero-based position within the union of the schema of its value. |
| 909 |
The value is then encoded per the indicated schema within the union. |
| 910 |
""" |
| 911 |
# resolve union |
| 912 |
index_of_schema = -1 |
| 913 |
for i, candidate_schema in enumerate(writer_schema.schemas): |
| 914 |
if Validate(candidate_schema, datum): |
| 915 |
index_of_schema = i |
| 916 |
if index_of_schema < 0: raise AvroTypeException(writer_schema, datum) |
| 917 |
|
| 918 |
# write data |
| 919 |
encoder.write_long(index_of_schema) |
| 920 |
self.write_data(writer_schema.schemas[index_of_schema], datum, encoder) |
| 921 |
|
| 922 |
def write_record(self, writer_schema, datum, encoder): |
| 923 |
""" |
| 924 |
A record is encoded by encoding the values of its fields |
| 925 |
in the order that they are declared. In other words, a record |
| 926 |
is encoded as just the concatenation of the encodings of its fields. |
| 927 |
Field values are encoded per their schema. |
| 928 |
""" |
| 929 |
for field in writer_schema.fields: |
| 930 |
self.write_data(field.type, datum.get(field.name), encoder) |
| 931 |
|
| 932 |
|
| 933 |
if __name__ == '__main__': |
| 934 |
raise Exception('Not a standalone module') |