Debian Codegraph @

lang/py3/avro/io.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 """Input/output utilities.
22
23 Includes:
24 - i/o-specific constants
25 - i/o-specific exceptions
26 - schema validation
27 - leaf value encoding and decoding
28 - datum reader/writer stuff (?)
29
30 Also includes a generic representation for data, which uses the
31 following mapping:
32 - Schema records are implemented as dict.
33 - Schema arrays are implemented as list.
34 - Schema maps are implemented as dict.
35 - Schema strings are implemented as unicode.
36 - Schema bytes are implemented as str.
37 - Schema ints are implemented as int.
38 - Schema longs are implemented as long.
39 - Schema floats are implemented as float.
40 - Schema doubles are implemented as float.
41 - Schema booleans are implemented as bool.
42 """
43
44 import binascii
45 import json
46 import logging
47 import struct
48 import sys
49
50 from avro import schema
51
52 logger = logging.getLogger(__name__)
53
54 # ------------------------------------------------------------------------------
55 # Constants
56
57
58 INT_MIN_VALUE = -(1 << 31)
59 INT_MAX_VALUE = (1 << 31) - 1
60 LONG_MIN_VALUE = -(1 << 63)
61 LONG_MAX_VALUE = (1 << 63) - 1
62
63 STRUCT_INT = struct.Struct('!I') # big-endian unsigned int
64 STRUCT_LONG = struct.Struct('!Q') # big-endian unsigned long long
65 STRUCT_FLOAT = struct.Struct('!f') # big-endian float
66 STRUCT_DOUBLE = struct.Struct('!d') # big-endian double
67 STRUCT_CRC32 = struct.Struct('>I') # big-endian unsigned int
68
69
70 # ------------------------------------------------------------------------------
71 # Exceptions
72
73
74 class AvroTypeException(schema.AvroException):
75 """Raised when datum is not an example of schema."""
76 def __init__(self, expected_schema, datum):
77 pretty_expected = json.dumps(json.loads(str(expected_schema)), indent=2)
78 fail_msg = "The datum %s is not an example of the schema %s"\
79 % (datum, pretty_expected)
80 schema.AvroException.__init__(self, fail_msg)
81
82
83 class SchemaResolutionException(schema.AvroException):
84 def __init__(self, fail_msg, writer_schema=None, reader_schema=None):
85 pretty_writers = json.dumps(json.loads(str(writer_schema)), indent=2)
86 pretty_readers = json.dumps(json.loads(str(reader_schema)), indent=2)
87 if writer_schema: fail_msg += "\nWriter's Schema: %s" % pretty_writers
88 if reader_schema: fail_msg += "\nReader's Schema: %s" % pretty_readers
89 schema.AvroException.__init__(self, fail_msg)
90
91
92 # ------------------------------------------------------------------------------
93 # Validate
94
95
96 def Validate(expected_schema, datum):
97 """Determines if a python datum is an instance of a schema.
98
99 Args:
100 expected_schema: Schema to validate against.
101 datum: Datum to validate.
102 Returns:
103 True if the datum is an instance of the schema.
104 """
105 schema_type = expected_schema.type
106 if schema_type == 'null':
107 return datum is None
108 elif schema_type == 'boolean':
109 return isinstance(datum, bool)
110 elif schema_type == 'string':
111 return isinstance(datum, str)
112 elif schema_type == 'bytes':
113 return isinstance(datum, bytes)
114 elif schema_type == 'int':
115 return (isinstance(datum, int)
116 and (INT_MIN_VALUE <= datum <= INT_MAX_VALUE))
117 elif schema_type == 'long':
118 return (isinstance(datum, int)
119 and (LONG_MIN_VALUE <= datum <= LONG_MAX_VALUE))
120 elif schema_type in ['float', 'double']:
121 return (isinstance(datum, int) or isinstance(datum, float))
122 elif schema_type == 'fixed':
123 return isinstance(datum, bytes) and (len(datum) == expected_schema.size)
124 elif schema_type == 'enum':
125 return datum in expected_schema.symbols
126 elif schema_type == 'array':
127 return (isinstance(datum, list)
128 and all(Validate(expected_schema.items, item) for item in datum))
129 elif schema_type == 'map':
130 return (isinstance(datum, dict)
131 and all(isinstance(key, str) for key in datum.keys())
132 and all(Validate(expected_schema.values, value)
133 for value in datum.values()))
134 elif schema_type in ['union', 'error_union']:
135 return any(Validate(union_branch, datum)
136 for union_branch in expected_schema.schemas)
137 elif schema_type in ['record', 'error', 'request']:
138 return (isinstance(datum, dict)
139 and all(Validate(field.type, datum.get(field.name))
140 for field in expected_schema.fields))
141 else:
142 raise AvroTypeException('Unknown Avro schema type: %r' % schema_type)
143
144
145 # ------------------------------------------------------------------------------
146 # Decoder/Encoder
147
148
149 class BinaryDecoder(object):
150 """Read leaf values."""
151 def __init__(self, reader):
152 """
153 reader is a Python object on which we can call read, seek, and tell.
154 """
155 self._reader = reader
156
157 @property
158 def reader(self):
159 """Reports the reader used by this decoder."""
160 return self._reader
161
162 def read(self, n):
163 """Read n bytes.
164
165 Args:
166 n: Number of bytes to read.
167 Returns:
168 The next n bytes from the input.
169 """
170 assert (n >= 0), n
171 input_bytes = self.reader.read(n)
172 assert (len(input_bytes) == n), input_bytes
173 return input_bytes
174
175 def read_null(self):
176 """
177 null is written as zero bytes
178 """
179 return None
180
181 def read_boolean(self):
182 """
183 a boolean is written as a single byte
184 whose value is either 0 (false) or 1 (true).
185 """
186 return ord(self.read(1)) == 1
187
188 def read_int(self):
189 """
190 int and long values are written using variable-length, zig-zag coding.
191 """
192 return self.read_long()
193
194 def read_long(self):
195 """
196 int and long values are written using variable-length, zig-zag coding.
197 """
198 b = ord(self.read(1))
199 n = b & 0x7F
200 shift = 7
201 while (b & 0x80) != 0:
202 b = ord(self.read(1))
203 n |= (b & 0x7F) << shift
204 shift += 7
205 datum = (n >> 1) ^ -(n & 1)
206 return datum
207
208 def read_float(self):
209 """
210 A float is written as 4 bytes.
211 The float is converted into a 32-bit integer using a method equivalent to
212 Java's floatToIntBits and then encoded in little-endian format.
213 """
214 bits = (((ord(self.read(1)) & 0xff)) |
215 ((ord(self.read(1)) & 0xff) << 8) |
216 ((ord(self.read(1)) & 0xff) << 16) |
217 ((ord(self.read(1)) & 0xff) << 24))
218 return STRUCT_FLOAT.unpack(STRUCT_INT.pack(bits))[0]
219
220 def read_double(self):
221 """
222 A double is written as 8 bytes.
223 The double is converted into a 64-bit integer using a method equivalent to
224 Java's doubleToLongBits and then encoded in little-endian format.
225 """
226 bits = (((ord(self.read(1)) & 0xff)) |
227 ((ord(self.read(1)) & 0xff) << 8) |
228 ((ord(self.read(1)) & 0xff) << 16) |
229 ((ord(self.read(1)) & 0xff) << 24) |
230 ((ord(self.read(1)) & 0xff) << 32) |
231 ((ord(self.read(1)) & 0xff) << 40) |
232 ((ord(self.read(1)) & 0xff) << 48) |
233 ((ord(self.read(1)) & 0xff) << 56))
234 return STRUCT_DOUBLE.unpack(STRUCT_LONG.pack(bits))[0]
235
236 def read_bytes(self):
237 """
238 Bytes are encoded as a long followed by that many bytes of data.
239 """
240 nbytes = self.read_long()
241 assert (nbytes >= 0), nbytes
242 return self.read(nbytes)
243
244 def read_utf8(self):
245 """
246 A string is encoded as a long followed by
247 that many bytes of UTF-8 encoded character data.
248 """
249 input_bytes = self.read_bytes()
250 try:
251 return input_bytes.decode('utf-8')
252 except UnicodeDecodeError as exn:
253 logger.error('Invalid UTF-8 input bytes: %r', input_bytes)
254 raise exn
255
256 def check_crc32(self, bytes):
257 checksum = STRUCT_CRC32.unpack(self.read(4))[0];
258 if binascii.crc32(bytes) & 0xffffffff != checksum:
259 raise schema.AvroException("Checksum failure")
260
261 def skip_null(self):
262 pass
263
264 def skip_boolean(self):
265 self.skip(1)
266
267 def skip_int(self):
268 self.skip_long()
269
270 def skip_long(self):
271 b = ord(self.read(1))
272 while (b & 0x80) != 0:
273 b = ord(self.read(1))
274
275 def skip_float(self):
276 self.skip(4)
277
278 def skip_double(self):
279 self.skip(8)
280
281 def skip_bytes(self):
282 self.skip(self.read_long())
283
284 def skip_utf8(self):
285 self.skip_bytes()
286
287 def skip(self, n):
288 self.reader.seek(self.reader.tell() + n)
289
290
291 # ------------------------------------------------------------------------------
292
293
294 class BinaryEncoder(object):
295 """Write leaf values."""
296
297 def __init__(self, writer):
298 """
299 writer is a Python object on which we can call write.
300 """
301 self._writer = writer
302
303 @property
304 def writer(self):
305 """Reports the writer used by this encoder."""
306 return self._writer
307
308
309 def write(self, datum):
310 """Write a sequence of bytes.
311
312 Args:
313 datum: Byte array, as a Python bytes.
314 """
315 assert isinstance(datum, bytes), ('Expecting bytes, got %r' % datum)
316 self.writer.write(datum)
317
318 def WriteByte(self, byte):
319 self.writer.write(bytes((byte,)))
320
321 def write_null(self, datum):
322 """
323 null is written as zero bytes
324 """
325 pass
326
327 def write_boolean(self, datum):
328 """
329 a boolean is written as a single byte
330 whose value is either 0 (false) or 1 (true).
331 """
332 # Python maps True to 1 and False to 0.
333 self.WriteByte(int(bool(datum)))
334
335 def write_int(self, datum):
336 """
337 int and long values are written using variable-length, zig-zag coding.
338 """
339 self.write_long(datum);
340
341 def write_long(self, datum):
342 """
343 int and long values are written using variable-length, zig-zag coding.
344 """
345 datum = (datum << 1) ^ (datum >> 63)
346 while (datum & ~0x7F) != 0:
347 self.WriteByte((datum & 0x7f) | 0x80)
348 datum >>= 7
349 self.WriteByte(datum)
350
351 def write_float(self, datum):
352 """
353 A float is written as 4 bytes.
354 The float is converted into a 32-bit integer using a method equivalent to
355 Java's floatToIntBits and then encoded in little-endian format.
356 """
357 bits = STRUCT_INT.unpack(STRUCT_FLOAT.pack(datum))[0]
358 self.WriteByte((bits) & 0xFF)
359 self.WriteByte((bits >> 8) & 0xFF)
360 self.WriteByte((bits >> 16) & 0xFF)
361 self.WriteByte((bits >> 24) & 0xFF)
362
363 def write_double(self, datum):
364 """
365 A double is written as 8 bytes.
366 The double is converted into a 64-bit integer using a method equivalent to
367 Java's doubleToLongBits and then encoded in little-endian format.
368 """
369 bits = STRUCT_LONG.unpack(STRUCT_DOUBLE.pack(datum))[0]
370 self.WriteByte((bits) & 0xFF)
371 self.WriteByte((bits >> 8) & 0xFF)
372 self.WriteByte((bits >> 16) & 0xFF)
373 self.WriteByte((bits >> 24) & 0xFF)
374 self.WriteByte((bits >> 32) & 0xFF)
375 self.WriteByte((bits >> 40) & 0xFF)
376 self.WriteByte((bits >> 48) & 0xFF)
377 self.WriteByte((bits >> 56) & 0xFF)
378
379 def write_bytes(self, datum):
380 """
381 Bytes are encoded as a long followed by that many bytes of data.
382 """
383 self.write_long(len(datum))
384 self.write(datum)
385
386 def write_utf8(self, datum):
387 """
388 A string is encoded as a long followed by
389 that many bytes of UTF-8 encoded character data.
390 """
391 datum = datum.encode("utf-8")
392 self.write_bytes(datum)
393
394 def write_crc32(self, bytes):
395 """
396 A 4-byte, big-endian CRC32 checksum
397 """
398 self.write(STRUCT_CRC32.pack(binascii.crc32(bytes) & 0xffffffff));
399
400
401 # ------------------------------------------------------------------------------
402 # DatumReader/Writer
403
404
405 class DatumReader(object):
406 """Deserialize Avro-encoded data into a Python data structure."""
407 @staticmethod
408 def check_props(schema_one, schema_two, prop_list):
409 for prop in prop_list:
410 if getattr(schema_one, prop) != getattr(schema_two, prop):
411 return False
412 return True
413
414 @staticmethod
415 def match_schemas(writer_schema, reader_schema):
416 w_type = writer_schema.type
417 r_type = reader_schema.type
418 if 'union' in [w_type, r_type] or 'error_union' in [w_type, r_type]:
419 return True
420 elif (w_type in schema.PRIMITIVE_TYPES and r_type in schema.PRIMITIVE_TYPES
421 and w_type == r_type):
422 return True
423 elif (w_type == r_type == 'record' and
424 DatumReader.check_props(writer_schema, reader_schema,
425 ['fullname'])):
426 return True
427 elif (w_type == r_type == 'error' and
428 DatumReader.check_props(writer_schema, reader_schema,
429 ['fullname'])):
430 return True
431 elif (w_type == r_type == 'request'):
432 return True
433 elif (w_type == r_type == 'fixed' and
434 DatumReader.check_props(writer_schema, reader_schema,
435 ['fullname', 'size'])):
436 return True
437 elif (w_type == r_type == 'enum' and
438 DatumReader.check_props(writer_schema, reader_schema,
439 ['fullname'])):
440 return True
441 elif (w_type == r_type == 'map' and
442 DatumReader.check_props(writer_schema.values,
443 reader_schema.values, ['type'])):
444 return True
445 elif (w_type == r_type == 'array' and
446 DatumReader.check_props(writer_schema.items,
447 reader_schema.items, ['type'])):
448 return True
449
450 # Handle schema promotion
451 if w_type == 'int' and r_type in ['long', 'float', 'double']:
452 return True
453 elif w_type == 'long' and r_type in ['float', 'double']:
454 return True
455 elif w_type == 'float' and r_type == 'double':
456 return True
457 return False
458
459 def __init__(self, writer_schema=None, reader_schema=None):
460 """
461 As defined in the Avro specification, we call the schema encoded
462 in the data the "writer's schema", and the schema expected by the
463 reader the "reader's schema".
464 """
465 self._writer_schema = writer_schema
466 self._reader_schema = reader_schema
467
468 # read/write properties
469 def set_writer_schema(self, writer_schema):
470 self._writer_schema = writer_schema
471 writer_schema = property(lambda self: self._writer_schema,
472 set_writer_schema)
473 def set_reader_schema(self, reader_schema):
474 self._reader_schema = reader_schema
475 reader_schema = property(lambda self: self._reader_schema,
476 set_reader_schema)
477
478 def read(self, decoder):
479 if self.reader_schema is None:
480 self.reader_schema = self.writer_schema
481 return self.read_data(self.writer_schema, self.reader_schema, decoder)
482
483 def read_data(self, writer_schema, reader_schema, decoder):
484 # schema matching
485 if not DatumReader.match_schemas(writer_schema, reader_schema):
486 fail_msg = 'Schemas do not match.'
487 raise SchemaResolutionException(fail_msg, writer_schema, reader_schema)
488
489 # schema resolution: reader's schema is a union, writer's schema is not
490 if (writer_schema.type not in ['union', 'error_union']
491 and reader_schema.type in ['union', 'error_union']):
492 for s in reader_schema.schemas:
493 if DatumReader.match_schemas(writer_schema, s):
494 return self.read_data(writer_schema, s, decoder)
495 fail_msg = 'Schemas do not match.'
496 raise SchemaResolutionException(fail_msg, writer_schema, reader_schema)
497
498 # function dispatch for reading data based on type of writer's schema
499 if writer_schema.type == 'null':
500 return decoder.read_null()
501 elif writer_schema.type == 'boolean':
502 return decoder.read_boolean()
503 elif writer_schema.type == 'string':
504 return decoder.read_utf8()
505 elif writer_schema.type == 'int':
506 return decoder.read_int()
507 elif writer_schema.type == 'long':
508 return decoder.read_long()
509 elif writer_schema.type == 'float':
510 return decoder.read_float()
511 elif writer_schema.type == 'double':
512 return decoder.read_double()
513 elif writer_schema.type == 'bytes':
514 return decoder.read_bytes()
515 elif writer_schema.type == 'fixed':
516 return self.read_fixed(writer_schema, reader_schema, decoder)
517 elif writer_schema.type == 'enum':
518 return self.read_enum(writer_schema, reader_schema, decoder)
519 elif writer_schema.type == 'array':
520 return self.read_array(writer_schema, reader_schema, decoder)
521 elif writer_schema.type == 'map':
522 return self.read_map(writer_schema, reader_schema, decoder)
523 elif writer_schema.type in ['union', 'error_union']:
524 return self.read_union(writer_schema, reader_schema, decoder)
525 elif writer_schema.type in ['record', 'error', 'request']:
526 return self.read_record(writer_schema, reader_schema, decoder)
527 else:
528 fail_msg = "Cannot read unknown schema type: %s" % writer_schema.type
529 raise schema.AvroException(fail_msg)
530
531 def skip_data(self, writer_schema, decoder):
532 if writer_schema.type == 'null':
533 return decoder.skip_null()
534 elif writer_schema.type == 'boolean':
535 return decoder.skip_boolean()
536 elif writer_schema.type == 'string':
537 return decoder.skip_utf8()
538 elif writer_schema.type == 'int':
539 return decoder.skip_int()
540 elif writer_schema.type == 'long':
541 return decoder.skip_long()
542 elif writer_schema.type == 'float':
543 return decoder.skip_float()
544 elif writer_schema.type == 'double':
545 return decoder.skip_double()
546 elif writer_schema.type == 'bytes':
547 return decoder.skip_bytes()
548 elif writer_schema.type == 'fixed':
549 return self.skip_fixed(writer_schema, decoder)
550 elif writer_schema.type == 'enum':
551 return self.skip_enum(writer_schema, decoder)
552 elif writer_schema.type == 'array':
553 return self.skip_array(writer_schema, decoder)
554 elif writer_schema.type == 'map':
555 return self.skip_map(writer_schema, decoder)
556 elif writer_schema.type in ['union', 'error_union']:
557 return self.skip_union(writer_schema, decoder)
558 elif writer_schema.type in ['record', 'error', 'request']:
559 return self.skip_record(writer_schema, decoder)
560 else:
561 fail_msg = "Unknown schema type: %s" % writer_schema.type
562 raise schema.AvroException(fail_msg)
563
564 def read_fixed(self, writer_schema, reader_schema, decoder):
565 """
566 Fixed instances are encoded using the number of bytes declared
567 in the schema.
568 """
569 return decoder.read(writer_schema.size)
570
571 def skip_fixed(self, writer_schema, decoder):
572 return decoder.skip(writer_schema.size)
573
574 def read_enum(self, writer_schema, reader_schema, decoder):
575 """
576 An enum is encoded by a int, representing the zero-based position
577 of the symbol in the schema.
578 """
579 # read data
580 index_of_symbol = decoder.read_int()
581 if index_of_symbol >= len(writer_schema.symbols):
582 fail_msg = "Can't access enum index %d for enum with %d symbols"\
583 % (index_of_symbol, len(writer_schema.symbols))
584 raise SchemaResolutionException(fail_msg, writer_schema, reader_schema)
585 read_symbol = writer_schema.symbols[index_of_symbol]
586
587 # schema resolution
588 if read_symbol not in reader_schema.symbols:
589 fail_msg = "Symbol %s not present in Reader's Schema" % read_symbol
590 raise SchemaResolutionException(fail_msg, writer_schema, reader_schema)
591
592 return read_symbol
593
594 def skip_enum(self, writer_schema, decoder):
595 return decoder.skip_int()
596
597 def read_array(self, writer_schema, reader_schema, decoder):
598 """
599 Arrays are encoded as a series of blocks.
600
601 Each block consists of a long count value,
602 followed by that many array items.
603 A block with count zero indicates the end of the array.
604 Each item is encoded per the array's item schema.
605
606 If a block's count is negative,
607 then the count is followed immediately by a long block size,
608 indicating the number of bytes in the block.
609 The actual count in this case
610 is the absolute value of the count written.
611 """
612 read_items = []
613 block_count = decoder.read_long()
614 while block_count != 0:
615 if block_count < 0:
616 block_count = -block_count
617 block_size = decoder.read_long()
618 for i in range(block_count):
619 read_items.append(self.read_data(writer_schema.items,
620 reader_schema.items, decoder))
621 block_count = decoder.read_long()
622 return read_items
623
624 def skip_array(self, writer_schema, decoder):
625 block_count = decoder.read_long()
626 while block_count != 0:
627 if block_count < 0:
628 block_size = decoder.read_long()
629 decoder.skip(block_size)
630 else:
631 for i in range(block_count):
632 self.skip_data(writer_schema.items, decoder)
633 block_count = decoder.read_long()
634
635 def read_map(self, writer_schema, reader_schema, decoder):
636 """
637 Maps are encoded as a series of blocks.
638
639 Each block consists of a long count value,
640 followed by that many key/value pairs.
641 A block with count zero indicates the end of the map.
642 Each item is encoded per the map's value schema.
643
644 If a block's count is negative,
645 then the count is followed immediately by a long block size,
646 indicating the number of bytes in the block.
647 The actual count in this case
648 is the absolute value of the count written.
649 """
650 read_items = {}
651 block_count = decoder.read_long()
652 while block_count != 0:
653 if block_count < 0:
654 block_count = -block_count
655 block_size = decoder.read_long()
656 for i in range(block_count):
657 key = decoder.read_utf8()
658 read_items[key] = self.read_data(writer_schema.values,
659 reader_schema.values, decoder)
660 block_count = decoder.read_long()
661 return read_items
662
663 def skip_map(self, writer_schema, decoder):
664 block_count = decoder.read_long()
665 while block_count != 0:
666 if block_count < 0:
667 block_size = decoder.read_long()
668 decoder.skip(block_size)
669 else:
670 for i in range(block_count):
671 decoder.skip_utf8()
672 self.skip_data(writer_schema.values, decoder)
673 block_count = decoder.read_long()
674
675 def read_union(self, writer_schema, reader_schema, decoder):
676 """
677 A union is encoded by first writing a long value indicating
678 the zero-based position within the union of the schema of its value.
679 The value is then encoded per the indicated schema within the union.
680 """
681 # schema resolution
682 index_of_schema = int(decoder.read_long())
683 if index_of_schema >= len(writer_schema.schemas):
684 fail_msg = "Can't access branch index %d for union with %d branches"\
685 % (index_of_schema, len(writer_schema.schemas))
686 raise SchemaResolutionException(fail_msg, writer_schema, reader_schema)
687 selected_writer_schema = writer_schema.schemas[index_of_schema]
688
689 # read data
690 return self.read_data(selected_writer_schema, reader_schema, decoder)
691
692 def skip_union(self, writer_schema, decoder):
693 index_of_schema = int(decoder.read_long())
694 if index_of_schema >= len(writer_schema.schemas):
695 fail_msg = "Can't access branch index %d for union with %d branches"\
696 % (index_of_schema, len(writer_schema.schemas))
697 raise SchemaResolutionException(fail_msg, writer_schema)
698 return self.skip_data(writer_schema.schemas[index_of_schema], decoder)
699
700 def read_record(self, writer_schema, reader_schema, decoder):
701 """
702 A record is encoded by encoding the values of its fields
703 in the order that they are declared. In other words, a record
704 is encoded as just the concatenation of the encodings of its fields.
705 Field values are encoded per their schema.
706
707 Schema Resolution:
708 * the ordering of fields may be different: fields are matched by name.
709 * schemas for fields with the same name in both records are resolved
710 recursively.
711 * if the writer's record contains a field with a name not present in the
712 reader's record, the writer's value for that field is ignored.
713 * if the reader's record schema has a field that contains a default value,
714 and writer's schema does not have a field with the same name, then the
715 reader should use the default value from its field.
716 * if the reader's record schema has a field with no default value, and
717 writer's schema does not have a field with the same name, then the
718 field's value is unset.
719 """
720 # schema resolution
721 readers_fields_dict = reader_schema.field_map
722 read_record = {}
723 for field in writer_schema.fields:
724 readers_field = readers_fields_dict.get(field.name)
725 if readers_field is not None:
726 field_val = self.read_data(field.type, readers_field.type, decoder)
727 read_record[field.name] = field_val
728 else:
729 self.skip_data(field.type, decoder)
730
731 # fill in default values
732 if len(readers_fields_dict) > len(read_record):
733 writers_fields_dict = writer_schema.field_map
734 for field_name, field in readers_fields_dict.items():
735 if field_name not in writers_fields_dict:
736 if field.has_default:
737 field_val = self._read_default_value(field.type, field.default)
738 read_record[field.name] = field_val
739 else:
740 fail_msg = 'No default value for field %s' % field_name
741 raise SchemaResolutionException(fail_msg, writer_schema,
742 reader_schema)
743 return read_record
744
745 def skip_record(self, writer_schema, decoder):
746 for field in writer_schema.fields:
747 self.skip_data(field.type, decoder)
748
749 def _read_default_value(self, field_schema, default_value):
750 """
751 Basically a JSON Decoder?
752 """
753 if field_schema.type == 'null':
754 return None
755 elif field_schema.type == 'boolean':
756 return bool(default_value)
757 elif field_schema.type == 'int':
758 return int(default_value)
759 elif field_schema.type == 'long':
760 return int(default_value)
761 elif field_schema.type in ['float', 'double']:
762 return float(default_value)
763 elif field_schema.type in ['enum', 'fixed', 'string', 'bytes']:
764 return default_value
765 elif field_schema.type == 'array':
766 read_array = []
767 for json_val in default_value:
768 item_val = self._read_default_value(field_schema.items, json_val)
769 read_array.append(item_val)
770 return read_array
771 elif field_schema.type == 'map':
772 read_map = {}
773 for key, json_val in default_value.items():
774 map_val = self._read_default_value(field_schema.values, json_val)
775 read_map[key] = map_val
776 return read_map
777 elif field_schema.type in ['union', 'error_union']:
778 return self._read_default_value(field_schema.schemas[0], default_value)
779 elif field_schema.type == 'record':
780 read_record = {}
781 for field in field_schema.fields:
782 json_val = default_value.get(field.name)
783 if json_val is None: json_val = field.default
784 field_val = self._read_default_value(field.type, json_val)
785 read_record[field.name] = field_val
786 return read_record
787 else:
788 fail_msg = 'Unknown type: %s' % field_schema.type
789 raise schema.AvroException(fail_msg)
790
791
792 # ------------------------------------------------------------------------------
793
794
795 class DatumWriter(object):
796 """DatumWriter for generic python objects."""
797 def __init__(self, writer_schema=None):
798 self._writer_schema = writer_schema
799
800 # read/write properties
801 def set_writer_schema(self, writer_schema):
802 self._writer_schema = writer_schema
803 writer_schema = property(lambda self: self._writer_schema,
804 set_writer_schema)
805
806 def write(self, datum, encoder):
807 # validate datum
808 if not Validate(self.writer_schema, datum):
809 raise AvroTypeException(self.writer_schema, datum)
810
811 self.write_data(self.writer_schema, datum, encoder)
812
813 def write_data(self, writer_schema, datum, encoder):
814 # function dispatch to write datum
815 if writer_schema.type == 'null':
816 encoder.write_null(datum)
817 elif writer_schema.type == 'boolean':
818 encoder.write_boolean(datum)
819 elif writer_schema.type == 'string':
820 encoder.write_utf8(datum)
821 elif writer_schema.type == 'int':
822 encoder.write_int(datum)
823 elif writer_schema.type == 'long':
824 encoder.write_long(datum)
825 elif writer_schema.type == 'float':
826 encoder.write_float(datum)
827 elif writer_schema.type == 'double':
828 encoder.write_double(datum)
829 elif writer_schema.type == 'bytes':
830 encoder.write_bytes(datum)
831 elif writer_schema.type == 'fixed':
832 self.write_fixed(writer_schema, datum, encoder)
833 elif writer_schema.type == 'enum':
834 self.write_enum(writer_schema, datum, encoder)
835 elif writer_schema.type == 'array':
836 self.write_array(writer_schema, datum, encoder)
837 elif writer_schema.type == 'map':
838 self.write_map(writer_schema, datum, encoder)
839 elif writer_schema.type in ['union', 'error_union']:
840 self.write_union(writer_schema, datum, encoder)
841 elif writer_schema.type in ['record', 'error', 'request']:
842 self.write_record(writer_schema, datum, encoder)
843 else:
844 fail_msg = 'Unknown type: %s' % writer_schema.type
845 raise schema.AvroException(fail_msg)
846
847 def write_fixed(self, writer_schema, datum, encoder):
848 """
849 Fixed instances are encoded using the number of bytes declared
850 in the schema.
851 """
852 encoder.write(datum)
853
854 def write_enum(self, writer_schema, datum, encoder):
855 """
856 An enum is encoded by a int, representing the zero-based position
857 of the symbol in the schema.
858 """
859 index_of_datum = writer_schema.symbols.index(datum)
860 encoder.write_int(index_of_datum)
861
862 def write_array(self, writer_schema, datum, encoder):
863 """
864 Arrays are encoded as a series of blocks.
865
866 Each block consists of a long count value,
867 followed by that many array items.
868 A block with count zero indicates the end of the array.
869 Each item is encoded per the array's item schema.
870
871 If a block's count is negative,
872 then the count is followed immediately by a long block size,
873 indicating the number of bytes in the block.
874 The actual count in this case
875 is the absolute value of the count written.
876 """
877 if len(datum) > 0:
878 encoder.write_long(len(datum))
879 for item in datum:
880 self.write_data(writer_schema.items, item, encoder)
881 encoder.write_long(0)
882
883 def write_map(self, writer_schema, datum, encoder):
884 """
885 Maps are encoded as a series of blocks.
886
887 Each block consists of a long count value,
888 followed by that many key/value pairs.
889 A block with count zero indicates the end of the map.
890 Each item is encoded per the map's value schema.
891
892 If a block's count is negative,
893 then the count is followed immediately by a long block size,
894 indicating the number of bytes in the block.
895 The actual count in this case
896 is the absolute value of the count written.
897 """
898 if len(datum) > 0:
899 encoder.write_long(len(datum))
900 for key, val in datum.items():
901 encoder.write_utf8(key)
902 self.write_data(writer_schema.values, val, encoder)
903 encoder.write_long(0)
904
905 def write_union(self, writer_schema, datum, encoder):
906 """
907 A union is encoded by first writing a long value indicating
908 the zero-based position within the union of the schema of its value.
909 The value is then encoded per the indicated schema within the union.
910 """
911 # resolve union
912 index_of_schema = -1
913 for i, candidate_schema in enumerate(writer_schema.schemas):
914 if Validate(candidate_schema, datum):
915 index_of_schema = i
916 if index_of_schema < 0: raise AvroTypeException(writer_schema, datum)
917
918 # write data
919 encoder.write_long(index_of_schema)
920 self.write_data(writer_schema.schemas[index_of_schema], datum, encoder)
921
922 def write_record(self, writer_schema, datum, encoder):
923 """
924 A record is encoded by encoding the values of its fields
925 in the order that they are declared. In other words, a record
926 is encoded as just the concatenation of the encodings of its fields.
927 Field values are encoded per their schema.
928 """
929 for field in writer_schema.fields:
930 self.write_data(field.type, datum.get(field.name), encoder)
931
932
933 if __name__ == '__main__':
934 raise Exception('Not a standalone module')