Debian Codegraph @

lang/py3/avro/datafile.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 """Read/Write Avro File Object Containers."""
22
23 import io
24 import logging
25 import os
26 import zlib
27
28 from avro import schema
29 from avro import io as avro_io
30
31 try:
32 import snappy
33 has_snappy = True
34 except ImportError:
35 has_snappy = False
36
37
38 logger = logging.getLogger(__name__)
39
40 # ------------------------------------------------------------------------------
41 # Constants
42
43 # Version of the container file:
44 VERSION = 1
45
46 # Magic code that starts a data container file:
47 MAGIC = b'Obj' + bytes([VERSION])
48
49 # Size of the magic code, in number of bytes:
50 MAGIC_SIZE = len(MAGIC)
51
52 # Size of the synchronization marker, in number of bytes:
53 SYNC_SIZE = 16
54
55 # Interval between synchronization markers, in number of bytes:
56 # TODO: make configurable
57 SYNC_INTERVAL = 1000 * SYNC_SIZE
58
59 # Schema of the container header:
60 META_SCHEMA = schema.Parse("""
61 {
62 "type": "record", "name": "org.apache.avro.file.Header",
63 "fields": [{
64 "name": "magic",
65 "type": {"type": "fixed", "name": "magic", "size": %(magic_size)d}
66 }, {
67 "name": "meta",
68 "type": {"type": "map", "values": "bytes"}
69 }, {
70 "name": "sync",
71 "type": {"type": "fixed", "name": "sync", "size": %(sync_size)d}
72 }]
73 }
74 """ % {
75 'magic_size': MAGIC_SIZE,
76 'sync_size': SYNC_SIZE,
77 })
78
79 # Codecs supported by container files:
80 VALID_CODECS = frozenset(['null', 'deflate'])
81 if has_snappy:
82 VALID_CODECS = frozenset.union(VALID_CODECS, ['snappy'])
83
84 # Not used yet
85 VALID_ENCODINGS = frozenset(['binary'])
86
87 # Metadata key associated to the codec:
88 CODEC_KEY = "avro.codec"
89
90 # Metadata key associated to the schema:
91 SCHEMA_KEY = "avro.schema"
92
93
94 # ------------------------------------------------------------------------------
95 # Exceptions
96
97
98 class DataFileException(schema.AvroException):
99 """Problem reading or writing file object containers."""
100
101 def __init__(self, msg):
102 super(DataFileException, self).__init__(msg)
103
104
105 # ------------------------------------------------------------------------------
106
107
108 class DataFileWriter(object):
109 """Writes Avro data files."""
110
111 @staticmethod
112 def GenerateSyncMarker():
113 """Generates a random synchronization marker."""
114 return os.urandom(SYNC_SIZE)
115
116 # TODO: make 'encoder' a metadata property
117 def __init__(
118 self,
119 writer,
120 datum_writer,
121 writer_schema=None,
122 codec='null',
123 ):
124 """Constructs a new DataFileWriter instance.
125
126 If the schema is not present, presume we're appending.
127
128 Args:
129 writer: File-like object to write into.
130 datum_writer:
131 writer_schema: Schema
132 codec:
133 """
134 self._writer = writer
135 self._encoder = avro_io.BinaryEncoder(writer)
136 self._datum_writer = datum_writer
137 self._buffer_writer = io.BytesIO()
138 self._buffer_encoder = avro_io.BinaryEncoder(self._buffer_writer)
139 self._block_count = 0
140 self._meta = {}
141
142 # Ensure we have a writer that accepts bytes:
143 self._writer.write(b'')
144
145 # Whether the header has already been written:
146 self._header_written = False
147
148 if writer_schema is not None:
149 if codec not in VALID_CODECS:
150 raise DataFileException('Unknown codec: %r' % codec)
151 self._sync_marker = DataFileWriter.GenerateSyncMarker()
152 self.SetMeta('avro.codec', codec)
153 self.SetMeta('avro.schema', str(writer_schema).encode('utf-8'))
154 self.datum_writer.writer_schema = writer_schema
155 else:
156 # open writer for reading to collect metadata
157 dfr = DataFileReader(writer, avro_io.DatumReader())
158
159 # TODO: collect arbitrary metadata
160 # collect metadata
161 self._sync_marker = dfr.sync_marker
162 self.SetMeta('avro.codec', dfr.GetMeta('avro.codec'))
163
164 # get schema used to write existing file
165 schema_from_file = dfr.GetMeta('avro.schema').decode('utf-8')
166 self.SetMeta('avro.schema', schema_from_file)
167 self.datum_writer.writer_schema = schema.Parse(schema_from_file)
168
169 # seek to the end of the file and prepare for writing
170 writer.seek(0, 2)
171 self._header_written = True
172
173 # read-only properties
174
175 @property
176 def writer(self):
177 return self._writer
178
179 @property
180 def encoder(self):
181 return self._encoder
182
183 @property
184 def datum_writer(self):
185 return self._datum_writer
186
187 @property
188 def buffer_encoder(self):
189 return self._buffer_encoder
190
191 @property
192 def sync_marker(self):
193 return self._sync_marker
194
195 @property
196 def meta(self):
197 return self._meta
198
199 def __enter__(self):
200 return self
201
202 def __exit__(self, type, value, traceback):
203 # Perform a close if there's no exception
204 if type is None:
205 self.close()
206
207 @property
208 def block_count(self):
209 return self._block_count
210
211 def GetMeta(self, key):
212 """Reports the metadata associated to the given key.
213
214 Args:
215 key: Key of the metadata to report the value of.
216 Returns:
217 The metadata value, as bytes, or None if the key does not exist.
218 """
219 return self._meta.get(key)
220
221 def SetMeta(self, key, value):
222 """Sets the metadata value for the given key.
223
224 Note: metadata is persisted and retrieved as bytes.
225
226 Args:
227 key: Key of the metadata to set.
228 value: Value of the metadata, as bytes or str.
229 Strings are automatically converted to bytes.
230 """
231 if isinstance(value, str):
232 value = value.encode('utf-8')
233 assert isinstance(value, bytes), (
234 'Invalid metadata value for key %r: %r' % (key, value))
235 self._meta[key] = value
236
237 def _WriteHeader(self):
238 header = {
239 'magic': MAGIC,
240 'meta': self.meta,
241 'sync': self.sync_marker,
242 }
243 logger.debug(
244 'Writing Avro data file header:\n%s\nAvro header schema:\n%s',
245 header, META_SCHEMA)
246 self.datum_writer.write_data(META_SCHEMA, header, self.encoder)
247 self._header_written = True
248
249 # TODO: make a schema for blocks and use datum_writer
250 def _WriteBlock(self):
251 if not self._header_written:
252 self._WriteHeader()
253
254 if self.block_count <= 0:
255 logger.info('Current block is empty, nothing to write.')
256 return
257
258 # write number of items in block
259 self.encoder.write_long(self.block_count)
260
261 # write block contents
262 uncompressed_data = self._buffer_writer.getvalue()
263 codec = self.GetMeta(CODEC_KEY).decode('utf-8')
264 if codec == 'null':
265 compressed_data = uncompressed_data
266 compressed_data_length = len(compressed_data)
267 elif codec == 'deflate':
268 # The first two characters and last character are zlib
269 # wrappers around deflate data.
270 compressed_data = zlib.compress(uncompressed_data)[2:-1]
271 compressed_data_length = len(compressed_data)
272 elif codec == 'snappy':
273 compressed_data = snappy.compress(uncompressed_data)
274 compressed_data_length = len(compressed_data) + 4 # crc32
275 else:
276 fail_msg = '"%s" codec is not supported.' % codec
277 raise DataFileException(fail_msg)
278
279 # Write length of block
280 self.encoder.write_long(compressed_data_length)
281
282 # Write block
283 self.writer.write(compressed_data)
284
285 # Write CRC32 checksum for Snappy
286 if self.GetMeta(CODEC_KEY) == 'snappy':
287 self.encoder.write_crc32(uncompressed_data)
288
289 # write sync marker
290 self.writer.write(self.sync_marker)
291
292 logger.debug(
293 'Writing block with count=%d nbytes=%d sync=%r',
294 self.block_count, compressed_data_length, self.sync_marker)
295
296 # reset buffer
297 self._buffer_writer.seek(0)
298 self._buffer_writer.truncate()
299 self._block_count = 0
300
301 def append(self, datum):
302 """Append a datum to the file."""
303 self.datum_writer.write(datum, self.buffer_encoder)
304 self._block_count += 1
305
306 # if the data to write is larger than the sync interval, write the block
307 if self._buffer_writer.tell() >= SYNC_INTERVAL:
308 self._WriteBlock()
309
310 def sync(self):
311 """
312 Return the current position as a value that may be passed to
313 DataFileReader.seek(long). Forces the end of the current block,
314 emitting a synchronization marker.
315 """
316 self._WriteBlock()
317 return self.writer.tell()
318
319 def flush(self):
320 """Flush the current state of the file, including metadata."""
321 self._WriteBlock()
322 self.writer.flush()
323
324 def close(self):
325 """Close the file."""
326 self.flush()
327 self.writer.close()
328
329
330 # ------------------------------------------------------------------------------
331
332
333 class DataFileReader(object):
334 """Read files written by DataFileWriter."""
335
336 # TODO: allow user to specify expected schema?
337 # TODO: allow user to specify the encoder
338 def __init__(self, reader, datum_reader):
339 """Initializes a new data file reader.
340
341 Args:
342 reader: Open file to read from.
343 datum_reader: Avro datum reader.
344 """
345 self._reader = reader
346 self._raw_decoder = avro_io.BinaryDecoder(reader)
347 self._datum_decoder = None # Maybe reset at every block.
348 self._datum_reader = datum_reader
349
350 # read the header: magic, meta, sync
351 self._read_header()
352
353 # ensure codec is valid
354 avro_codec_raw = self.GetMeta('avro.codec')
355 if avro_codec_raw is None:
356 self.codec = "null"
357 else:
358 self.codec = avro_codec_raw.decode('utf-8')
359 if self.codec not in VALID_CODECS:
360 raise DataFileException('Unknown codec: %s.' % self.codec)
361
362 self._file_length = self._GetInputFileLength()
363
364 # get ready to read
365 self._block_count = 0
366 self.datum_reader.writer_schema = (
367 schema.Parse(self.GetMeta(SCHEMA_KEY).decode('utf-8')))
368
369 def __enter__(self):
370 return self
371
372 def __exit__(self, type, value, traceback):
373 # Perform a close if there's no exception
374 if type is None:
375 self.close()
376
377 def __iter__(self):
378 return self
379
380 def __next__(self):
381 """Implements the iterator interface."""
382 return next(self)
383
384 # read-only properties
385 @property
386 def reader(self):
387 return self._reader
388
389 @property
390 def raw_decoder(self):
391 return self._raw_decoder
392
393 @property
394 def datum_decoder(self):
395 return self._datum_decoder
396
397 @property
398 def datum_reader(self):
399 return self._datum_reader
400
401 @property
402 def sync_marker(self):
403 return self._sync_marker
404
405 @property
406 def meta(self):
407 return self._meta
408
409 @property
410 def file_length(self):
411 """Length of the input file, in bytes."""
412 return self._file_length
413
414 # read/write properties
415 @property
416 def block_count(self):
417 return self._block_count
418
419 def GetMeta(self, key):
420 """Reports the value of a given metadata key.
421
422 Args:
423 key: Metadata key (string) to report the value of.
424 Returns:
425 Value associated to the metadata key, as bytes.
426 """
427 return self._meta.get(key)
428
429 def SetMeta(self, key, value):
430 """Sets a metadata.
431
432 Args:
433 key: Metadata key (string) to set.
434 value: Metadata value to set, as bytes.
435 """
436 if isinstance(value, str):
437 value = value.encode('utf-8')
438 self._meta[key] = value
439
440 def _GetInputFileLength(self):
441 """Reports the length of the input file, in bytes.
442
443 Leaves the current position unmodified.
444
445 Returns:
446 The length of the input file, in bytes.
447 """
448 current_pos = self.reader.tell()
449 self.reader.seek(0, 2)
450 file_length = self.reader.tell()
451 self.reader.seek(current_pos)
452 return file_length
453
454 def is_EOF(self):
455 return self.reader.tell() == self.file_length
456
457 def _read_header(self):
458 # seek to the beginning of the file to get magic block
459 self.reader.seek(0, 0)
460
461 # read header into a dict
462 header = self.datum_reader.read_data(
463 META_SCHEMA, META_SCHEMA, self.raw_decoder)
464
465 # check magic number
466 if header.get('magic') != MAGIC:
467 fail_msg = "Not an Avro data file: %s doesn't match %s."\
468 % (header.get('magic'), MAGIC)
469 raise schema.AvroException(fail_msg)
470
471 # set metadata
472 self._meta = header['meta']
473
474 # set sync marker
475 self._sync_marker = header['sync']
476
477 def _read_block_header(self):
478 self._block_count = self.raw_decoder.read_long()
479 if self.codec == "null":
480 # Skip a long; we don't need to use the length.
481 self.raw_decoder.skip_long()
482 self._datum_decoder = self._raw_decoder
483 elif self.codec == 'deflate':
484 # Compressed data is stored as (length, data), which
485 # corresponds to how the "bytes" type is encoded.
486 data = self.raw_decoder.read_bytes()
487 # -15 is the log of the window size; negative indicates
488 # "raw" (no zlib headers) decompression. See zlib.h.
489 uncompressed = zlib.decompress(data, -15)
490 self._datum_decoder = avro_io.BinaryDecoder(io.BytesIO(uncompressed))
491 elif self.codec == 'snappy':
492 # Compressed data includes a 4-byte CRC32 checksum
493 length = self.raw_decoder.read_long()
494 data = self.raw_decoder.read(length - 4)
495 uncompressed = snappy.decompress(data)
496 self._datum_decoder = avro_io.BinaryDecoder(io.BytesIO(uncompressed))
497 self.raw_decoder.check_crc32(uncompressed);
498 else:
499 raise DataFileException("Unknown codec: %r" % self.codec)
500
501 def _skip_sync(self):
502 """
503 Read the length of the sync marker; if it matches the sync marker,
504 return True. Otherwise, seek back to where we started and return False.
505 """
506 proposed_sync_marker = self.reader.read(SYNC_SIZE)
507 if proposed_sync_marker != self.sync_marker:
508 self.reader.seek(-SYNC_SIZE, 1)
509 return False
510 else:
511 return True
512
513 # TODO: handle block of length zero
514 # TODO: clean this up with recursion
515 def __next__(self):
516 """Return the next datum in the file."""
517 if self.block_count == 0:
518 if self.is_EOF():
519 raise StopIteration
520 elif self._skip_sync():
521 if self.is_EOF(): raise StopIteration
522 self._read_block_header()
523 else:
524 self._read_block_header()
525
526 datum = self.datum_reader.read(self.datum_decoder)
527 self._block_count -= 1
528 return datum
529
530 def close(self):
531 """Close this reader."""
532 self.reader.close()
533
534
535 if __name__ == '__main__':
536 raise Exception('Not a standalone module')