Debian Codegraph @

lang/py3/avro/tests/test_io.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 import binascii
22 import io
23 import logging
24 import sys
25 import unittest
26
27 from avro import io as avro_io
28 from avro import schema
29
30
31 SCHEMAS_TO_VALIDATE = (
32 ('"null"', None),
33 ('"boolean"', True),
34 ('"string"', 'adsfasdf09809dsf-=adsf'),
35 ('"bytes"', b'12345abcd'),
36 ('"int"', 1234),
37 ('"long"', 1234),
38 ('"float"', 1234.0),
39 ('"double"', 1234.0),
40 ('{"type": "fixed", "name": "Test", "size": 1}', b'B'),
41 ('{"type": "enum", "name": "Test", "symbols": ["A", "B"]}', 'B'),
42 ('{"type": "array", "items": "long"}', [1, 3, 2]),
43 ('{"type": "map", "values": "long"}', {'a': 1, 'b': 3, 'c': 2}),
44 ('["string", "null", "long"]', None),
45 ("""\
46 {"type": "record",
47 "name": "Test",
48 "fields": [{"name": "f", "type": "long"}]}
49 """, {'f': 5}),
50 ("""
51 {
52 "type": "record",
53 "name": "Lisp",
54 "fields": [{
55 "name": "value",
56 "type": [
57 "null",
58 "string",
59 {
60 "type": "record",
61 "name": "Cons",
62 "fields": [{"name": "car", "type": "Lisp"},
63 {"name": "cdr", "type": "Lisp"}]
64 }
65 ]
66 }]
67 }
68 """, {'value': {'car': {'value': 'head'}, 'cdr': {'value': None}}}),
69 )
70
71 BINARY_ENCODINGS = (
72 (0, '00'),
73 (-1, '01'),
74 (1, '02'),
75 (-2, '03'),
76 (2, '04'),
77 (-64, '7f'),
78 (64, '80 01'),
79 (8192, '80 80 01'),
80 (-8193, '81 80 01'),
81 )
82
83 DEFAULT_VALUE_EXAMPLES = (
84 ('"null"', 'null', None),
85 ('"boolean"', 'true', True),
86 ('"string"', '"foo"', 'foo'),
87 ('"bytes"', '"\u00FF\u00FF"', '\xff\xff'),
88 ('"int"', '5', 5),
89 ('"long"', '5', 5),
90 ('"float"', '1.1', 1.1),
91 ('"double"', '1.1', 1.1),
92 ('{"type": "fixed", "name": "F", "size": 2}', '"\u00FF\u00FF"', '\xff\xff'),
93 ('{"type": "enum", "name": "F", "symbols": ["FOO", "BAR"]}', '"FOO"', 'FOO'),
94 ('{"type": "array", "items": "int"}', '[1, 2, 3]', [1, 2, 3]),
95 ('{"type": "map", "values": "int"}', '{"a": 1, "b": 2}', {'a': 1, 'b': 2}),
96 ('["int", "null"]', '5', 5),
97 ('{"type": "record", "name": "F", "fields": [{"name": "A", "type": "int"}]}',
98 '{"A": 5}', {'A': 5}),
99 )
100
101 LONG_RECORD_SCHEMA = schema.Parse("""
102 {
103 "type": "record",
104 "name": "Test",
105 "fields": [
106 {"name": "A", "type": "int"},
107 {"name": "B", "type": "int"},
108 {"name": "C", "type": "int"},
109 {"name": "D", "type": "int"},
110 {"name": "E", "type": "int"},
111 {"name": "F", "type": "int"},
112 {"name": "G", "type": "int"}
113 ]
114 }
115 """)
116
117 LONG_RECORD_DATUM = {'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7}
118
119
120 def avro_hexlify(reader):
121 """Return the hex value, as a string, of a binary-encoded int or long."""
122 bytes = []
123 current_byte = reader.read(1)
124 bytes.append(binascii.hexlify(current_byte).decode())
125 while (ord(current_byte) & 0x80) != 0:
126 current_byte = reader.read(1)
127 bytes.append(binascii.hexlify(current_byte).decode())
128 return ' '.join(bytes)
129
130
131 def write_datum(datum, writer_schema):
132 writer = io.BytesIO()
133 encoder = avro_io.BinaryEncoder(writer)
134 datum_writer = avro_io.DatumWriter(writer_schema)
135 datum_writer.write(datum, encoder)
136 return writer, encoder, datum_writer
137
138
139 def read_datum(buffer, writer_schema, reader_schema=None):
140 reader = io.BytesIO(buffer.getvalue())
141 decoder = avro_io.BinaryDecoder(reader)
142 datum_reader = avro_io.DatumReader(writer_schema, reader_schema)
143 return datum_reader.read(decoder)
144
145
146 def check_binary_encoding(number_type):
147 logging.debug('Testing binary encoding for type %s', number_type)
148 correct = 0
149 for datum, hex_encoding in BINARY_ENCODINGS:
150 logging.debug('Datum: %d', datum)
151 logging.debug('Correct Encoding: %s', hex_encoding)
152
153 writer_schema = schema.Parse('"%s"' % number_type.lower())
154 writer, encoder, datum_writer = write_datum(datum, writer_schema)
155 writer.seek(0)
156 hex_val = avro_hexlify(writer)
157
158 logging.debug('Read Encoding: %s', hex_val)
159 if hex_encoding == hex_val: correct += 1
160 return correct
161
162
163 def check_skip_number(number_type):
164 logging.debug('Testing skip number for %s', number_type)
165 correct = 0
166 for value_to_skip, hex_encoding in BINARY_ENCODINGS:
167 VALUE_TO_READ = 6253
168 logging.debug('Value to Skip: %d', value_to_skip)
169
170 # write the value to skip and a known value
171 writer_schema = schema.Parse('"%s"' % number_type.lower())
172 writer, encoder, datum_writer = write_datum(value_to_skip, writer_schema)
173 datum_writer.write(VALUE_TO_READ, encoder)
174
175 # skip the value
176 reader = io.BytesIO(writer.getvalue())
177 decoder = avro_io.BinaryDecoder(reader)
178 decoder.skip_long()
179
180 # read data from string buffer
181 datum_reader = avro_io.DatumReader(writer_schema)
182 read_value = datum_reader.read(decoder)
183
184 logging.debug('Read Value: %d', read_value)
185 if read_value == VALUE_TO_READ: correct += 1
186 return correct
187
188
189 # ------------------------------------------------------------------------------
190
191
192 class TestIO(unittest.TestCase):
193 #
194 # BASIC FUNCTIONALITY
195 #
196
197 def testValidate(self):
198 passed = 0
199 for example_schema, datum in SCHEMAS_TO_VALIDATE:
200 logging.debug('Schema: %r', example_schema)
201 logging.debug('Datum: %r', datum)
202 validated = avro_io.Validate(schema.Parse(example_schema), datum)
203 logging.debug('Valid: %s', validated)
204 if validated: passed += 1
205 self.assertEqual(passed, len(SCHEMAS_TO_VALIDATE))
206
207 def testRoundTrip(self):
208 correct = 0
209 for example_schema, datum in SCHEMAS_TO_VALIDATE:
210 logging.debug('Schema: %s', example_schema)
211 logging.debug('Datum: %s', datum)
212
213 writer_schema = schema.Parse(example_schema)
214 writer, encoder, datum_writer = write_datum(datum, writer_schema)
215 round_trip_datum = read_datum(writer, writer_schema)
216
217 logging.debug('Round Trip Datum: %s', round_trip_datum)
218 if datum == round_trip_datum: correct += 1
219 self.assertEqual(correct, len(SCHEMAS_TO_VALIDATE))
220
221 #
222 # BINARY ENCODING OF INT AND LONG
223 #
224
225 def testBinaryIntEncoding(self):
226 correct = check_binary_encoding('int')
227 self.assertEqual(correct, len(BINARY_ENCODINGS))
228
229 def testBinaryLongEncoding(self):
230 correct = check_binary_encoding('long')
231 self.assertEqual(correct, len(BINARY_ENCODINGS))
232
233 def testSkipInt(self):
234 correct = check_skip_number('int')
235 self.assertEqual(correct, len(BINARY_ENCODINGS))
236
237 def testSkipLong(self):
238 correct = check_skip_number('long')
239 self.assertEqual(correct, len(BINARY_ENCODINGS))
240
241 #
242 # SCHEMA RESOLUTION
243 #
244
245 def testSchemaPromotion(self):
246 # note that checking writer_schema.type in read_data
247 # allows us to handle promotion correctly
248 promotable_schemas = ['"int"', '"long"', '"float"', '"double"']
249 incorrect = 0
250 for i, ws in enumerate(promotable_schemas):
251 writer_schema = schema.Parse(ws)
252 datum_to_write = 219
253 for rs in promotable_schemas[i + 1:]:
254 reader_schema = schema.Parse(rs)
255 writer, enc, dw = write_datum(datum_to_write, writer_schema)
256 datum_read = read_datum(writer, writer_schema, reader_schema)
257 logging.debug('Writer: %s Reader: %s', writer_schema, reader_schema)
258 logging.debug('Datum Read: %s', datum_read)
259 if datum_read != datum_to_write: incorrect += 1
260 self.assertEqual(incorrect, 0)
261
262 def testUnknownSymbol(self):
263 writer_schema = schema.Parse("""\
264 {"type": "enum", "name": "Test",
265 "symbols": ["FOO", "BAR"]}""")
266 datum_to_write = 'FOO'
267
268 reader_schema = schema.Parse("""\
269 {"type": "enum", "name": "Test",
270 "symbols": ["BAR", "BAZ"]}""")
271
272 writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema)
273 reader = io.BytesIO(writer.getvalue())
274 decoder = avro_io.BinaryDecoder(reader)
275 datum_reader = avro_io.DatumReader(writer_schema, reader_schema)
276 self.assertRaises(avro_io.SchemaResolutionException, datum_reader.read, decoder)
277
278 def testDefaultValue(self):
279 writer_schema = LONG_RECORD_SCHEMA
280 datum_to_write = LONG_RECORD_DATUM
281
282 correct = 0
283 for field_type, default_json, default_datum in DEFAULT_VALUE_EXAMPLES:
284 reader_schema = schema.Parse("""\
285 {"type": "record", "name": "Test",
286 "fields": [{"name": "H", "type": %s, "default": %s}]}
287 """ % (field_type, default_json))
288 datum_to_read = {'H': default_datum}
289
290 writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema)
291 datum_read = read_datum(writer, writer_schema, reader_schema)
292 logging.debug('Datum Read: %s', datum_read)
293 if datum_to_read == datum_read: correct += 1
294 self.assertEqual(correct, len(DEFAULT_VALUE_EXAMPLES))
295
296 def testNoDefaultValue(self):
297 writer_schema = LONG_RECORD_SCHEMA
298 datum_to_write = LONG_RECORD_DATUM
299
300 reader_schema = schema.Parse("""\
301 {"type": "record", "name": "Test",
302 "fields": [{"name": "H", "type": "int"}]}""")
303
304 writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema)
305 reader = io.BytesIO(writer.getvalue())
306 decoder = avro_io.BinaryDecoder(reader)
307 datum_reader = avro_io.DatumReader(writer_schema, reader_schema)
308 self.assertRaises(avro_io.SchemaResolutionException, datum_reader.read, decoder)
309
310 def testProjection(self):
311 writer_schema = LONG_RECORD_SCHEMA
312 datum_to_write = LONG_RECORD_DATUM
313
314 reader_schema = schema.Parse("""\
315 {"type": "record", "name": "Test",
316 "fields": [{"name": "E", "type": "int"},
317 {"name": "F", "type": "int"}]}""")
318 datum_to_read = {'E': 5, 'F': 6}
319
320 writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema)
321 datum_read = read_datum(writer, writer_schema, reader_schema)
322 logging.debug('Datum Read: %s', datum_read)
323 self.assertEqual(datum_to_read, datum_read)
324
325 def testFieldOrder(self):
326 writer_schema = LONG_RECORD_SCHEMA
327 datum_to_write = LONG_RECORD_DATUM
328
329 reader_schema = schema.Parse("""\
330 {"type": "record", "name": "Test",
331 "fields": [{"name": "F", "type": "int"},
332 {"name": "E", "type": "int"}]}""")
333 datum_to_read = {'E': 5, 'F': 6}
334
335 writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema)
336 datum_read = read_datum(writer, writer_schema, reader_schema)
337 logging.debug('Datum Read: %s', datum_read)
338 self.assertEqual(datum_to_read, datum_read)
339
340 def testTypeException(self):
341 writer_schema = schema.Parse("""\
342 {"type": "record", "name": "Test",
343 "fields": [{"name": "F", "type": "int"},
344 {"name": "E", "type": "int"}]}""")
345 datum_to_write = {'E': 5, 'F': 'Bad'}
346 self.assertRaises(
347 avro_io.AvroTypeException, write_datum, datum_to_write, writer_schema)
348
349
350 if __name__ == '__main__':
351 raise Exception('Use run_tests.py')