| 1 |
#!/usr/bin/env python3 |
| 2 |
# -*- mode: python -*- |
| 3 |
# -*- coding: utf-8 -*- |
| 4 |
|
| 5 |
# Licensed to the Apache Software Foundation (ASF) under one |
| 6 |
# or more contributor license agreements. See the NOTICE file |
| 7 |
# distributed with this work for additional information |
| 8 |
# regarding copyright ownership. The ASF licenses this file |
| 9 |
# to you under the Apache License, Version 2.0 (the |
| 10 |
# "License"); you may not use this file except in compliance |
| 11 |
# with the License. You may obtain a copy of the License at |
| 12 |
# |
| 13 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 14 |
# |
| 15 |
# Unless required by applicable law or agreed to in writing, software |
| 16 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 17 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 18 |
# See the License for the specific language governing permissions and |
| 19 |
# limitations under the License. |
| 20 |
|
| 21 |
import binascii |
| 22 |
import io |
| 23 |
import logging |
| 24 |
import sys |
| 25 |
import unittest |
| 26 |
|
| 27 |
from avro import io as avro_io |
| 28 |
from avro import schema |
| 29 |
|
| 30 |
|
| 31 |
SCHEMAS_TO_VALIDATE = ( |
| 32 |
('"null"', None), |
| 33 |
('"boolean"', True), |
| 34 |
('"string"', 'adsfasdf09809dsf-=adsf'), |
| 35 |
('"bytes"', b'12345abcd'), |
| 36 |
('"int"', 1234), |
| 37 |
('"long"', 1234), |
| 38 |
('"float"', 1234.0), |
| 39 |
('"double"', 1234.0), |
| 40 |
('{"type": "fixed", "name": "Test", "size": 1}', b'B'), |
| 41 |
('{"type": "enum", "name": "Test", "symbols": ["A", "B"]}', 'B'), |
| 42 |
('{"type": "array", "items": "long"}', [1, 3, 2]), |
| 43 |
('{"type": "map", "values": "long"}', {'a': 1, 'b': 3, 'c': 2}), |
| 44 |
('["string", "null", "long"]', None), |
| 45 |
("""\ |
| 46 |
{"type": "record", |
| 47 |
"name": "Test", |
| 48 |
"fields": [{"name": "f", "type": "long"}]} |
| 49 |
""", {'f': 5}), |
| 50 |
(""" |
| 51 |
{ |
| 52 |
"type": "record", |
| 53 |
"name": "Lisp", |
| 54 |
"fields": [{ |
| 55 |
"name": "value", |
| 56 |
"type": [ |
| 57 |
"null", |
| 58 |
"string", |
| 59 |
{ |
| 60 |
"type": "record", |
| 61 |
"name": "Cons", |
| 62 |
"fields": [{"name": "car", "type": "Lisp"}, |
| 63 |
{"name": "cdr", "type": "Lisp"}] |
| 64 |
} |
| 65 |
] |
| 66 |
}] |
| 67 |
} |
| 68 |
""", {'value': {'car': {'value': 'head'}, 'cdr': {'value': None}}}), |
| 69 |
) |
| 70 |
|
| 71 |
BINARY_ENCODINGS = ( |
| 72 |
(0, '00'), |
| 73 |
(-1, '01'), |
| 74 |
(1, '02'), |
| 75 |
(-2, '03'), |
| 76 |
(2, '04'), |
| 77 |
(-64, '7f'), |
| 78 |
(64, '80 01'), |
| 79 |
(8192, '80 80 01'), |
| 80 |
(-8193, '81 80 01'), |
| 81 |
) |
| 82 |
|
| 83 |
DEFAULT_VALUE_EXAMPLES = ( |
| 84 |
('"null"', 'null', None), |
| 85 |
('"boolean"', 'true', True), |
| 86 |
('"string"', '"foo"', 'foo'), |
| 87 |
('"bytes"', '"\u00FF\u00FF"', '\xff\xff'), |
| 88 |
('"int"', '5', 5), |
| 89 |
('"long"', '5', 5), |
| 90 |
('"float"', '1.1', 1.1), |
| 91 |
('"double"', '1.1', 1.1), |
| 92 |
('{"type": "fixed", "name": "F", "size": 2}', '"\u00FF\u00FF"', '\xff\xff'), |
| 93 |
('{"type": "enum", "name": "F", "symbols": ["FOO", "BAR"]}', '"FOO"', 'FOO'), |
| 94 |
('{"type": "array", "items": "int"}', '[1, 2, 3]', [1, 2, 3]), |
| 95 |
('{"type": "map", "values": "int"}', '{"a": 1, "b": 2}', {'a': 1, 'b': 2}), |
| 96 |
('["int", "null"]', '5', 5), |
| 97 |
('{"type": "record", "name": "F", "fields": [{"name": "A", "type": "int"}]}', |
| 98 |
'{"A": 5}', {'A': 5}), |
| 99 |
) |
| 100 |
|
| 101 |
LONG_RECORD_SCHEMA = schema.Parse(""" |
| 102 |
{ |
| 103 |
"type": "record", |
| 104 |
"name": "Test", |
| 105 |
"fields": [ |
| 106 |
{"name": "A", "type": "int"}, |
| 107 |
{"name": "B", "type": "int"}, |
| 108 |
{"name": "C", "type": "int"}, |
| 109 |
{"name": "D", "type": "int"}, |
| 110 |
{"name": "E", "type": "int"}, |
| 111 |
{"name": "F", "type": "int"}, |
| 112 |
{"name": "G", "type": "int"} |
| 113 |
] |
| 114 |
} |
| 115 |
""") |
| 116 |
|
| 117 |
LONG_RECORD_DATUM = {'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7} |
| 118 |
|
| 119 |
|
| 120 |
def avro_hexlify(reader): |
| 121 |
"""Return the hex value, as a string, of a binary-encoded int or long.""" |
| 122 |
bytes = [] |
| 123 |
current_byte = reader.read(1) |
| 124 |
bytes.append(binascii.hexlify(current_byte).decode()) |
| 125 |
while (ord(current_byte) & 0x80) != 0: |
| 126 |
current_byte = reader.read(1) |
| 127 |
bytes.append(binascii.hexlify(current_byte).decode()) |
| 128 |
return ' '.join(bytes) |
| 129 |
|
| 130 |
|
| 131 |
def write_datum(datum, writer_schema): |
| 132 |
writer = io.BytesIO() |
| 133 |
encoder = avro_io.BinaryEncoder(writer) |
| 134 |
datum_writer = avro_io.DatumWriter(writer_schema) |
| 135 |
datum_writer.write(datum, encoder) |
| 136 |
return writer, encoder, datum_writer |
| 137 |
|
| 138 |
|
| 139 |
def read_datum(buffer, writer_schema, reader_schema=None): |
| 140 |
reader = io.BytesIO(buffer.getvalue()) |
| 141 |
decoder = avro_io.BinaryDecoder(reader) |
| 142 |
datum_reader = avro_io.DatumReader(writer_schema, reader_schema) |
| 143 |
return datum_reader.read(decoder) |
| 144 |
|
| 145 |
|
| 146 |
def check_binary_encoding(number_type): |
| 147 |
logging.debug('Testing binary encoding for type %s', number_type) |
| 148 |
correct = 0 |
| 149 |
for datum, hex_encoding in BINARY_ENCODINGS: |
| 150 |
logging.debug('Datum: %d', datum) |
| 151 |
logging.debug('Correct Encoding: %s', hex_encoding) |
| 152 |
|
| 153 |
writer_schema = schema.Parse('"%s"' % number_type.lower()) |
| 154 |
writer, encoder, datum_writer = write_datum(datum, writer_schema) |
| 155 |
writer.seek(0) |
| 156 |
hex_val = avro_hexlify(writer) |
| 157 |
|
| 158 |
logging.debug('Read Encoding: %s', hex_val) |
| 159 |
if hex_encoding == hex_val: correct += 1 |
| 160 |
return correct |
| 161 |
|
| 162 |
|
| 163 |
def check_skip_number(number_type): |
| 164 |
logging.debug('Testing skip number for %s', number_type) |
| 165 |
correct = 0 |
| 166 |
for value_to_skip, hex_encoding in BINARY_ENCODINGS: |
| 167 |
VALUE_TO_READ = 6253 |
| 168 |
logging.debug('Value to Skip: %d', value_to_skip) |
| 169 |
|
| 170 |
# write the value to skip and a known value |
| 171 |
writer_schema = schema.Parse('"%s"' % number_type.lower()) |
| 172 |
writer, encoder, datum_writer = write_datum(value_to_skip, writer_schema) |
| 173 |
datum_writer.write(VALUE_TO_READ, encoder) |
| 174 |
|
| 175 |
# skip the value |
| 176 |
reader = io.BytesIO(writer.getvalue()) |
| 177 |
decoder = avro_io.BinaryDecoder(reader) |
| 178 |
decoder.skip_long() |
| 179 |
|
| 180 |
# read data from string buffer |
| 181 |
datum_reader = avro_io.DatumReader(writer_schema) |
| 182 |
read_value = datum_reader.read(decoder) |
| 183 |
|
| 184 |
logging.debug('Read Value: %d', read_value) |
| 185 |
if read_value == VALUE_TO_READ: correct += 1 |
| 186 |
return correct |
| 187 |
|
| 188 |
|
| 189 |
# ------------------------------------------------------------------------------ |
| 190 |
|
| 191 |
|
| 192 |
class TestIO(unittest.TestCase): |
| 193 |
# |
| 194 |
# BASIC FUNCTIONALITY |
| 195 |
# |
| 196 |
|
| 197 |
def testValidate(self): |
| 198 |
passed = 0 |
| 199 |
for example_schema, datum in SCHEMAS_TO_VALIDATE: |
| 200 |
logging.debug('Schema: %r', example_schema) |
| 201 |
logging.debug('Datum: %r', datum) |
| 202 |
validated = avro_io.Validate(schema.Parse(example_schema), datum) |
| 203 |
logging.debug('Valid: %s', validated) |
| 204 |
if validated: passed += 1 |
| 205 |
self.assertEqual(passed, len(SCHEMAS_TO_VALIDATE)) |
| 206 |
|
| 207 |
def testRoundTrip(self): |
| 208 |
correct = 0 |
| 209 |
for example_schema, datum in SCHEMAS_TO_VALIDATE: |
| 210 |
logging.debug('Schema: %s', example_schema) |
| 211 |
logging.debug('Datum: %s', datum) |
| 212 |
|
| 213 |
writer_schema = schema.Parse(example_schema) |
| 214 |
writer, encoder, datum_writer = write_datum(datum, writer_schema) |
| 215 |
round_trip_datum = read_datum(writer, writer_schema) |
| 216 |
|
| 217 |
logging.debug('Round Trip Datum: %s', round_trip_datum) |
| 218 |
if datum == round_trip_datum: correct += 1 |
| 219 |
self.assertEqual(correct, len(SCHEMAS_TO_VALIDATE)) |
| 220 |
|
| 221 |
# |
| 222 |
# BINARY ENCODING OF INT AND LONG |
| 223 |
# |
| 224 |
|
| 225 |
def testBinaryIntEncoding(self): |
| 226 |
correct = check_binary_encoding('int') |
| 227 |
self.assertEqual(correct, len(BINARY_ENCODINGS)) |
| 228 |
|
| 229 |
def testBinaryLongEncoding(self): |
| 230 |
correct = check_binary_encoding('long') |
| 231 |
self.assertEqual(correct, len(BINARY_ENCODINGS)) |
| 232 |
|
| 233 |
def testSkipInt(self): |
| 234 |
correct = check_skip_number('int') |
| 235 |
self.assertEqual(correct, len(BINARY_ENCODINGS)) |
| 236 |
|
| 237 |
def testSkipLong(self): |
| 238 |
correct = check_skip_number('long') |
| 239 |
self.assertEqual(correct, len(BINARY_ENCODINGS)) |
| 240 |
|
| 241 |
# |
| 242 |
# SCHEMA RESOLUTION |
| 243 |
# |
| 244 |
|
| 245 |
def testSchemaPromotion(self): |
| 246 |
# note that checking writer_schema.type in read_data |
| 247 |
# allows us to handle promotion correctly |
| 248 |
promotable_schemas = ['"int"', '"long"', '"float"', '"double"'] |
| 249 |
incorrect = 0 |
| 250 |
for i, ws in enumerate(promotable_schemas): |
| 251 |
writer_schema = schema.Parse(ws) |
| 252 |
datum_to_write = 219 |
| 253 |
for rs in promotable_schemas[i + 1:]: |
| 254 |
reader_schema = schema.Parse(rs) |
| 255 |
writer, enc, dw = write_datum(datum_to_write, writer_schema) |
| 256 |
datum_read = read_datum(writer, writer_schema, reader_schema) |
| 257 |
logging.debug('Writer: %s Reader: %s', writer_schema, reader_schema) |
| 258 |
logging.debug('Datum Read: %s', datum_read) |
| 259 |
if datum_read != datum_to_write: incorrect += 1 |
| 260 |
self.assertEqual(incorrect, 0) |
| 261 |
|
| 262 |
def testUnknownSymbol(self): |
| 263 |
writer_schema = schema.Parse("""\ |
| 264 |
{"type": "enum", "name": "Test", |
| 265 |
"symbols": ["FOO", "BAR"]}""") |
| 266 |
datum_to_write = 'FOO' |
| 267 |
|
| 268 |
reader_schema = schema.Parse("""\ |
| 269 |
{"type": "enum", "name": "Test", |
| 270 |
"symbols": ["BAR", "BAZ"]}""") |
| 271 |
|
| 272 |
writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema) |
| 273 |
reader = io.BytesIO(writer.getvalue()) |
| 274 |
decoder = avro_io.BinaryDecoder(reader) |
| 275 |
datum_reader = avro_io.DatumReader(writer_schema, reader_schema) |
| 276 |
self.assertRaises(avro_io.SchemaResolutionException, datum_reader.read, decoder) |
| 277 |
|
| 278 |
def testDefaultValue(self): |
| 279 |
writer_schema = LONG_RECORD_SCHEMA |
| 280 |
datum_to_write = LONG_RECORD_DATUM |
| 281 |
|
| 282 |
correct = 0 |
| 283 |
for field_type, default_json, default_datum in DEFAULT_VALUE_EXAMPLES: |
| 284 |
reader_schema = schema.Parse("""\ |
| 285 |
{"type": "record", "name": "Test", |
| 286 |
"fields": [{"name": "H", "type": %s, "default": %s}]} |
| 287 |
""" % (field_type, default_json)) |
| 288 |
datum_to_read = {'H': default_datum} |
| 289 |
|
| 290 |
writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema) |
| 291 |
datum_read = read_datum(writer, writer_schema, reader_schema) |
| 292 |
logging.debug('Datum Read: %s', datum_read) |
| 293 |
if datum_to_read == datum_read: correct += 1 |
| 294 |
self.assertEqual(correct, len(DEFAULT_VALUE_EXAMPLES)) |
| 295 |
|
| 296 |
def testNoDefaultValue(self): |
| 297 |
writer_schema = LONG_RECORD_SCHEMA |
| 298 |
datum_to_write = LONG_RECORD_DATUM |
| 299 |
|
| 300 |
reader_schema = schema.Parse("""\ |
| 301 |
{"type": "record", "name": "Test", |
| 302 |
"fields": [{"name": "H", "type": "int"}]}""") |
| 303 |
|
| 304 |
writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema) |
| 305 |
reader = io.BytesIO(writer.getvalue()) |
| 306 |
decoder = avro_io.BinaryDecoder(reader) |
| 307 |
datum_reader = avro_io.DatumReader(writer_schema, reader_schema) |
| 308 |
self.assertRaises(avro_io.SchemaResolutionException, datum_reader.read, decoder) |
| 309 |
|
| 310 |
def testProjection(self): |
| 311 |
writer_schema = LONG_RECORD_SCHEMA |
| 312 |
datum_to_write = LONG_RECORD_DATUM |
| 313 |
|
| 314 |
reader_schema = schema.Parse("""\ |
| 315 |
{"type": "record", "name": "Test", |
| 316 |
"fields": [{"name": "E", "type": "int"}, |
| 317 |
{"name": "F", "type": "int"}]}""") |
| 318 |
datum_to_read = {'E': 5, 'F': 6} |
| 319 |
|
| 320 |
writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema) |
| 321 |
datum_read = read_datum(writer, writer_schema, reader_schema) |
| 322 |
logging.debug('Datum Read: %s', datum_read) |
| 323 |
self.assertEqual(datum_to_read, datum_read) |
| 324 |
|
| 325 |
def testFieldOrder(self): |
| 326 |
writer_schema = LONG_RECORD_SCHEMA |
| 327 |
datum_to_write = LONG_RECORD_DATUM |
| 328 |
|
| 329 |
reader_schema = schema.Parse("""\ |
| 330 |
{"type": "record", "name": "Test", |
| 331 |
"fields": [{"name": "F", "type": "int"}, |
| 332 |
{"name": "E", "type": "int"}]}""") |
| 333 |
datum_to_read = {'E': 5, 'F': 6} |
| 334 |
|
| 335 |
writer, encoder, datum_writer = write_datum(datum_to_write, writer_schema) |
| 336 |
datum_read = read_datum(writer, writer_schema, reader_schema) |
| 337 |
logging.debug('Datum Read: %s', datum_read) |
| 338 |
self.assertEqual(datum_to_read, datum_read) |
| 339 |
|
| 340 |
def testTypeException(self): |
| 341 |
writer_schema = schema.Parse("""\ |
| 342 |
{"type": "record", "name": "Test", |
| 343 |
"fields": [{"name": "F", "type": "int"}, |
| 344 |
{"name": "E", "type": "int"}]}""") |
| 345 |
datum_to_write = {'E': 5, 'F': 'Bad'} |
| 346 |
self.assertRaises( |
| 347 |
avro_io.AvroTypeException, write_datum, datum_to_write, writer_schema) |
| 348 |
|
| 349 |
|
| 350 |
if __name__ == '__main__': |
| 351 |
raise Exception('Use run_tests.py') |