| 1 |
#!/usr/bin/env python3 |
| 2 |
# -*- mode: python -*- |
| 3 |
# -*- coding: utf-8 -*- |
| 4 |
|
| 5 |
# Licensed to the Apache Software Foundation (ASF) under one |
| 6 |
# or more contributor license agreements. See the NOTICE file |
| 7 |
# distributed with this work for additional information |
| 8 |
# regarding copyright ownership. The ASF licenses this file |
| 9 |
# to you under the Apache License, Version 2.0 (the |
| 10 |
# "License"); you may not use this file except in compliance |
| 11 |
# with the License. You may obtain a copy of the License at |
| 12 |
# |
| 13 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 14 |
# |
| 15 |
# Unless required by applicable law or agreed to in writing, software |
| 16 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 17 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 18 |
# See the License for the specific language governing permissions and |
| 19 |
# limitations under the License. |
| 20 |
|
| 21 |
import logging |
| 22 |
import os |
| 23 |
import tempfile |
| 24 |
import unittest |
| 25 |
|
| 26 |
from avro import datafile |
| 27 |
from avro import io |
| 28 |
from avro import schema |
| 29 |
|
| 30 |
|
| 31 |
# ------------------------------------------------------------------------------ |
| 32 |
|
| 33 |
|
| 34 |
SCHEMAS_TO_VALIDATE = ( |
| 35 |
('"null"', None), |
| 36 |
('"boolean"', True), |
| 37 |
('"string"', 'adsfasdf09809dsf-=adsf'), |
| 38 |
('"bytes"', b'12345abcd'), |
| 39 |
('"int"', 1234), |
| 40 |
('"long"', 1234), |
| 41 |
('"float"', 1234.0), |
| 42 |
('"double"', 1234.0), |
| 43 |
('{"type": "fixed", "name": "Test", "size": 1}', b'B'), |
| 44 |
('{"type": "enum", "name": "Test", "symbols": ["A", "B"]}', 'B'), |
| 45 |
('{"type": "array", "items": "long"}', [1, 3, 2]), |
| 46 |
('{"type": "map", "values": "long"}', {'a': 1, 'b': 3, 'c': 2}), |
| 47 |
('["string", "null", "long"]', None), |
| 48 |
|
| 49 |
(""" |
| 50 |
{ |
| 51 |
"type": "record", |
| 52 |
"name": "Test", |
| 53 |
"fields": [{"name": "f", "type": "long"}] |
| 54 |
} |
| 55 |
""", |
| 56 |
{'f': 5}), |
| 57 |
|
| 58 |
(""" |
| 59 |
{ |
| 60 |
"type": "record", |
| 61 |
"name": "Lisp", |
| 62 |
"fields": [{ |
| 63 |
"name": "value", |
| 64 |
"type": [ |
| 65 |
"null", |
| 66 |
"string", |
| 67 |
{ |
| 68 |
"type": "record", |
| 69 |
"name": "Cons", |
| 70 |
"fields": [{"name": "car", "type": "Lisp"}, |
| 71 |
{"name": "cdr", "type": "Lisp"}] |
| 72 |
} |
| 73 |
] |
| 74 |
}] |
| 75 |
} |
| 76 |
""", |
| 77 |
{'value': {'car': {'value': 'head'}, 'cdr': {'value': None}}}), |
| 78 |
) |
| 79 |
|
| 80 |
CODECS_TO_VALIDATE = ('null', 'deflate') |
| 81 |
|
| 82 |
try: |
| 83 |
import snappy |
| 84 |
CODECS_TO_VALIDATE += ('snappy',) |
| 85 |
except ImportError: |
| 86 |
logging.info('Snappy not present, will skip testing it.') |
| 87 |
|
| 88 |
|
| 89 |
# ------------------------------------------------------------------------------ |
| 90 |
|
| 91 |
|
| 92 |
class TestDataFile(unittest.TestCase): |
| 93 |
|
| 94 |
@classmethod |
| 95 |
def setUpClass(cls): |
| 96 |
cls._temp_dir = ( |
| 97 |
tempfile.TemporaryDirectory(prefix=cls.__name__, suffix='.tmp')) |
| 98 |
logging.debug('Created temporary directory: %s', cls._temp_dir.name) |
| 99 |
|
| 100 |
@classmethod |
| 101 |
def tearDownClass(cls): |
| 102 |
logging.debug('Cleaning up temporary directory: %s', cls._temp_dir.name) |
| 103 |
cls._temp_dir.cleanup() |
| 104 |
|
| 105 |
def NewTempFile(self): |
| 106 |
"""Creates a new temporary file. |
| 107 |
|
| 108 |
File is automatically cleaned up after test. |
| 109 |
|
| 110 |
Returns: |
| 111 |
The path of the new temporary file. |
| 112 |
""" |
| 113 |
temp_file = tempfile.NamedTemporaryFile( |
| 114 |
dir=self._temp_dir.name, |
| 115 |
prefix='test', |
| 116 |
suffix='.avro', |
| 117 |
delete=False, |
| 118 |
) |
| 119 |
return temp_file.name |
| 120 |
|
| 121 |
def testRoundTrip(self): |
| 122 |
correct = 0 |
| 123 |
for iexample, (writer_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE): |
| 124 |
for codec in CODECS_TO_VALIDATE: |
| 125 |
file_path = self.NewTempFile() |
| 126 |
|
| 127 |
# Write the datum this many times in the data file: |
| 128 |
nitems = 10 |
| 129 |
|
| 130 |
logging.debug( |
| 131 |
'Performing round-trip with codec %r in file %s for example #%d\n' |
| 132 |
'Writing datum: %r using writer schema:\n%s', |
| 133 |
codec, file_path, iexample, |
| 134 |
datum, writer_schema) |
| 135 |
|
| 136 |
logging.debug('Creating data file %r', file_path) |
| 137 |
with open(file_path, 'wb') as writer: |
| 138 |
datum_writer = io.DatumWriter() |
| 139 |
schema_object = schema.Parse(writer_schema) |
| 140 |
with datafile.DataFileWriter( |
| 141 |
writer=writer, |
| 142 |
datum_writer=datum_writer, |
| 143 |
writer_schema=schema_object, |
| 144 |
codec=codec, |
| 145 |
) as dfw: |
| 146 |
for _ in range(nitems): |
| 147 |
dfw.append(datum) |
| 148 |
|
| 149 |
logging.debug('Reading data from %r', file_path) |
| 150 |
with open(file_path, 'rb') as reader: |
| 151 |
datum_reader = io.DatumReader() |
| 152 |
with datafile.DataFileReader(reader, datum_reader) as dfr: |
| 153 |
round_trip_data = list(dfr) |
| 154 |
|
| 155 |
logging.debug( |
| 156 |
'Round-trip data has %d items: %r', |
| 157 |
len(round_trip_data), round_trip_data) |
| 158 |
|
| 159 |
if ([datum] * nitems) == round_trip_data: |
| 160 |
correct += 1 |
| 161 |
else: |
| 162 |
logging.error( |
| 163 |
'Round-trip data does not match:\n' |
| 164 |
'Expect: %r\n' |
| 165 |
'Actual: %r', |
| 166 |
[datum] * nitems, |
| 167 |
round_trip_data) |
| 168 |
|
| 169 |
self.assertEqual( |
| 170 |
correct, |
| 171 |
len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE)) |
| 172 |
|
| 173 |
def testAppend(self): |
| 174 |
correct = 0 |
| 175 |
for iexample, (writer_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE): |
| 176 |
for codec in CODECS_TO_VALIDATE: |
| 177 |
file_path = self.NewTempFile() |
| 178 |
|
| 179 |
logging.debug( |
| 180 |
'Performing append with codec %r in file %s for example #%d\n' |
| 181 |
'Writing datum: %r using writer schema:\n%s', |
| 182 |
codec, file_path, iexample, |
| 183 |
datum, writer_schema) |
| 184 |
|
| 185 |
logging.debug('Creating data file %r', file_path) |
| 186 |
with open(file_path, 'wb') as writer: |
| 187 |
datum_writer = io.DatumWriter() |
| 188 |
schema_object = schema.Parse(writer_schema) |
| 189 |
with datafile.DataFileWriter( |
| 190 |
writer=writer, |
| 191 |
datum_writer=datum_writer, |
| 192 |
writer_schema=schema_object, |
| 193 |
codec=codec, |
| 194 |
) as dfw: |
| 195 |
dfw.append(datum) |
| 196 |
|
| 197 |
logging.debug('Appending data to %r', file_path) |
| 198 |
for i in range(9): |
| 199 |
with open(file_path, 'ab+') as writer: |
| 200 |
with datafile.DataFileWriter(writer, io.DatumWriter()) as dfw: |
| 201 |
dfw.append(datum) |
| 202 |
|
| 203 |
logging.debug('Reading appended data from %r', file_path) |
| 204 |
with open(file_path, 'rb') as reader: |
| 205 |
datum_reader = io.DatumReader() |
| 206 |
with datafile.DataFileReader(reader, datum_reader) as dfr: |
| 207 |
appended_data = list(dfr) |
| 208 |
|
| 209 |
logging.debug( |
| 210 |
'Appended data has %d items: %r', |
| 211 |
len(appended_data), appended_data) |
| 212 |
|
| 213 |
if ([datum] * 10) == appended_data: |
| 214 |
correct += 1 |
| 215 |
else: |
| 216 |
logging.error( |
| 217 |
'Appended data does not match:\n' |
| 218 |
'Expect: %r\n' |
| 219 |
'Actual: %r', |
| 220 |
[datum] * 10, |
| 221 |
appended_data) |
| 222 |
|
| 223 |
self.assertEqual( |
| 224 |
correct, |
| 225 |
len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE)) |
| 226 |
|
| 227 |
def testContextManager(self): |
| 228 |
file_path = self.NewTempFile() |
| 229 |
|
| 230 |
# Test the writer with a 'with' statement. |
| 231 |
with open(file_path, 'wb') as writer: |
| 232 |
datum_writer = io.DatumWriter() |
| 233 |
sample_schema, sample_datum = SCHEMAS_TO_VALIDATE[1] |
| 234 |
schema_object = schema.Parse(sample_schema) |
| 235 |
with datafile.DataFileWriter(writer, datum_writer, schema_object) as dfw: |
| 236 |
dfw.append(sample_datum) |
| 237 |
self.assertTrue(writer.closed) |
| 238 |
|
| 239 |
# Test the reader with a 'with' statement. |
| 240 |
datums = [] |
| 241 |
with open(file_path, 'rb') as reader: |
| 242 |
datum_reader = io.DatumReader() |
| 243 |
with datafile.DataFileReader(reader, datum_reader) as dfr: |
| 244 |
for datum in dfr: |
| 245 |
datums.append(datum) |
| 246 |
self.assertTrue(reader.closed) |
| 247 |
|
| 248 |
def testMetadata(self): |
| 249 |
file_path = self.NewTempFile() |
| 250 |
|
| 251 |
# Test the writer with a 'with' statement. |
| 252 |
with open(file_path, 'wb') as writer: |
| 253 |
datum_writer = io.DatumWriter() |
| 254 |
sample_schema, sample_datum = SCHEMAS_TO_VALIDATE[1] |
| 255 |
schema_object = schema.Parse(sample_schema) |
| 256 |
with datafile.DataFileWriter(writer, datum_writer, schema_object) as dfw: |
| 257 |
dfw.SetMeta('test.string', 'foo') |
| 258 |
dfw.SetMeta('test.number', '1') |
| 259 |
dfw.append(sample_datum) |
| 260 |
self.assertTrue(writer.closed) |
| 261 |
|
| 262 |
# Test the reader with a 'with' statement. |
| 263 |
datums = [] |
| 264 |
with open(file_path, 'rb') as reader: |
| 265 |
datum_reader = io.DatumReader() |
| 266 |
with datafile.DataFileReader(reader, datum_reader) as dfr: |
| 267 |
self.assertEqual(b'foo', dfr.GetMeta('test.string')) |
| 268 |
self.assertEqual(b'1', dfr.GetMeta('test.number')) |
| 269 |
for datum in dfr: |
| 270 |
datums.append(datum) |
| 271 |
self.assertTrue(reader.closed) |
| 272 |
|
| 273 |
|
| 274 |
# ------------------------------------------------------------------------------ |
| 275 |
|
| 276 |
|
| 277 |
if __name__ == '__main__': |
| 278 |
raise Exception('Use run_tests.py') |