Debian Codegraph @

lang/py3/avro/tests/test_datafile.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 import logging
22 import os
23 import tempfile
24 import unittest
25
26 from avro import datafile
27 from avro import io
28 from avro import schema
29
30
31 # ------------------------------------------------------------------------------
32
33
34 SCHEMAS_TO_VALIDATE = (
35 ('"null"', None),
36 ('"boolean"', True),
37 ('"string"', 'adsfasdf09809dsf-=adsf'),
38 ('"bytes"', b'12345abcd'),
39 ('"int"', 1234),
40 ('"long"', 1234),
41 ('"float"', 1234.0),
42 ('"double"', 1234.0),
43 ('{"type": "fixed", "name": "Test", "size": 1}', b'B'),
44 ('{"type": "enum", "name": "Test", "symbols": ["A", "B"]}', 'B'),
45 ('{"type": "array", "items": "long"}', [1, 3, 2]),
46 ('{"type": "map", "values": "long"}', {'a': 1, 'b': 3, 'c': 2}),
47 ('["string", "null", "long"]', None),
48
49 ("""
50 {
51 "type": "record",
52 "name": "Test",
53 "fields": [{"name": "f", "type": "long"}]
54 }
55 """,
56 {'f': 5}),
57
58 ("""
59 {
60 "type": "record",
61 "name": "Lisp",
62 "fields": [{
63 "name": "value",
64 "type": [
65 "null",
66 "string",
67 {
68 "type": "record",
69 "name": "Cons",
70 "fields": [{"name": "car", "type": "Lisp"},
71 {"name": "cdr", "type": "Lisp"}]
72 }
73 ]
74 }]
75 }
76 """,
77 {'value': {'car': {'value': 'head'}, 'cdr': {'value': None}}}),
78 )
79
80 CODECS_TO_VALIDATE = ('null', 'deflate')
81
82 try:
83 import snappy
84 CODECS_TO_VALIDATE += ('snappy',)
85 except ImportError:
86 logging.info('Snappy not present, will skip testing it.')
87
88
89 # ------------------------------------------------------------------------------
90
91
92 class TestDataFile(unittest.TestCase):
93
94 @classmethod
95 def setUpClass(cls):
96 cls._temp_dir = (
97 tempfile.TemporaryDirectory(prefix=cls.__name__, suffix='.tmp'))
98 logging.debug('Created temporary directory: %s', cls._temp_dir.name)
99
100 @classmethod
101 def tearDownClass(cls):
102 logging.debug('Cleaning up temporary directory: %s', cls._temp_dir.name)
103 cls._temp_dir.cleanup()
104
105 def NewTempFile(self):
106 """Creates a new temporary file.
107
108 File is automatically cleaned up after test.
109
110 Returns:
111 The path of the new temporary file.
112 """
113 temp_file = tempfile.NamedTemporaryFile(
114 dir=self._temp_dir.name,
115 prefix='test',
116 suffix='.avro',
117 delete=False,
118 )
119 return temp_file.name
120
121 def testRoundTrip(self):
122 correct = 0
123 for iexample, (writer_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE):
124 for codec in CODECS_TO_VALIDATE:
125 file_path = self.NewTempFile()
126
127 # Write the datum this many times in the data file:
128 nitems = 10
129
130 logging.debug(
131 'Performing round-trip with codec %r in file %s for example #%d\n'
132 'Writing datum: %r using writer schema:\n%s',
133 codec, file_path, iexample,
134 datum, writer_schema)
135
136 logging.debug('Creating data file %r', file_path)
137 with open(file_path, 'wb') as writer:
138 datum_writer = io.DatumWriter()
139 schema_object = schema.Parse(writer_schema)
140 with datafile.DataFileWriter(
141 writer=writer,
142 datum_writer=datum_writer,
143 writer_schema=schema_object,
144 codec=codec,
145 ) as dfw:
146 for _ in range(nitems):
147 dfw.append(datum)
148
149 logging.debug('Reading data from %r', file_path)
150 with open(file_path, 'rb') as reader:
151 datum_reader = io.DatumReader()
152 with datafile.DataFileReader(reader, datum_reader) as dfr:
153 round_trip_data = list(dfr)
154
155 logging.debug(
156 'Round-trip data has %d items: %r',
157 len(round_trip_data), round_trip_data)
158
159 if ([datum] * nitems) == round_trip_data:
160 correct += 1
161 else:
162 logging.error(
163 'Round-trip data does not match:\n'
164 'Expect: %r\n'
165 'Actual: %r',
166 [datum] * nitems,
167 round_trip_data)
168
169 self.assertEqual(
170 correct,
171 len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE))
172
173 def testAppend(self):
174 correct = 0
175 for iexample, (writer_schema, datum) in enumerate(SCHEMAS_TO_VALIDATE):
176 for codec in CODECS_TO_VALIDATE:
177 file_path = self.NewTempFile()
178
179 logging.debug(
180 'Performing append with codec %r in file %s for example #%d\n'
181 'Writing datum: %r using writer schema:\n%s',
182 codec, file_path, iexample,
183 datum, writer_schema)
184
185 logging.debug('Creating data file %r', file_path)
186 with open(file_path, 'wb') as writer:
187 datum_writer = io.DatumWriter()
188 schema_object = schema.Parse(writer_schema)
189 with datafile.DataFileWriter(
190 writer=writer,
191 datum_writer=datum_writer,
192 writer_schema=schema_object,
193 codec=codec,
194 ) as dfw:
195 dfw.append(datum)
196
197 logging.debug('Appending data to %r', file_path)
198 for i in range(9):
199 with open(file_path, 'ab+') as writer:
200 with datafile.DataFileWriter(writer, io.DatumWriter()) as dfw:
201 dfw.append(datum)
202
203 logging.debug('Reading appended data from %r', file_path)
204 with open(file_path, 'rb') as reader:
205 datum_reader = io.DatumReader()
206 with datafile.DataFileReader(reader, datum_reader) as dfr:
207 appended_data = list(dfr)
208
209 logging.debug(
210 'Appended data has %d items: %r',
211 len(appended_data), appended_data)
212
213 if ([datum] * 10) == appended_data:
214 correct += 1
215 else:
216 logging.error(
217 'Appended data does not match:\n'
218 'Expect: %r\n'
219 'Actual: %r',
220 [datum] * 10,
221 appended_data)
222
223 self.assertEqual(
224 correct,
225 len(CODECS_TO_VALIDATE) * len(SCHEMAS_TO_VALIDATE))
226
227 def testContextManager(self):
228 file_path = self.NewTempFile()
229
230 # Test the writer with a 'with' statement.
231 with open(file_path, 'wb') as writer:
232 datum_writer = io.DatumWriter()
233 sample_schema, sample_datum = SCHEMAS_TO_VALIDATE[1]
234 schema_object = schema.Parse(sample_schema)
235 with datafile.DataFileWriter(writer, datum_writer, schema_object) as dfw:
236 dfw.append(sample_datum)
237 self.assertTrue(writer.closed)
238
239 # Test the reader with a 'with' statement.
240 datums = []
241 with open(file_path, 'rb') as reader:
242 datum_reader = io.DatumReader()
243 with datafile.DataFileReader(reader, datum_reader) as dfr:
244 for datum in dfr:
245 datums.append(datum)
246 self.assertTrue(reader.closed)
247
248 def testMetadata(self):
249 file_path = self.NewTempFile()
250
251 # Test the writer with a 'with' statement.
252 with open(file_path, 'wb') as writer:
253 datum_writer = io.DatumWriter()
254 sample_schema, sample_datum = SCHEMAS_TO_VALIDATE[1]
255 schema_object = schema.Parse(sample_schema)
256 with datafile.DataFileWriter(writer, datum_writer, schema_object) as dfw:
257 dfw.SetMeta('test.string', 'foo')
258 dfw.SetMeta('test.number', '1')
259 dfw.append(sample_datum)
260 self.assertTrue(writer.closed)
261
262 # Test the reader with a 'with' statement.
263 datums = []
264 with open(file_path, 'rb') as reader:
265 datum_reader = io.DatumReader()
266 with datafile.DataFileReader(reader, datum_reader) as dfr:
267 self.assertEqual(b'foo', dfr.GetMeta('test.string'))
268 self.assertEqual(b'1', dfr.GetMeta('test.number'))
269 for datum in dfr:
270 datums.append(datum)
271 self.assertTrue(reader.closed)
272
273
274 # ------------------------------------------------------------------------------
275
276
277 if __name__ == '__main__':
278 raise Exception('Use run_tests.py')