Debian Codegraph @

lang/py3/avro/tests/test_script.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 import csv
22 import io
23 import json
24 import logging
25 import operator
26 import os
27 import subprocess
28 import tempfile
29 import unittest
30
31 import avro.datafile
32 import avro.io
33 import avro.schema
34
35
36 # ------------------------------------------------------------------------------
37
38
39 NUM_RECORDS = 7
40
41 SCHEMA = """
42 {
43 "namespace": "test.avro",
44 "name": "LooneyTunes",
45 "type": "record",
46 "fields": [
47 {"name": "first", "type": "string"},
48 {"name": "last", "type": "string"},
49 {"name": "type", "type": "string"}
50 ]
51 }
52 """
53
54 LOONIES = (
55 ('daffy', 'duck', 'duck'),
56 ('bugs', 'bunny', 'bunny'),
57 ('tweety', '', 'bird'),
58 ('road', 'runner', 'bird'),
59 ('wile', 'e', 'coyote'),
60 ('pepe', 'le pew', 'skunk'),
61 ('foghorn', 'leghorn', 'rooster'),
62 )
63
64
65 def looney_records():
66 for f, l, t in LOONIES:
67 yield {'first': f, 'last' : l, 'type' : t}
68
69
70 def GetRootDir():
71 test_dir = os.path.dirname(os.path.abspath(__file__))
72 root_dir = os.path.dirname(os.path.dirname(test_dir))
73 return root_dir
74
75
76 def GetScriptPath():
77 root_dir = GetRootDir()
78 avro_script_path = os.path.join(root_dir, 'scripts', 'avro')
79 assert os.path.exists(avro_script_path), \
80 ('Avro script not found: %r' % avro_script_path)
81 return avro_script_path
82
83
84 # Absolute path of the 'avro' script:
85 SCRIPT_PATH = GetScriptPath()
86
87
88 def RunScript(*args, stdin=None):
89 command = [SCRIPT_PATH]
90 command.extend(args)
91 env = dict(os.environ)
92 env['PYTHONPATH'] = '%s:%s' % (GetRootDir(), env.get('PYTHONPATH', ''))
93 logging.debug('Running command:\n%s', ' \\\n\t'.join(command))
94 process = subprocess.Popen(
95 args=command,
96 env=env,
97 stdin=stdin,
98 stdout=subprocess.PIPE,
99 stderr=subprocess.PIPE,
100 )
101 (out, err) = process.communicate()
102 assert (process.returncode == os.EX_OK), \
103 ('Command %r failed with exit code %r, output %r and error %r'
104 % (command, process.returncode, out, err))
105 return out
106
107
108 _TEST_JSON_VALUE = {
109 'first': 'daffy',
110 'last': 'duck',
111 'type': 'duck',
112 }
113
114
115 # ------------------------------------------------------------------------------
116
117
118 class TestCat(unittest.TestCase):
119
120 @staticmethod
121 def WriteAvroFile(file_path):
122 schema = avro.schema.Parse(SCHEMA)
123 with open(file_path, 'wb') as writer:
124 with avro.datafile.DataFileWriter(
125 writer=writer,
126 datum_writer=avro.io.DatumWriter(),
127 writer_schema=schema,
128 ) as writer:
129 for record in looney_records():
130 writer.append(record)
131
132 def setUp(self):
133 # TODO: flag to not delete the files
134 delete = True
135 self._avro_file = (
136 tempfile.NamedTemporaryFile(prefix='test-', suffix='.avro', delete=delete))
137 TestCat.WriteAvroFile(self._avro_file.name)
138
139 def tearDown(self):
140 self._avro_file.close()
141
142 def _RunCat(self, *args, raw=False):
143 """Runs the specified 'avro cat test-file ...' command.
144
145 Args:
146 *args: extra parameters to the 'avro cat' command.
147 raw: Whether to decode stdout as UTF-8.
148 Returns:
149 The command stdout (as bytes if raw is set, or else as string).
150 """
151 out = RunScript('cat', self._avro_file.name, *args)
152 if raw:
153 return out
154 else:
155 return out.decode('utf-8')
156
157 def testPrint(self):
158 lines = self._RunCat().splitlines()
159 return len(lines) == NUM_RECORDS
160
161 def testFilter(self):
162 lines = self._RunCat('--filter', "r['type']=='bird'").splitlines()
163 return len(lines) == 2
164
165 def testSkip(self):
166 skip = 3
167 lines = self._RunCat('--skip', str(skip)).splitlines()
168 return len(lines) == NUM_RECORDS - skip
169
170 def testCsv(self):
171 reader = csv.reader(io.StringIO(self._RunCat('-f', 'csv')))
172 self.assertEqual(len(list(reader)), NUM_RECORDS)
173
174 def testCsvHeader(self):
175 reader = csv.DictReader(io.StringIO(self._RunCat('-f', 'csv', '--header')))
176 expected = {'type': 'duck', 'last': 'duck', 'first': 'daffy'}
177
178 data = next(reader)
179 self.assertEqual(expected, data)
180
181 def testPrintSchema(self):
182 out = self._RunCat('--print-schema')
183 self.assertEqual(json.loads(out)['namespace'], 'test.avro')
184
185 def testHelp(self):
186 # Just see we have these
187 self._RunCat('-h')
188 self._RunCat('--help')
189
190 def testJsonPretty(self):
191 out = self._RunCat('--format', 'json-pretty', '-n', '1')
192 self.assertEqual(
193 json.loads(out),
194 _TEST_JSON_VALUE,
195 'Output mismatch\n'
196 'Expect: %r\n'
197 'Actual: %r'
198 % (_TEST_JSON_VALUE, out))
199
200 def testVersion(self):
201 out = RunScript('cat', '--version').decode('utf-8')
202
203 def testFiles(self):
204 lines = self._RunCat(self._avro_file.name).splitlines()
205 self.assertEqual(len(lines), 2 * NUM_RECORDS)
206
207 def testFields(self):
208 # One field selection (no comma)
209 lines = self._RunCat('--fields', 'last').splitlines()
210 self.assertEqual(json.loads(lines[0]), {'last': 'duck'})
211
212 # Field selection (with comma and space)
213 lines = self._RunCat('--fields', 'first, last').splitlines()
214 self.assertEqual(json.loads(lines[0]), {'first': 'daffy', 'last': 'duck'})
215
216 # Empty fields should get all
217 lines = self._RunCat('--fields', '').splitlines()
218 self.assertEqual(
219 json.loads(lines[0]),
220 {'first': 'daffy', 'last': 'duck', 'type': 'duck'})
221
222 # Non existing fields are ignored
223 lines = self._RunCat('--fields', 'first,last,age').splitlines()
224 self.assertEqual(
225 json.loads(lines[0]),
226 {'first': 'daffy', 'last': 'duck'})
227
228
229 # ------------------------------------------------------------------------------
230
231
232 class TestWrite(unittest.TestCase):
233
234 def setUp(self):
235 delete = False
236
237 self._json_file = tempfile.NamedTemporaryFile(
238 prefix='test-', suffix='.json', delete=delete)
239 with open(self._json_file.name, 'w') as f:
240 for record in looney_records():
241 json.dump(record, f)
242 f.write('\n')
243
244 self._csv_file = tempfile.NamedTemporaryFile(
245 prefix='test-', suffix='.csv', delete=delete)
246 with open(self._csv_file.name, 'w') as f:
247 writer = csv.writer(f)
248 get = operator.itemgetter('first', 'last', 'type')
249 for record in looney_records():
250 writer.writerow(get(record))
251
252 self._schema_file = tempfile.NamedTemporaryFile(
253 prefix='test-', suffix='.avsc', delete=delete)
254 with open(self._schema_file.name, 'w') as f:
255 f.write(SCHEMA)
256
257 def tearDown(self):
258 self._csv_file.close()
259 self._json_file.close()
260 self._schema_file.close()
261
262 def _RunWrite(self, *args, stdin=None):
263 """Runs the specified 'avro write ...' command.
264
265 Args:
266 *args: extra parameters to the 'avro write' command.
267 stdin: Optional string to feed the 'avro write' command stdin with.
268 Returns:
269 The command stdout as bytes.
270 """
271 return RunScript(
272 'write', '--schema', self._schema_file.name,
273 stdin=stdin, *args
274 )
275
276 def LoadAvro(self, filename):
277 out = RunScript('cat', filename).decode('utf-8')
278 return tuple(map(json.loads, out.splitlines()))
279
280 def testVersion(self):
281 out = RunScript('write', '--version').decode('utf-8')
282
283 def FormatCheck(self, format, filename):
284 with tempfile.NamedTemporaryFile(prefix='test-', suffix='.dat') as temp:
285 with open(temp.name, 'wb') as out:
286 out.write(self._RunWrite(filename, '-f', format))
287
288 records = self.LoadAvro(temp.name)
289 self.assertEqual(len(records), NUM_RECORDS)
290 self.assertEqual(records[0]['first'], 'daffy')
291
292 def testWriteJson(self):
293 self.FormatCheck('json', self._json_file.name)
294
295 def testWriteCsv(self):
296 self.FormatCheck('csv', self._csv_file.name)
297
298 def testOutfile(self):
299 with tempfile.NamedTemporaryFile(prefix='test-', suffix='.dat') as temp:
300 os.remove(temp.name)
301 self._RunWrite(self._json_file.name, '-o', temp.name)
302 self.assertEqual(len(self.LoadAvro(temp.name)), NUM_RECORDS)
303
304 def testMultiFile(self):
305 with tempfile.NamedTemporaryFile(prefix='test-', suffix='.dat') as temp:
306 with open(temp.name, 'wb') as out:
307 out.write(self._RunWrite(self._json_file.name, self._json_file.name))
308
309 self.assertEqual(len(self.LoadAvro(temp.name)), 2 * NUM_RECORDS)
310
311 def testStdin(self):
312 with tempfile.NamedTemporaryFile(prefix='test-', suffix='.dat') as temp:
313 with open(self._json_file.name, 'rb') as input_content:
314 with open(temp.name, 'wb') as out:
315 out.write(self._RunWrite('--input-type', 'json', stdin=input_content))
316
317 self.assertEqual(len(self.LoadAvro(temp.name)), NUM_RECORDS)
318
319
320 if __name__ == '__main__':
321 raise Exception('Use run_tests.py')