Debian Codegraph @

lang/py3/scripts/avro

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 """Command line utlity for reading and writing Avro files."""
22
23 import argparse
24 import csv
25 import functools
26 import itertools
27 import json
28 import os
29 import sys
30 import traceback
31
32 import avro
33 from avro import datafile
34 from avro import io as avro_io
35 from avro import schema
36
37
38 # ------------------------------------------------------------------------------
39
40
41 class AvroError(Exception):
42 """Errors in this module."""
43 pass
44
45
46 def print_json(row):
47 print(json.dumps(row))
48
49
50 def print_json_pretty(row):
51 print(json.dumps(row, indent=4, sort_keys=True))
52
53 _csv_writer = csv.writer(sys.stdout)
54
55 def _write_row(row):
56 _csv_writer.writerow(row)
57
58
59 def print_csv(row):
60 # Sort record fields to ensure consistent ordering:
61 _write_row([row[key] for key in sorted(row)])
62
63
64 def select_printer(format):
65 return {
66 'json' : print_json,
67 'json-pretty' : print_json_pretty,
68 'csv' : print_csv
69 }[format]
70
71
72 def record_match(expr, record):
73 return eval(expr, None, {'r' : record})
74
75
76 def parse_fields(fields):
77 fields = fields or ''
78 if not fields.strip():
79 return None
80
81 return [field.strip() for field in fields.split(',') if field.strip()]
82
83
84 def field_selector(fields):
85 fields = set(fields)
86 def keys_filter(obj):
87 return dict((k, obj[k]) for k in (set(obj) & fields))
88 return keys_filter
89
90
91 def print_avro(avro, opts):
92 if opts.header and (opts.format != 'csv'):
93 raise AvroError('--header applies only to CSV format')
94
95 # Apply filter first
96 if opts.filter:
97 avro = filter(functools.partial(record_match, opts.filter), avro)
98
99 for i in range(opts.skip):
100 try:
101 next(avro)
102 except StopIteration:
103 return
104
105 fields = parse_fields(opts.fields)
106 if fields:
107 avro = map(field_selector(fields), avro)
108
109 printer = select_printer(opts.format)
110 for i, record in enumerate(avro):
111 if i == 0 and opts.header:
112 _write_row(sorted(record.keys()))
113 if i >= opts.count:
114 break
115 printer(record)
116
117
118 def print_schema(avro):
119 schema = avro.meta['avro.schema'].decode('utf-8')
120 print(json.dumps(json.loads(schema), indent=4))
121
122
123 def cat(opts, args):
124 if not args:
125 raise AvroError('No files to show')
126
127 for filename in args:
128 try:
129 fo = open(filename, 'rb')
130 except (OSError, IOError) as e:
131 raise AvroError('Cannot open %s - %s' % (filename, e))
132
133 avro = datafile.DataFileReader(fo, avro_io.DatumReader())
134
135 if opts.print_schema:
136 print_schema(avro)
137 continue
138
139 print_avro(avro, opts)
140
141
142 # ------------------------------------------------------------------------------
143
144
145 def _open(filename, mode):
146 if filename == '-':
147 return {
148 'rt' : sys.stdin,
149 'wb' : sys.stdout.buffer,
150 }[mode]
151
152 return open(filename, mode)
153
154
155 def iter_json(info, schema):
156 return map(json.loads, info)
157
158
159 def convert(value, field):
160 type = field.type.type
161 if type == 'union':
162 return convert_union(value, field)
163
164 return {
165 'int' : int,
166 'long' : int,
167 'float' : float,
168 'double' : float,
169 'string' : str,
170 'bytes' : bytes,
171 'boolean' : bool,
172 'null' : lambda _: None,
173 'union' : lambda v: convert_union(v, field),
174 }[type](value)
175
176
177 def convert_union(value, field):
178 for name in [s.name for s in field.type.schemas]:
179 try:
180 return convert(name)(value)
181 except ValueError:
182 continue
183
184
185 def iter_csv(info, schema):
186 header = [field.name for field in schema.fields]
187 for row in csv.reader(info):
188 values = [convert(v, f) for v, f in zip(row, schema.fields)]
189 yield dict(zip(header, values))
190
191
192 def guess_input_type(files):
193 if not files:
194 return None
195
196 ext = os.path.splitext(files[0])[1].lower()
197 if ext in ('.json', '.js'):
198 return 'json'
199 elif ext in ('.csv',):
200 return 'csv'
201
202 return None
203
204
205 def write(opts, files):
206 if not opts.schema:
207 raise AvroError('No schema specified')
208
209 input_type = opts.input_type or guess_input_type(files)
210 if not input_type:
211 raise AvroError('Cannot guess input file type (not .json or .csv)')
212
213 try:
214 with open(opts.schema, 'rt') as f:
215 json_schema = f.read()
216 writer_schema = schema.Parse(json_schema)
217 out = _open(opts.output, 'wb')
218 except (IOError, OSError) as e:
219 raise AvroError('Cannot open file - %s' % e)
220
221 record_parser_map = {
222 'json': iter_json,
223 'csv': iter_csv,
224 }
225
226 with datafile.DataFileWriter(
227 writer=out,
228 datum_writer=avro_io.DatumWriter(),
229 writer_schema=writer_schema,
230 ) as writer:
231 iter_records = record_parser_map[input_type]
232 for filename in (files or ['-']):
233 reader = _open(filename, 'rt')
234 for record in iter_records(reader, writer_schema):
235 writer.append(record)
236
237
238 # ------------------------------------------------------------------------------
239
240
241 def main(argv=None):
242 argv = argv or sys.argv
243
244 parser = argparse.ArgumentParser(
245 description='Display/write for Avro files',
246 usage='%(prog)s cat|write [options] FILE [FILE...]',
247 )
248
249 parser.add_argument(
250 '--version',
251 action='version',
252 version='%(prog)s ' + avro.VERSION,
253 )
254
255 # cat options:
256 cat_options = parser.add_argument_group(title='cat options')
257 cat_options.add_argument(
258 '-n', '--count',
259 type=int,
260 default=float('Infinity'),
261 help='number of records to print',
262 )
263 cat_options.add_argument(
264 '-s', '--skip',
265 type=int,
266 default=0,
267 help='number of records to skip',
268 )
269 cat_options.add_argument(
270 '-f', '--format',
271 default='json',
272 choices=['json', 'csv', 'json-pretty'],
273 help='record format',
274 )
275 cat_options.add_argument(
276 '--header',
277 default=False,
278 action='store_true',
279 help='print CSV header',
280 )
281 cat_options.add_argument(
282 '--filter',
283 default=None,
284 help='filter records (e.g. r["age"]>1)',
285 )
286 cat_options.add_argument(
287 '--print-schema',
288 default=False,
289 action='store_true',
290 help='print schema',
291 )
292 cat_options.add_argument(
293 '--fields',
294 default=None,
295 help='fields to show, comma separated (show all by default)',
296 )
297
298 # write options
299 write_options = parser.add_argument_group(title='write options')
300 write_options.add_argument(
301 '--schema',
302 help='schema file (required)',
303 )
304 write_options.add_argument(
305 '--input-type',
306 choices=['json', 'csv'],
307 default=None,
308 help='input file(s) type (json or csv)',
309 )
310 write_options.add_argument(
311 '-o', '--output',
312 default='-',
313 help='output file',
314 )
315
316 opts, args = parser.parse_known_args(argv[1:])
317 if len(args) < 1:
318 parser.error('You much specify `cat` or `write`.')
319
320 command = args.pop(0)
321 try:
322 if command == 'cat':
323 cat(opts, args)
324 elif command == 'write':
325 write(opts, args)
326 else:
327 raise AvroError('Unknown command - %s' % command)
328 except AvroError as e:
329 parser.error('%s' % e) # Will exit
330 except Exception as e:
331 traceback.print_exc()
332 raise SystemExit('panic: %s' % e)
333
334
335 if __name__ == '__main__':
336 main()