Debian Codegraph @

lang/py3/avro/tests/av_bench.py

raw
1 #!/usr/bin/env python3
2 # -*- mode: python -*-
3 # -*- coding: utf-8 -*-
4
5 # Licensed to the Apache Software Foundation (ASF) under one
6 # or more contributor license agreements. See the NOTICE file
7 # distributed with this work for additional information
8 # regarding copyright ownership. The ASF licenses this file
9 # to you under the Apache License, Version 2.0 (the
10 # "License"); you may not use this file except in compliance
11 # with the License. You may obtain a copy of the License at
12 #
13 # http://www.apache.org/licenses/LICENSE-2.0
14 #
15 # Unless required by applicable law or agreed to in writing, software
16 # distributed under the License is distributed on an "AS IS" BASIS,
17 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 # See the License for the specific language governing permissions and
19 # limitations under the License.
20
21 import logging
22 import random
23 import string
24 import sys
25 import time
26
27 import avro.datafile
28 import avro.io
29 import avro.schema
30
31
32 TYPES = ('A', 'CNAME',)
33 FILENAME = 'datafile.avr'
34
35
36 def GenerateRandomName():
37 return ''.join(random.sample(string.ascii_lowercase, 15))
38
39
40 def GenerateRandomIP():
41 return '%s.%s.%s.%s' % (
42 random.randint(0, 255),
43 random.randint(0, 255),
44 random.randint(0, 255),
45 random.randint(0, 255),
46 )
47
48
49 def Write(nrecords):
50 """Writes a data file with the specified number of random records.
51
52 Args:
53 nrecords: Number of records to write.
54 """
55 schema_s = """
56 {
57 "type": "record",
58 "name": "Query",
59 "fields" : [
60 {"name": "query", "type": "string"},
61 {"name": "response", "type": "string"},
62 {"name": "type", "type": "string", "default": "A"}
63 ]
64 }
65 """
66 schema = avro.schema.Parse(schema_s)
67 writer = avro.io.DatumWriter(schema)
68
69 with open(FILENAME, 'wb') as out:
70 with avro.datafile.DataFileWriter(
71 out, writer, schema,
72 # codec='deflate'
73 ) as data_writer:
74 for _ in range(nrecords):
75 response = GenerateRandomIP()
76 query = GenerateRandomName()
77 type = random.choice(TYPES)
78 data_writer.append({
79 'query': query,
80 'response': response,
81 'type': type,
82 })
83
84
85 def Read(expect_nrecords):
86 """Reads the data file generated by Write()."""
87 with open(FILENAME, 'rb') as f:
88 reader = avro.io.DatumReader()
89 with avro.datafile.DataFileReader(f, reader) as file_reader:
90 nrecords = 0
91 for record in file_reader:
92 nrecords += 1
93 assert (nrecords == expect_nrecords), (
94 'Expecting %d records, got %d.' % (expected_nrecords, nrecords))
95
96
97 def Timing(f, *args):
98 s = time.time()
99 f(*args)
100 e = time.time()
101 return e - s
102
103
104 def Main(args):
105 nrecords = int(args[1])
106 print('Write %0.4f' % Timing(Write, nrecords))
107 print('Read %0.4f' % Timing(Read, nrecords))
108
109
110 if __name__ == '__main__':
111 log_formatter = logging.Formatter(
112 '%(asctime)s %(levelname)s %(filename)s:%(lineno)s : %(message)s')
113 logging.root.setLevel(logging.DEBUG)
114 console_handler = logging.StreamHandler()
115 console_handler.setFormatter(log_formatter)
116 console_handler.setLevel(logging.DEBUG)
117 logging.root.addHandler(console_handler)
118
119 Main(sys.argv)