Debian Codegraph @

contrib/ci/csv_writer.py

raw
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 # Copyright 2015 Objectif Libre
4 #
5 # Licensed under the Apache License, Version 2.0 (the "License"); you may
6 # not use this file except in compliance with the License. You may obtain
7 # a copy of the License at
8 #
9 # http://www.apache.org/licenses/LICENSE-2.0
10 #
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14 # License for the specific language governing permissions and limitations
15 # under the License.
16 #
17 import calendar
18 import copy
19 import csv
20 import datetime
21 import random
22 import sys
23 import uuid
24
25 from oslo_utils import timeutils
26
27 from cloudkitty import json_utils as json
28
29
30 COMPUTE = {
31 "type": "compute",
32 "desc": {},
33 "vol": {
34 "qty": 1,
35 "unit": "instance"}}
36
37 COMPUTE_RESOURCE = {
38 "availability_zone": "nova",
39 "flavor": "m1.nano",
40 "image_id": "f5600101-8fa2-4864-899e-ebcb7ed6b568",
41 "memory": "64",
42 "metadata": {
43 "farm": "prod"},
44 "name": "prod1",
45 "project_id": "f266f30b11f246b589fd266f85eeec39",
46 "user_id": "55b3379b949243009ee96972fbf51ed1",
47 "vcpus": "1"}
48
49 IMAGE = {
50 "type": "image",
51 "desc": {},
52 "vol": {
53 "qty": 214106112.0,
54 "unit": "B"}}
55
56 IMAGE_RESOURCE = {
57 "name": "cirros-0.3.4-x86_64-uec-ramdisk",
58 "checksum": "be575a2b939972276ef675752936977f",
59 "disk_format": "ari",
60 "protected": "False",
61 "container_format": "ari",
62 "min_disk": "0",
63 "is_public": "True",
64 "min_ram": "0",
65 "project_id": "f1873b13951542268bf7eed7cf971e52",
66 "resource_id": "08017fbc-b13a-4d8d-b002-4eb4eff54cd4",
67 "source": "openstack",
68 "user_id": "None",
69 "size": "214106112"}
70
71 VOLUME = {
72 "type": "volume",
73 "desc": {},
74 "vol": {
75 "qty": 1,
76 "unit": "GB"}}
77
78 VOLUME_RESOURCE = {
79 'instance_uuid': 'None',
80 'status': 'available',
81 'display_name': 'test-vol',
82 'event_type': 'volume.create.end',
83 'availability_zone': 'nova',
84 'tenant_id': 'cd27b013b9db4f4099e273e4b9949023',
85 'created_at': '2015-04-28 13:34:25',
86 'snapshot_id': 'None',
87 'volume_type': '314150bc-221f-4676-a7cf-16f12850b217',
88 'host': 'volume.devstack@lvmdriver-1#lvmdriver-1',
89 'replication_driver_data': 'None',
90 'replication_status': 'disabled',
91 'volume_id': '2bed6a3d-468a-459b-802b-44930016c0a3',
92 'replication_extended_status': 'None',
93 'user_id': '2524d5a52ce64a569d131d7dc1dfb455',
94 'launched_at': '2015-04-28 13:34:26.869928',
95 'size': '1',
96 "project_id": "f1873b13951542268bf7eed7cf971e52",
97 "resource_id": "08017fbc-b13a-4d8d-b002-4eb4eff54cd4",
98 "source": "openstack"}
99
100 NETWORK_BW_IN = {
101 "type": "network.bw.in",
102 "desc": {},
103 "vol": {
104 "qty": 4546.0,
105 "unit": "B"}}
106
107 NETWORK_BW_OUT = {
108 "type": "network.bw.out",
109 "desc": {},
110 "vol": {
111 "qty": 50.0,
112 "unit": "MB"}}
113
114 NETWORK_BW_RESOURCE = {
115 'instance_id': 'eef9673d-5d24-43fd-89f5-2929acc7e193',
116 'instance_type': '42',
117 'mac': 'fa:16:3e:dd:b2:80',
118 'fref': 'None',
119 'name': 'tap12a7d4e1-fc',
120 "project_id": "f1873b13951542268bf7eed7cf971e52",
121 "resource_id": "08017fbc-b13a-4d8d-b002-4eb4eff54cd4",
122 "source": "openstack",
123 "user_id": "None"}
124
125 FLOATING = {
126 "type": "network.floating",
127 "desc": {},
128 "vol": {
129 "qty": 1.0,
130 "unit": "ip"}}
131
132 FLOATING_RESOURCE = {
133 'router_id': '3d9b2725-fc90-43d0-b119-630e80e1ec51',
134 'status': 'DOWN',
135 'event_type': 'floatingip.update.end',
136 'tenant_id': 'cd27b013b9db4f4099e273e4b9949023',
137 'floating_network_id': '14198fb4-dc96-45fb-9dde-546f6b0f892f',
138 'host': 'network.devstack',
139 'fixed_ip_address': '10.0.0.5',
140 'floating_ip_address': '172.24.4.3',
141 'port_id': '12a7d4e1-fcc3-4a3c-8e57-c4baf7787b57',
142 "project_id": "cd27b013b9db4f4099e273e4b9949023",
143 "resource_id": "ebf6485d-7f6f-4c67-97f7-7896324e12d4",
144 "source": "openstack",
145 "user_id": "7319b5d1269d4166a402868b570aad19",
146 'id': 'ebf6485d-7f6f-4c67-97f7-7896324e12d4'}
147
148
149 class VariationMapper(object):
150 day_map = {
151 0: 'mon',
152 1: 'tue',
153 2: 'wed',
154 3: 'thu',
155 4: 'fri',
156 5: 'sat',
157 6: 'sun'}
158 var_map = {
159 'mon': {},
160 'tue': {},
161 'wed': {},
162 'thu': {},
163 'fri': {},
164 'sat': {},
165 'sun': {}}
166
167 def __init__(self, default=1.0):
168 self.default = default
169
170 def get_var(self, dt):
171 weekday = self.day_map[dt.weekday()]
172 if weekday in self.var_map:
173 wday_map = self.var_map[weekday]
174 if dt.hour in wday_map:
175 return wday_map[dt.hour]
176 elif 'default' in wday_map:
177 return wday_map['default']
178 elif 'default' in self.var_map:
179 return self.var_map['default']
180 return self.default
181
182
183 class VolumeVariationMapper(VariationMapper):
184 def get_vol(self, dt):
185 value = self.get_var(dt)
186 var_value = value * 0.1
187 return random.gauss(value, var_value)
188
189
190 class BaseGenerator(object):
191 base_sample = None
192 base_resource = None
193 rand = True
194 field_maps = {}
195
196 def __init__(self, nb_res=1, var_map=None, vol_map=None):
197 self.nb_res = nb_res
198 self.var_map = var_map if var_map else VariationMapper()
199 self.vol_map = vol_map
200 self.init_mapper()
201 self.resources = []
202
203 def init_mapper(self):
204 pass
205
206 def generate_resources(self):
207 for i in range(self.nb_res):
208 res = copy.deepcopy(self.base_resource)
209 for field, mapping in self.field_maps.items():
210 if hasattr(self, mapping):
211 mapping = getattr(self, mapping)
212 if isinstance(mapping, dict):
213 if self.rand:
214 value = random.choice(mapping.keys())
215 else:
216 value = mapping.keys()[i]
217 res[field] = value
218 for k, v in mapping[value].items():
219 res[k] = v
220 elif isinstance(mapping, list):
221 if self.rand:
222 value = random.choice(mapping)
223 else:
224 value = mapping[i]
225 res[field] = value
226 elif callable(mapping):
227 res[field] = mapping(i)
228 else:
229 res[field] = mapping
230 self.resources.append(res)
231
232 def generate_samples(self, dt):
233 samples = []
234 res_var = int(self.var_map.get_var(dt))
235 for i in range(res_var):
236 sample = copy.deepcopy(self.base_sample)
237 sample['desc'] = self.resources[i]
238 if self.vol_map:
239 qty = self.vol_map.get_vol(dt)
240 sample['vol']['qty'] = qty
241 elif 'size' in sample['desc']:
242 sample['vol']['qty'] = sample['desc']['size']
243 # Packing
244 sample['desc'] = json.dumps(self.resources[i])
245 sample['vol'] = json.dumps(sample['vol'])
246
247 samples.append(sample)
248 return samples
249
250
251 class ComputeVarMapper(VariationMapper):
252 var_map = {
253 'mon': {
254 'default': 1.0,
255 12: 2.0,
256 13: 3.0,
257 14: 2.0,
258 18: 2.0,
259 19: 3.0,
260 20: 4.0,
261 21: 4.0,
262 22: 3.0,
263 23: 2.0,
264 },
265 'tue': {
266 'default': 1.0,
267 12: 2.0,
268 13: 3.0,
269 14: 2.0,
270 18: 2.0,
271 19: 3.0,
272 20: 4.0,
273 21: 4.0,
274 22: 3.0,
275 23: 2.0,
276 },
277 'wed': {
278 'default': 1.0,
279 12: 2.0,
280 13: 3.0,
281 14: 2.0,
282 18: 2.0,
283 19: 3.0,
284 20: 4.0,
285 21: 4.0,
286 22: 3.0,
287 23: 2.0,
288 },
289 'thu': {
290 'default': 1.0,
291 12: 2.0,
292 13: 3.0,
293 14: 2.0,
294 18: 2.0,
295 19: 3.0,
296 20: 4.0,
297 21: 4.0,
298 22: 3.0,
299 23: 2.0,
300 },
301 'fri': {
302 'default': 1.0,
303 12: 2.0,
304 13: 3.0,
305 14: 2.0,
306 18: 2.0,
307 19: 3.0,
308 20: 4.0,
309 21: 4.0,
310 22: 3.0,
311 23: 2.0,
312 },
313 'sat': {
314 'default': 2.0,
315 12: 3.0,
316 13: 4.0,
317 14: 3.0,
318 18: 3.0,
319 19: 4.0,
320 20: 4.0,
321 21: 4.0,
322 22: 4.0,
323 23: 3.0,
324 },
325 'sun': {
326 'default': 2.0,
327 12: 3.0,
328 13: 4.0,
329 14: 3.0,
330 18: 3.0,
331 19: 4.0,
332 20: 4.0,
333 21: 4.0,
334 22: 4.0,
335 23: 3.0,
336 }}
337
338
339 class ComputeGenerator(BaseGenerator):
340 base_sample = COMPUTE
341 base_resource = COMPUTE_RESOURCE
342 field_maps = {'flavor': 'flavors',
343 'image': 'images',
344 'name': 'generate_name',
345 'resource_id': 'res_id'}
346
347 def init_mapper(self):
348 self.flavors = {
349 'm1.nano': {
350 'vcpus': '1',
351 'memory': '64'},
352 'm1.micro': {
353 'vcpus': '1',
354 'memory': '128'}}
355 self.images = []
356 self.res_id = []
357 for i in range(self.nb_res):
358 self.res_id.append(str(uuid.uuid1()))
359
360 def generate_name(self, *args):
361 basename = 'instance{}'
362 return basename.format(args[0])
363
364
365 class ImageGenerator(BaseGenerator):
366 base_sample = IMAGE
367 base_resource = IMAGE_RESOURCE
368 field_maps = {'name': 'images'}
369 rand = False
370
371 def init_mapper(self):
372 self.images = {
373 'cirros-0.3.4-x86_64-uec-kernel': {
374 'checksum': '836c69cbcd1dc4f225daedbab6edc7c7',
375 'disk_format': 'ari',
376 'container_format': 'ari',
377 'size': '4969360',
378 'resource_id': '5dd34048-6eeb-4b6c-aa51-62487733e5a1'},
379 'cirros-0.3.4-x86_64-uec-ramdisk': {
380 'checksum': '68085af2609d03e51c7662395b5b6e4b',
381 'disk_format': 'aki',
382 'container_format': 'aki',
383 'size': '3723817',
384 'resource_id': 'e512a97d-1ed5-4b27-a55a-1b9e5087936a'},
385 'Fedora-x86_64-20-20131211.1-sda': {
386 'checksum': '51bc16b900bf0f814bb6c0c3dd8f0790',
387 'disk_format': 'qcow2',
388 'container_format': 'bare',
389 'size': '214106112',
390 'resource_id': '3ee99f3f-7ecf-47b2-9a40-6df2d66ef5ae'}}
391
392
393 class VolumeGenerator(BaseGenerator):
394 base_sample = VOLUME
395 base_resource = VOLUME_RESOURCE
396 field_maps = {'volume_id': 'volumes'}
397 rand = False
398
399 def init_mapper(self):
400 self.volumes = {
401 '2bed6a3d-468a-459b-802b-44930016c0a3': {
402 'size': '10'},
403 '4fd33321-6a5f-4351-94ca-db398cd708e9': {
404 'size': '20'}}
405
406 def generate_name(self, *args):
407 basename = 'volume{}'
408 return basename.format(args[0])
409
410
411 class NetBWVolMapper(VolumeVariationMapper):
412 var_map = {
413 'mon': {
414 'default': 1024,
415 12: 2048,
416 13: 3072,
417 14: 2048,
418 18: 2048,
419 19: 3072,
420 20: 4096,
421 21: 4096,
422 22: 3072,
423 23: 2048,
424 },
425 'tue': {
426 'default': 1024,
427 12: 2048,
428 13: 3072,
429 14: 2048,
430 18: 2048,
431 19: 3072,
432 20: 4096,
433 21: 4096,
434 22: 3072,
435 23: 2048,
436 },
437 'wed': {
438 'default': 1024,
439 12: 2048,
440 13: 3072,
441 14: 2048,
442 18: 2048,
443 19: 3072,
444 20: 4096,
445 21: 4096,
446 22: 3072,
447 23: 2048,
448 },
449 'thu': {
450 'default': 1024,
451 12: 2048,
452 13: 3072,
453 14: 2048,
454 18: 2048,
455 19: 3072,
456 20: 4096,
457 21: 4096,
458 22: 3072,
459 23: 2048,
460 },
461 'fri': {
462 'default': 1024,
463 12: 2048,
464 13: 3072,
465 14: 2048,
466 18: 2048,
467 19: 3072,
468 20: 4096,
469 21: 4096,
470 22: 3072,
471 23: 2048,
472 },
473 'sat': {
474 'default': 2048,
475 12: 3072,
476 13: 4096,
477 14: 3072,
478 18: 3072,
479 19: 4096,
480 20: 4096,
481 21: 4096,
482 22: 4096,
483 23: 3072,
484 },
485 'sun': {
486 'default': 2048,
487 12: 3072,
488 13: 4096,
489 14: 3072,
490 18: 3072,
491 19: 4096,
492 20: 4096,
493 21: 4096,
494 22: 4096,
495 23: 3072,
496 }}
497
498
499 class NetworkBWGenerator(BaseGenerator):
500 base_sample = NETWORK_BW_IN
501 base_resource = NETWORK_BW_RESOURCE
502 field_maps = {'instance_id': 'instances',
503 'name': 'generate_name',
504 'mac': 'generate_mac',
505 'resource_id': 'res_id'}
506 rand = False
507
508 def init_mapper(self):
509 self.instances = []
510 self.res_id = []
511 for i in range(self.nb_res):
512 self.res_id.append(str(uuid.uuid1()))
513
514 def generate_name(self, *args):
515 basename = 'tap{}-fc'
516 return basename.format(args[0])
517
518 def generate_mac(self, *args):
519 basemac = 'fa:16:3e:{:0=2x}:{:0=2x}:{:0=2x}'
520 return basemac.format(
521 random.randint(1, 255),
522 random.randint(1, 255),
523 random.randint(1, 255))
524
525 def generate_samples(self, dt):
526 samples = []
527 self.base_sample = NETWORK_BW_OUT
528 samples.extend(super(NetworkBWGenerator, self).generate_samples(dt))
529 self.base_sample = NETWORK_BW_IN
530 samples.extend(super(NetworkBWGenerator, self).generate_samples(dt))
531 return samples
532
533
534 class FloatingGenerator(BaseGenerator):
535 base_sample = FLOATING
536 base_resource = FLOATING_RESOURCE
537 field_maps = {'fixed_ip_address': 'generate_ip_addr',
538 'floating_ip_address': 'generate_floating_addr',
539 'port_id': 'generate_port_id',
540 'resource_id': 'res_id',
541 'id': 'res_id'}
542 rand = False
543
544 def init_mapper(self):
545 self.res_id = []
546 for i in range(self.nb_res):
547 self.res_id.append(str(uuid.uuid1()))
548
549 def generate_name(self, *args):
550 basename = 'volume{}'
551 return basename.format(args[0])
552
553 def generate_port_id(self, *args):
554 return str(uuid.uuid1())
555
556 def generate_ip_addr(self, *args):
557 baseip = '10.0.0.{}'
558 return baseip.format(random.randint(5, 250))
559
560 def generate_floating_addr(self, *args):
561 baseip = '172.24.4.{}'
562 return baseip.format(random.randint(5, 250))
563
564
565 def write_samples(writer, dt, samples):
566 for sample in samples:
567 ts = calendar.timegm(dt.timetuple())
568 sample['begin'] = ts
569 sample['end'] = ts + 3600
570 writer.writerow(sample)
571
572
573 def main():
574 # Generators
575 compute_var = ComputeVarMapper()
576 image = ImageGenerator(3)
577 image.generate_resources()
578 volume = VolumeGenerator(2)
579 volume.generate_resources()
580 floating = FloatingGenerator(4, compute_var)
581 floating.generate_resources()
582 compute = ComputeGenerator(4, compute_var)
583 compute.images = [resource['resource_id']
584 for resource in image.resources]
585 compute.generate_resources()
586 net_bw = NetworkBWGenerator(4, compute_var, NetBWVolMapper())
587 net_bw.instances = [resource['resource_id']
588 for resource in compute.resources]
589 net_bw.generate_resources()
590 generators = [compute, image, volume, net_bw, floating]
591
592 # Date
593 now = timeutils.utcnow()
594 hour_delta = datetime.timedelta(hours=1)
595 cur_date = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
596 cur_month = cur_date.month
597
598 filename = sys.argv[1] if len(sys.argv) > 1 else 'generated.csv'
599 with open(filename, 'wb') as csvfile:
600 writer = csv.DictWriter(csvfile,
601 ['begin', 'end', 'type', 'desc', 'vol'])
602 writer.writeheader()
603 while cur_date.month == cur_month:
604 for generator in generators:
605 samples = generator.generate_samples(cur_date)
606 write_samples(writer, cur_date, samples)
607 cur_date += hour_delta
608
609
610 if __name__ == '__main__':
611 main()