Debian Codegraph @

util/calculate-load.py

raw
1 import argparse
2 import re
3 import subprocess
4 import os
5 import json
6 import math
7 from datetime import datetime, timedelta, timezone
8
9
10 # Function to calculate average payload size and round up to the next full integer
11 def calculate_payload_size_in_window(payload_sizes):
12 return sum(payload_sizes)
13
14
15 # Parse command-line arguments
16 def parse_arguments():
17 parser = argparse.ArgumentParser(
18 description=
19 'Process DLT messages to calculate the highest average payload size.')
20
21 # Add the mode argument with choices for 'online' and 'offline'
22 parser.add_argument('--soft-limit-factor',
23 type=float,
24 default=1.5,
25 help='Factor to multiply the measured values with to get the soft limit recommendation (default: 1.1)')
26 parser.add_argument('--hard-limit-factor',
27 type=float,
28 default=2,
29 help='Factor to multiply the measured values with to get the hard limit recommendation (default: 1.3)')
30 parser.add_argument('--factor-multiplier',
31 type=int,
32 default=10,
33 help='Multiplier used for rounding the limits up. I.e. 13 -> 20, 123 -> 130 (default: 10)')
34
35 parser.add_argument('--sum-contexts',
36 action='store_true',
37 default=False,
38 help='If set to true, all contexts will be summed, otherwise one measurement per context (default: True')
39
40 # Add the mode argument with choices for 'online' and 'offline'
41 parser.add_argument('-m', '--mode',
42 type=str,
43 choices=['online', 'offline'],
44 default='online',
45 help='Mode to process DLT messages, online will read logs directly via dlt-receive, '
46 'offline will expect a dlt offline trace converted '
47 'to text using dlt-convert -a file > dlt.txt (default: online)')
48
49 parser.add_argument('-o', '--output',
50 type=str,
51 choices=['json', 'dlt'],
52 default='dlt',
53 help='Output mode of script (default: json)')
54
55 parser.add_argument('-e', '--ecu',
56 type=str,
57 default=None,
58 required=True,
59 help='The ECU ID to filter DLT messages')
60
61 parser.add_argument('-p', '--port',
62 type=int,
63 default=3490,
64 help='Port to receive DLT messages (default: 3490) (only used in online mode)')
65 parser.add_argument('-a', '--address',
66 type=str,
67 default='127.0.0.1',
68 help='Address to receive DLT messages (default: 127.0.0.1) (only used in online mode)')
69 parser.add_argument('-t', '--time',
70 type=int,
71 default=300,
72 help='Runtime duration in seconds (default: 300) (only used in online mode)')
73
74 # Create a subgroup for offline mode arguments
75 offline_group = parser.add_argument_group('Offline Mode Arguments')
76 offline_group.add_argument('-f', '--file',
77 type=str,
78 help='Path to the DLT file when in offline mode')
79
80 parser.add_argument(
81 '-d',
82 '--debug',
83 action='store_true',
84 help='Include debug logs in the average (default: false)')
85 parser.add_argument(
86 '-v',
87 '--verbose',
88 action='store_true',
89 help='Include verbose logs in the average (default: false)')
90
91 parser.add_argument('app_id',
92 type=str,
93 nargs='?',
94 help='Application ID to filter DLT messages, if not passed, all messages will be processed')
95
96 parser.add_argument('context_id',
97 type=str,
98 default=None,
99 nargs='?',
100 help='Optional ID to filter DLT messages, needs application id!')
101
102 args = parser.parse_args()
103 if not args.ecu:
104 parser.error("ECU ID is required.")
105
106 # Validate the arguments based on the mode
107 if args.mode == 'offline':
108 if not args.file:
109 parser.error("Offline mode requires a file path using the -f or --file option.")
110 if not os.path.isfile(args.file):
111 parser.error("File path provided does not exist.")
112 else:
113 # In online mode, the file should not be used
114 if args.file:
115 parser.error("File path is not used in online mode.")
116 return args
117
118
119 def get_next_line_from_file(file):
120 with open(file, 'r', errors='ignore') as file:
121 for line in file:
122 yield line.strip()
123
124
125 def get_next_line_from_dlt_receive(process):
126 return process.stdout.readline().decode('utf-8', 'ignore').strip()
127
128 def keep_running(end_time):
129 if end_time:
130 return datetime.now(timezone.utc) < end_time
131 return True
132
133 def round_up_to(n, multiplier):
134 return math.ceil(n / multiplier) * multiplier
135
136 def main():
137 args = parse_arguments()
138 end_time = None
139 load = {}
140
141 if args.mode == 'offline':
142 file_reader = get_next_line_from_file(args.file)
143 else:
144 # Prepare the command to receive messages with provided arguments
145 command = f"dlt-receive -a {args.address} -p {args.port}"
146 process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
147 # Set the duration for which to receive messages
148 end_time = datetime.now(timezone.utc) + timedelta(seconds=args.time)
149
150 if not args.app_id:
151 line_regex = re.compile(rf"{args.ecu}-{{0,3}}\s+(\w{{1,4}}-{{0,3}})\s+(\w{{1,4}}-{{0,3}})\s+log")
152 else:
153 if args.context_id:
154 line_regex = re.compile(rf"{args.ecu}-{{0,3}}\s+({args.app_id}-{{0,3}})\s+({args.context_id}-{{0,3}})\s+log")
155 else:
156 line_regex = re.compile(rf"{args.ecu}-{{0,3}}\s+({args.app_id}-{{0,3}})\s+(\w{{1,4}}-{{0,3}})\s+log")
157
158 try:
159 # Process messages until the end time is reached
160 while keep_running(end_time):
161 if args.mode == 'offline':
162 line = next(file_reader)
163 else:
164 line = get_next_line_from_dlt_receive(process)
165
166 if not line:
167 continue # Skip empty lines
168
169 match = line_regex.search(line)
170 if not match:
171 continue
172 line_app_id = match.group(1).rstrip('-')
173 line_context_id = match.group(2).rstrip('-')
174
175 # Check if the line contains the Application ID
176 if (('log debug' in line and not args.debug) or
177 ('log verbose' in line and not args.verbose)):
178 continue # Skip this line as it's a debug/verbose log and the flags are not set
179
180 # Extract the payload size using regex
181 match = re.search(r'\[(.*)$', line)
182 if not match:
183 continue
184 if line_app_id not in load:
185 load[line_app_id] = {}
186 if line_context_id not in load[line_app_id]:
187 load[line_app_id][line_context_id] = {}
188 load[line_app_id][line_context_id]["payload_sizes"] = []
189 load[line_app_id][line_context_id]["highest_average"] = 0
190 load[line_app_id][line_context_id]["window_start_time"] = datetime.now(timezone.utc)
191
192 payload = match.group(1).strip()[:-1]
193 payload_size = len(payload)
194
195 load[line_app_id][line_context_id]["payload_sizes"].append(payload_size)
196
197 # Check if 60 seconds have passed or if we're at the end of the runtime
198 current_time = datetime.now(timezone.utc)
199 if (current_time - load[line_app_id][line_context_id]["window_start_time"]).total_seconds() >= 60 or not keep_running(end_time):
200 average_payload_size = calculate_payload_size_in_window(load[line_app_id][line_context_id]["payload_sizes"])
201 load[line_app_id][line_context_id]["highest_average"] = max(load[line_app_id][line_context_id]["highest_average"], average_payload_size)
202 load[line_app_id][line_context_id]["payload_sizes"] = [] # Reset for the next window
203 load[line_app_id][line_context_id]["window_start_time"] = current_time # Reset window start time
204 except StopIteration:
205 # done reading file
206 pass
207 finally:
208 results = {}
209
210 # If there are any remaining payload sizes that have not been averaged, do so now
211 for app_id, context_ids in load.items():
212 app_id_total = 0
213 results[app_id] = {}
214 results[app_id]["contexts"] = {}
215 for context_id, data in context_ids.items():
216 payload_sizes = data["payload_sizes"]
217 average_payload_size = calculate_payload_size_in_window(payload_sizes)
218 highest_average = max(data["highest_average"], average_payload_size) / 60
219 app_id_total += highest_average
220
221 if not args.sum_contexts:
222 results[app_id]["contexts"][context_id] = {}
223 results[app_id]["contexts"][context_id]["measured"] = highest_average
224 results[app_id]["contexts"][context_id]["soft"] = round_up_to(highest_average * args.soft_limit_factor, args.factor_multiplier)
225 results[app_id]["contexts"][context_id]["hard"] = round_up_to(highest_average * args.hard_limit_factor, args.factor_multiplier)
226 results[app_id]["contexts"] = dict(sorted(results[app_id]["contexts"].items()))
227
228 results[app_id]
229 if not args.context_id:
230 results[app_id]["measured"] = app_id_total
231 results[app_id]["soft"] = round_up_to(app_id_total * args.soft_limit_factor, args.factor_multiplier)
232 results[app_id]["hard"] = round_up_to(app_id_total * args.hard_limit_factor, args.factor_multiplier)
233
234 results = dict(sorted(results.items()))
235 if args.output == "json":
236 print(json.dumps(results, indent=4))
237 else:
238 if args.sum_contexts:
239 for app_id, data in results.items():
240 print(f"{app_id} {data['soft']} {data['hard']}")
241 else:
242 for app_id, context_ids in results.items():
243 for context_id, data in context_ids["contexts"].items():
244 print(f"{app_id} {context_id} {data['soft']} {data['hard']}")
245
246 if args.mode == 'offline':
247 pass
248 else:
249 # Ensure the subprocess is terminated
250 process.terminate()
251
252
253 if __name__ == "__main__":
254 main()