| 1 |
import argparse |
| 2 |
import re |
| 3 |
import subprocess |
| 4 |
import os |
| 5 |
import json |
| 6 |
import math |
| 7 |
from datetime import datetime, timedelta, timezone |
| 8 |
|
| 9 |
|
| 10 |
# Function to calculate average payload size and round up to the next full integer |
| 11 |
def calculate_payload_size_in_window(payload_sizes): |
| 12 |
return sum(payload_sizes) |
| 13 |
|
| 14 |
|
| 15 |
# Parse command-line arguments |
| 16 |
def parse_arguments(): |
| 17 |
parser = argparse.ArgumentParser( |
| 18 |
description= |
| 19 |
'Process DLT messages to calculate the highest average payload size.') |
| 20 |
|
| 21 |
# Add the mode argument with choices for 'online' and 'offline' |
| 22 |
parser.add_argument('--soft-limit-factor', |
| 23 |
type=float, |
| 24 |
default=1.5, |
| 25 |
help='Factor to multiply the measured values with to get the soft limit recommendation (default: 1.1)') |
| 26 |
parser.add_argument('--hard-limit-factor', |
| 27 |
type=float, |
| 28 |
default=2, |
| 29 |
help='Factor to multiply the measured values with to get the hard limit recommendation (default: 1.3)') |
| 30 |
parser.add_argument('--factor-multiplier', |
| 31 |
type=int, |
| 32 |
default=10, |
| 33 |
help='Multiplier used for rounding the limits up. I.e. 13 -> 20, 123 -> 130 (default: 10)') |
| 34 |
|
| 35 |
parser.add_argument('--sum-contexts', |
| 36 |
action='store_true', |
| 37 |
default=False, |
| 38 |
help='If set to true, all contexts will be summed, otherwise one measurement per context (default: True') |
| 39 |
|
| 40 |
# Add the mode argument with choices for 'online' and 'offline' |
| 41 |
parser.add_argument('-m', '--mode', |
| 42 |
type=str, |
| 43 |
choices=['online', 'offline'], |
| 44 |
default='online', |
| 45 |
help='Mode to process DLT messages, online will read logs directly via dlt-receive, ' |
| 46 |
'offline will expect a dlt offline trace converted ' |
| 47 |
'to text using dlt-convert -a file > dlt.txt (default: online)') |
| 48 |
|
| 49 |
parser.add_argument('-o', '--output', |
| 50 |
type=str, |
| 51 |
choices=['json', 'dlt'], |
| 52 |
default='dlt', |
| 53 |
help='Output mode of script (default: json)') |
| 54 |
|
| 55 |
parser.add_argument('-e', '--ecu', |
| 56 |
type=str, |
| 57 |
default=None, |
| 58 |
required=True, |
| 59 |
help='The ECU ID to filter DLT messages') |
| 60 |
|
| 61 |
parser.add_argument('-p', '--port', |
| 62 |
type=int, |
| 63 |
default=3490, |
| 64 |
help='Port to receive DLT messages (default: 3490) (only used in online mode)') |
| 65 |
parser.add_argument('-a', '--address', |
| 66 |
type=str, |
| 67 |
default='127.0.0.1', |
| 68 |
help='Address to receive DLT messages (default: 127.0.0.1) (only used in online mode)') |
| 69 |
parser.add_argument('-t', '--time', |
| 70 |
type=int, |
| 71 |
default=300, |
| 72 |
help='Runtime duration in seconds (default: 300) (only used in online mode)') |
| 73 |
|
| 74 |
# Create a subgroup for offline mode arguments |
| 75 |
offline_group = parser.add_argument_group('Offline Mode Arguments') |
| 76 |
offline_group.add_argument('-f', '--file', |
| 77 |
type=str, |
| 78 |
help='Path to the DLT file when in offline mode') |
| 79 |
|
| 80 |
parser.add_argument( |
| 81 |
'-d', |
| 82 |
'--debug', |
| 83 |
action='store_true', |
| 84 |
help='Include debug logs in the average (default: false)') |
| 85 |
parser.add_argument( |
| 86 |
'-v', |
| 87 |
'--verbose', |
| 88 |
action='store_true', |
| 89 |
help='Include verbose logs in the average (default: false)') |
| 90 |
|
| 91 |
parser.add_argument('app_id', |
| 92 |
type=str, |
| 93 |
nargs='?', |
| 94 |
help='Application ID to filter DLT messages, if not passed, all messages will be processed') |
| 95 |
|
| 96 |
parser.add_argument('context_id', |
| 97 |
type=str, |
| 98 |
default=None, |
| 99 |
nargs='?', |
| 100 |
help='Optional ID to filter DLT messages, needs application id!') |
| 101 |
|
| 102 |
args = parser.parse_args() |
| 103 |
if not args.ecu: |
| 104 |
parser.error("ECU ID is required.") |
| 105 |
|
| 106 |
# Validate the arguments based on the mode |
| 107 |
if args.mode == 'offline': |
| 108 |
if not args.file: |
| 109 |
parser.error("Offline mode requires a file path using the -f or --file option.") |
| 110 |
if not os.path.isfile(args.file): |
| 111 |
parser.error("File path provided does not exist.") |
| 112 |
else: |
| 113 |
# In online mode, the file should not be used |
| 114 |
if args.file: |
| 115 |
parser.error("File path is not used in online mode.") |
| 116 |
return args |
| 117 |
|
| 118 |
|
| 119 |
def get_next_line_from_file(file): |
| 120 |
with open(file, 'r', errors='ignore') as file: |
| 121 |
for line in file: |
| 122 |
yield line.strip() |
| 123 |
|
| 124 |
|
| 125 |
def get_next_line_from_dlt_receive(process): |
| 126 |
return process.stdout.readline().decode('utf-8', 'ignore').strip() |
| 127 |
|
| 128 |
def keep_running(end_time): |
| 129 |
if end_time: |
| 130 |
return datetime.now(timezone.utc) < end_time |
| 131 |
return True |
| 132 |
|
| 133 |
def round_up_to(n, multiplier): |
| 134 |
return math.ceil(n / multiplier) * multiplier |
| 135 |
|
| 136 |
def main(): |
| 137 |
args = parse_arguments() |
| 138 |
end_time = None |
| 139 |
load = {} |
| 140 |
|
| 141 |
if args.mode == 'offline': |
| 142 |
file_reader = get_next_line_from_file(args.file) |
| 143 |
else: |
| 144 |
# Prepare the command to receive messages with provided arguments |
| 145 |
command = f"dlt-receive -a {args.address} -p {args.port}" |
| 146 |
process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True) |
| 147 |
# Set the duration for which to receive messages |
| 148 |
end_time = datetime.now(timezone.utc) + timedelta(seconds=args.time) |
| 149 |
|
| 150 |
if not args.app_id: |
| 151 |
line_regex = re.compile(rf"{args.ecu}-{{0,3}}\s+(\w{{1,4}}-{{0,3}})\s+(\w{{1,4}}-{{0,3}})\s+log") |
| 152 |
else: |
| 153 |
if args.context_id: |
| 154 |
line_regex = re.compile(rf"{args.ecu}-{{0,3}}\s+({args.app_id}-{{0,3}})\s+({args.context_id}-{{0,3}})\s+log") |
| 155 |
else: |
| 156 |
line_regex = re.compile(rf"{args.ecu}-{{0,3}}\s+({args.app_id}-{{0,3}})\s+(\w{{1,4}}-{{0,3}})\s+log") |
| 157 |
|
| 158 |
try: |
| 159 |
# Process messages until the end time is reached |
| 160 |
while keep_running(end_time): |
| 161 |
if args.mode == 'offline': |
| 162 |
line = next(file_reader) |
| 163 |
else: |
| 164 |
line = get_next_line_from_dlt_receive(process) |
| 165 |
|
| 166 |
if not line: |
| 167 |
continue # Skip empty lines |
| 168 |
|
| 169 |
match = line_regex.search(line) |
| 170 |
if not match: |
| 171 |
continue |
| 172 |
line_app_id = match.group(1).rstrip('-') |
| 173 |
line_context_id = match.group(2).rstrip('-') |
| 174 |
|
| 175 |
# Check if the line contains the Application ID |
| 176 |
if (('log debug' in line and not args.debug) or |
| 177 |
('log verbose' in line and not args.verbose)): |
| 178 |
continue # Skip this line as it's a debug/verbose log and the flags are not set |
| 179 |
|
| 180 |
# Extract the payload size using regex |
| 181 |
match = re.search(r'\[(.*)$', line) |
| 182 |
if not match: |
| 183 |
continue |
| 184 |
if line_app_id not in load: |
| 185 |
load[line_app_id] = {} |
| 186 |
if line_context_id not in load[line_app_id]: |
| 187 |
load[line_app_id][line_context_id] = {} |
| 188 |
load[line_app_id][line_context_id]["payload_sizes"] = [] |
| 189 |
load[line_app_id][line_context_id]["highest_average"] = 0 |
| 190 |
load[line_app_id][line_context_id]["window_start_time"] = datetime.now(timezone.utc) |
| 191 |
|
| 192 |
payload = match.group(1).strip()[:-1] |
| 193 |
payload_size = len(payload) |
| 194 |
|
| 195 |
load[line_app_id][line_context_id]["payload_sizes"].append(payload_size) |
| 196 |
|
| 197 |
# Check if 60 seconds have passed or if we're at the end of the runtime |
| 198 |
current_time = datetime.now(timezone.utc) |
| 199 |
if (current_time - load[line_app_id][line_context_id]["window_start_time"]).total_seconds() >= 60 or not keep_running(end_time): |
| 200 |
average_payload_size = calculate_payload_size_in_window(load[line_app_id][line_context_id]["payload_sizes"]) |
| 201 |
load[line_app_id][line_context_id]["highest_average"] = max(load[line_app_id][line_context_id]["highest_average"], average_payload_size) |
| 202 |
load[line_app_id][line_context_id]["payload_sizes"] = [] # Reset for the next window |
| 203 |
load[line_app_id][line_context_id]["window_start_time"] = current_time # Reset window start time |
| 204 |
except StopIteration: |
| 205 |
# done reading file |
| 206 |
pass |
| 207 |
finally: |
| 208 |
results = {} |
| 209 |
|
| 210 |
# If there are any remaining payload sizes that have not been averaged, do so now |
| 211 |
for app_id, context_ids in load.items(): |
| 212 |
app_id_total = 0 |
| 213 |
results[app_id] = {} |
| 214 |
results[app_id]["contexts"] = {} |
| 215 |
for context_id, data in context_ids.items(): |
| 216 |
payload_sizes = data["payload_sizes"] |
| 217 |
average_payload_size = calculate_payload_size_in_window(payload_sizes) |
| 218 |
highest_average = max(data["highest_average"], average_payload_size) / 60 |
| 219 |
app_id_total += highest_average |
| 220 |
|
| 221 |
if not args.sum_contexts: |
| 222 |
results[app_id]["contexts"][context_id] = {} |
| 223 |
results[app_id]["contexts"][context_id]["measured"] = highest_average |
| 224 |
results[app_id]["contexts"][context_id]["soft"] = round_up_to(highest_average * args.soft_limit_factor, args.factor_multiplier) |
| 225 |
results[app_id]["contexts"][context_id]["hard"] = round_up_to(highest_average * args.hard_limit_factor, args.factor_multiplier) |
| 226 |
results[app_id]["contexts"] = dict(sorted(results[app_id]["contexts"].items())) |
| 227 |
|
| 228 |
results[app_id] |
| 229 |
if not args.context_id: |
| 230 |
results[app_id]["measured"] = app_id_total |
| 231 |
results[app_id]["soft"] = round_up_to(app_id_total * args.soft_limit_factor, args.factor_multiplier) |
| 232 |
results[app_id]["hard"] = round_up_to(app_id_total * args.hard_limit_factor, args.factor_multiplier) |
| 233 |
|
| 234 |
results = dict(sorted(results.items())) |
| 235 |
if args.output == "json": |
| 236 |
print(json.dumps(results, indent=4)) |
| 237 |
else: |
| 238 |
if args.sum_contexts: |
| 239 |
for app_id, data in results.items(): |
| 240 |
print(f"{app_id} {data['soft']} {data['hard']}") |
| 241 |
else: |
| 242 |
for app_id, context_ids in results.items(): |
| 243 |
for context_id, data in context_ids["contexts"].items(): |
| 244 |
print(f"{app_id} {context_id} {data['soft']} {data['hard']}") |
| 245 |
|
| 246 |
if args.mode == 'offline': |
| 247 |
pass |
| 248 |
else: |
| 249 |
# Ensure the subprocess is terminated |
| 250 |
process.terminate() |
| 251 |
|
| 252 |
|
| 253 |
if __name__ == "__main__": |
| 254 |
main() |