Debian Codegraph @

timeit

raw
1 #!/usr/bin/python
2 # -*- coding: utf-8 -*-
3 # vim: set ts=4 sw=4 et :
4 #
5 # timeit - program to read a stdin and send the data to stdout with timing data
6 #
7 # Copyright 2019 Sony Corporation
8 #
9 # This program is provided under the Gnu General Public License (GPL)
10 # version 2 ONLY. This program is distributed WITHOUT ANY WARRANTY.
11 # See the LICENSE file, which should have accompanied this program,
12 # for the text of the license.
13 #
14 # 2019-10-18 by Tim Bird <[email protected]>
15 #
16 # Some examples:
17 # $ ssh -vvv user@host 2>&1 | timeit -l
18 # show the time for each debug output line from ssh, starting from
19 # launch time
20 #
21 # To do:
22 # - validate 'device' mode (open a character device and time that)
23 # - support use_stdboth - read from and time both stdout and stderr
24 # - use Popoen(..., stderr=STDOUT)
25 # - support writing to stdin of launched program
26 # - right now, we block on the read from program stdout, which
27 # means our loop never gets the chance to send data
28 #
29 # Bugs:
30 # - cr's are getting lost in stdin mode for interactive sessions
31 # - that is: ssh host | timeit results in no carriage returns during
32 # shell interaction on the target
33 #
34 # CHANGELOG:
35 # 2019.10.17 - Version 0.9.0
36 # - copy initial code from grabserial
37 # - remove serial stuff and adjust for stdin processing
38
39 import os
40 import sys
41 import getopt
42 import time
43 import datetime
44 import re
45 import errno
46 from subprocess import *
47 import shlex
48
49 try:
50 import thread
51 except ImportError:
52 import _thread as thread
53
54 VERSION = (0, 9, 0)
55
56 cmd = "timeit"
57
58 verbose = 0 # pylint: disable=I0011,C0103
59 cmdinput = u"" # pylint: disable=I0011,C0103
60
61
62 def vprint(message):
63 if verbose:
64 print(message)
65
66
67 def usage(rcode):
68 print("""%s : line-oriented timing program
69 Usage: %s [options]
70 options:
71 -h, --help Print this message
72 -d, --device=<devpath> Specify a device to read from, to time lines from
73 -e, --endtime=<secs> End the program after the specified seconds have
74 elapsed.
75 -c, --command=<cmd> Execute a command to time lines from
76 -s, --string=<str> Send a string to the command before reading
77 -a, --again Restart application after -e expires, -q is triggered.
78 -T, --systime Print system time for each line received. The time
79 is the absolute local time when the first character
80 of each line is received by %s
81 -F, --timeformat=<val> Specifies system time format for each received line
82 e.g.
83 -F \"%%Y-%%m-%%d %%H:%%M:%%S.%%f\"
84 (default is \"%%H:%%M:%%S.%%f\")
85 -m, --match=<pat> Specify a regular expression pattern to match to
86 set a base time. Time values for lines after the
87 line matching the pattern will be relative to
88 this base time.
89 -i, --inlinepat=<pat> Specify an regular expression pattern, with match
90 time to be reported at end of run. It works mid-line.
91 -q, --quitpat=<pat> Specify a regular expression pattern to end the
92 program. It works mid-line.
93 -l, --launchtime Set base time from launch of program.
94 -o, --output=<name> Output data to the named file.
95 Uses: %%Y-%%m-%%dT%%H:%%M:%%S on "%%".
96 -A, --append Append (rather than overwrite) output data to the
97 file specifed with -o option
98 -Q, --quiet Silent on stdout, serial port data is only written
99 to file, if specified.
100 -v, --verbose Show verbose runtime messages
101 -V, --version Show version number and exit
102 -n, --nodelta Skip printing delta between read lines.
103 --crtonewline Promote a carriage return to be treated as a
104 --stderr Time stderr instead of stdout
105 """ % (cmd, cmd, cmd))
106 sys.exit(rcode)
107
108
109 def read_input():
110 global cmdinput # pylint: disable=I0011,C0103
111
112 # NOTE: cmdinput is in unicode (to make handling similar between
113 # python2 and python3)
114
115 while 1:
116 if sys.version_info < (3, 0):
117 try:
118 # raw_input in python 2.x returns byte string
119 # decode to unicode
120 cmdinput = raw_input().decode(sys.stdin.encoding)
121 except EOFError:
122 # if we're piping input, we want to stop trying to read
123 # it when the pipe closes, or the file ends
124 break
125 else:
126 # raw_input is gone in python3
127 # https://www.python.org/dev/peps/pep-3111/
128 # input() returns string in unicode already
129 try:
130 cmdinput = input() # pylint: disable=I0011,W0141
131 except EOFError:
132 break
133
134 # OK - no more user input, just wait for program exit
135 # FIXTHIS - could exit this thread here?
136 # should just be able to return
137 while 1:
138 time.sleep(1)
139
140 # timeit - main routine to grab line data and time the output of each line.
141 # Takes a list of arguments, as they would have been passed in sys.argv
142 # that is, a list of strings.
143 # also can take an optional file descriptor for where to send the data
144 # by default, data read from the subprocess or device is sent to sys.stdout,
145 # but you can specify your own (already open) file descriptor, or None. This
146 # would only make sense if you specified another out_filename with
147 # "-o","myoutputfilename"
148 # Return value: True if we should 'restart' the program
149 def timeit(arglist, outputfd=sys.stdout):
150 global verbose # pylint: disable=I0011,C0103
151 global cmdinput # pylint: disable=I0011,C0103
152
153 # parse the command line options
154 try:
155 opts, args = getopt.getopt( arglist,
156 "hli:d:c:s:aTF:m:e:o:AQvVq:n", [
157 "help",
158 "launchtime",
159 "inlinepat=",
160 "device=",
161 "command=",
162 "string=",
163 "again",
164 "systime",
165 "timeformat=",
166 "match=",
167 "endtime=",
168 "output=",
169 "append",
170 "quiet",
171 "verbose",
172 "version",
173 "quitpat=",
174 "nodelta",
175 "crtonewline",
176 "stderr",
177 ])
178 except getopt.GetoptError:
179 # print help info and exit
180 print("Error parsing command line options")
181 usage(2)
182
183 # assume reading from stdin, but that can change
184 fd = sys.stdin
185
186 show_time = True
187 show_systime = False
188 basepat = ""
189 inlinepat = ''
190 quitpat = ''
191 basetime = 0
192 inline_time = None
193 endtime = 0
194 out_filename = None
195 out = None
196 out_permissions = "wb"
197 append = False
198 command = ""
199 use_stdin = True # read stdin of this process
200 use_stderr = False # read stderr of called process
201 cr_to_nl = False
202 restart = False
203 quiet = False
204 systime_format = "%H:%M:%S.%f"
205 use_delta = True
206 out_filenamehasdate = 0
207
208 for opt, arg in opts:
209 if opt in ["-h", "--help"]:
210 usage(0)
211 if opt in ["-d", "--device"]:
212 use_stdin = False
213 device = arg
214 fd.close()
215 try:
216 # FIXTHIS - don't assume device is always writable
217 fd = open(device, "rwb")
218 except IOError as e:
219 print("Error opening device '%s': %s (%s)" % \
220 (device, e, errno.errorcode[e.errno]))
221 sys.exit(e.errno)
222
223 # assume device can also be written to
224 outfd = fd
225
226 if opt in ["-s", "--string"]:
227 string = arg
228 if opt in ["-c", "--command"]:
229 use_stdin = False
230 command = arg
231 if opt in ["-t", "--time"]:
232 show_time = True
233 show_systime = False
234 if opt in ["-a", "--again"]:
235 restart = True
236 if opt in ["-T", "--systime"]:
237 show_time = False
238 show_systime = True
239 if opt in ["-F", "--timeformat"]:
240 systime_format = arg
241 if opt in ["-m", "--match"]:
242 basepat = arg
243 if opt in ["-i", "--inlinepat"]:
244 inlinepat = arg
245 if opt in ["-q", "--quitpat"]:
246 quitpat = arg
247 if opt in ["-l", "--launchtime"]:
248 print('setting basetime to time of program launch')
249 basetime = time.time()
250 if opt in ["-e", "--endtime"]:
251 endstr = arg
252 try:
253 endtime = time.time()+float(endstr)
254 except ValueError:
255 print("Error: invalid endtime %s specified" % arg)
256 fd.close()
257 sys.exit(3)
258 if opt in ["-o", "--output"]:
259 out_filename = arg
260 if out_filename == "%":
261 out_filename = "%Y-%m-%dT%H:%M:%S"
262 if "%d" in out_filename:
263 out_pattern = out_filename
264 out_filenamehasdate = 1
265 if "%" in out_filename:
266 out_filename = datetime.datetime.now().strftime(out_filename)
267 if opt in ["-A", "--append"]:
268 out_permissions = "a+b"
269 append = True
270 if opt in ["-Q", "--quiet"]:
271 quiet = True
272 if opt in ["-v", "--verbose"]:
273 verbose = 1
274 if opt in ["-V", "--version"]:
275 print(cmd + " version %d.%d.%d" % VERSION)
276 fd.close()
277 sys.exit(0)
278 if opt in ["-n", "--nodelta"]:
279 use_delta = False
280 if opt in ["--crtonewline"]:
281 cr_to_nl = True
282 if opt in ["--stderr"]:
283 # FIXTHIS - not my own stderr!
284 # --stderr only makes sense when executing a command
285 use_stderr = True
286
287 # if running a command, launch it now
288 if command:
289 # FIXTHIS - support timing lines on both stdout and stderr
290 arg_list = shlex.split(command)
291 try:
292 print("Launching '%s'" % arg_list)
293 pd = Popen(arg_list, stdin=PIPE, stdout=PIPE, stderr=PIPE,
294 close_fds=True)
295 except CalledProcessError as e:
296 sys.stderr.write("Error running '%s': %s, error code = %d" % \
297 (command, e.output, e.returncode))
298 sys.exit(e.returncode)
299
300 # Popen.stdin has a handle to write to the process' stdin
301 # Popen.stdout has a handle to read the process' stdout
302 # Popen.stderr has a handle to read the process' stderr
303
304 if use_stderr:
305 fd = pd.stderr
306 else:
307 fd = pd.stdout
308 outfd = pd.stdin
309
310 # if verbose, show what our settings are
311 if endtime and not restart:
312 vprint("Program set to end in %s seconds" % endstr)
313 if endtime and restart:
314 vprint("Program set to restart after %s seconds." % endstr)
315 if show_time:
316 vprint("Printing timing information for each line")
317 if show_systime:
318 vprint("Printing absolute timing information for each line")
319 if basepat:
320 vprint("Matching pattern '%s' to set base time" % basepat)
321 if inlinepat:
322 vprint("Inline pattern '%s' to report time of at end of run"
323 % inlinepat)
324 if quitpat:
325 if restart:
326 vprint("Inline pattern '%s' to restart program" % quitpat)
327 else:
328 vprint("Inline pattern '%s' to exit program" % quitpat)
329 if out_filename:
330 try:
331 # open in binary mode, to pass through data as unmodified
332 # as possible
333 out = open(out_filename, out_permissions)
334 if out_filenamehasdate:
335 out_opendate = datetime.date.today()
336 except IOError:
337 print("Can't open output file '%s'" % out_filename)
338 sys.exit(1)
339 if append:
340 vprint("Appending data to '%s'" % out_filename)
341 else:
342 vprint("Saving data to '%s'" % out_filename)
343 if quiet:
344 vprint("Keeping quiet on stdout")
345
346 prev1 = 0
347 linetime = 0
348 newline = True
349 curline = ""
350 vprint("Use Control-C to stop...")
351
352 if not use_stdin:
353 # capture stdin to send data to a device or called command
354 try:
355 thread.start_new_thread(read_input, ())
356 except thread.error:
357 print("Error starting thread for read input\n")
358
359 stop_reason = "an unknown reason"
360 # read from the fd until something stops the program
361 while True:
362 try:
363 if cmdinput:
364 outfd.write((cmdinput + u"\n").encode("utf8"))
365 cmdinput = u""
366
367 # read a byte
368 # NOTE: x should be a byte string in both python 2 and 3
369 # NOTE: this is a blocking call - not sure if cmdinput can
370 # accumulate and be sent to device while we're blocked here
371 x = fd.read(1)
372 if use_stdin and not x:
373 # empty read on stdin means end of file
374 stop_reason = "end of input"
375 break
376
377 # see if we're supposed to stop yet
378 if endtime and time.time() > endtime:
379 stop_reason = "time expiration"
380 break
381
382 # if we didn't read anything, loop
383 if len(x) == 0:
384 # can only get here in device or command mode (not use_stdin)
385 # but even then - does this work?
386 # need to do a quick poll in order for timing to be as accurate a possible
387 # ie - I can't delay here
388 stop_reason = "end of input"
389 break
390
391 # convert carriage returns to newlines.
392 if x == b"\r":
393 if cr_to_nl:
394 x = b"\n"
395 else:
396 continue
397
398 # set basetime to when first char is received
399 if not basetime:
400 basetime = time.time()
401
402 # if outputting data to a file with a date in its name and the
403 # date has changed, then close it and open a new file.
404 if (out_filename
405 and out_filenamehasdate
406 and newline
407 and datetime.date.today() > out_opendate
408 and not endtime):
409 vprint("Closing output file: '%s'\n" % out_filename)
410 out.close()
411 out_filename = datetime.datetime.now().strftime(out_pattern)
412 vprint("Opening new output file: '%s'\n" % out_filename)
413 try:
414 out = open(out_filename, out_permissions)
415 out_opendate = datetime.date.today()
416 except IOError:
417 print("Can't open output file '%s'" % out_filename)
418 sys.exit(1)
419
420 if show_time and newline:
421 linetime = time.time()
422 elapsed = linetime-basetime
423 delta = elapsed-prev1
424 msg = "[%4.6f %2.6f] " % (elapsed, delta)
425 if not quiet:
426 if outputfd:
427 outputfd.write(msg)
428 if out:
429 try:
430 out.write(msg.encode(sys.stdout.encoding))
431 except UnicodeEncodeError:
432 try:
433 out.write(msg.encode("utf8"))
434 except UnicodeEncodeError:
435 out.write(msg)
436
437 prev1 = elapsed
438 newline = False
439
440 if show_systime and newline:
441 linetime = time.time()
442 linetimestr = datetime.datetime.now().strftime(systime_format)
443 elapsed = linetime-basetime
444 if use_delta:
445 delta = elapsed-prev1
446 msg = "[%s %2.6f] " % (linetimestr, delta)
447 else:
448 msg = "[%s] " % (linetimestr)
449 if not quiet:
450 outputfd.write(msg)
451 if out:
452 try:
453 out.write(msg.encode(sys.stdout.encoding))
454 except UnicodeEncodeError:
455 try:
456 out.write(msg.encode("utf8"))
457 except UnicodeEncodeError:
458 out.write(msg)
459
460 prev1 = elapsed
461 newline = False
462
463 # FIXTHIS - should I buffer the output here??
464 if not quiet:
465 # x is a bytestr
466 outputfd.write(x.decode("utf8", "ignore"))
467 outputfd.flush()
468
469 if out:
470 # save bytestring data exactly as received from serial port
471 # (ie there is no 'decode' here)
472 out.write(x)
473
474 # curline is in unicode
475 curline += x.decode("utf8", "ignore")
476
477 # watch for patterns
478 if inlinepat and not inline_time and \
479 re.search(inlinepat, curline):
480 # inlinepat is in curline:
481 inline_time = time.time()
482
483 # Exit the loop if quitpat matches
484 if quitpat and re.search(quitpat, curline):
485 stop_reason = "match of quit pattern '" + \
486 quitpat + "' was found"
487 break
488
489 if x == b"\n":
490 newline = True
491 if basepat and re.match(basepat, curline):
492 basetime = linetime
493 elapsed = 0
494 prev1 = 0
495 curline = ""
496 sys.stdout.flush()
497 if out:
498 out.flush()
499
500 except EnvironmentError:
501 stop_reason = "some external error"
502
503 # An actual error. We don't want to restart the program in this
504 # case, so this function will return false.
505 restart = False
506 break
507 except KeyboardInterrupt:
508 stop_reason = "keyboard interrupt"
509
510 # Looks like user wants to stop, don't restart.
511 restart = False
512 break
513
514 fd.close()
515 if inline_time:
516 inline_time_str = '%4.6f' % (inline_time-basetime)
517 msg = u'\nThe inlinepat: "%s" was matched at %s\n' % \
518 (inlinepat, inline_time_str)
519 if not quiet:
520 outputfd.write(msg)
521 outputfd.flush()
522 if out:
523 try:
524 out.write(msg.encode(sys.stdout.encoding))
525 except UnicodeEncodeError:
526 try:
527 out.write(msg.encode("utf8"))
528 except UnicodeEncodeError:
529 out.write(msg)
530 out.flush()
531
532 if out:
533 out.close()
534
535 vprint("%s stopped due to %s" % (cmd, stop_reason))
536
537 return restart
538
539 if __name__ == "__main__":
540 while True:
541 restart = timeit(sys.argv[1:])
542
543 # Return value is true if we should 'restart' the program
544 if restart:
545 vprint("Restarting %s\n" %
546 datetime.datetime.now().strftime("%H:%M:%S.%f"))
547 else:
548 break
549
550 # emacs custom variables for using tabs
551 # indent-tabs-mode: nil
552 # tab-width: 4