Debian Codegraph @

cmake/tribits/python_utils/GeneralScriptSupport.py

raw
1 # @HEADER
2 # *****************************************************************************
3 # TriBITS: Tribal Build, Integrate, and Test System
4 #
5 # Copyright 2013-2016 NTESS and the TriBITS contributors.
6 # SPDX-License-Identifier: BSD-3-Clause
7 # *****************************************************************************
8 # @HEADER
9
10 """
11 Python module containing general support functions for creating scripts
12 """
13
14 #
15 # Check the python version
16 #
17
18 import sys
19 if sys.version_info < (2,7):
20 print("Error, Python version is " + sys.version + " < 2.7!")
21 sys.exit(1)
22
23 #
24 # Import commands
25 #
26 import os
27 import re
28 import math
29 import subprocess
30 import time
31 import datetime
32 import optparse
33 import traceback
34
35 from Python2and3 import b, s, u
36
37 verboseDebug = False
38
39
40 #
41 # Determine what system we are on:
42 #
43
44 rePlatformName = re.compile(r"^[a-zA-Z]+")
45 platformStr = rePlatformName.findall(sys.platform)[0]
46 #print("\nplatformStr = " + platformStr)
47
48
49 ######################################
50 # Script location functions
51 ######################################
52
53
54 def getScriptBaseDir():
55 return os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
56
57
58 def getScriptName():
59 return os.path.basename(os.path.dirname(sys.argv[0]))
60
61 def getCompleteFileDirname(filename):
62 return os.path.dirname(os.path.realpath(os.path.abspath(filename)))
63
64 ######################################
65 # List helper functions
66 ######################################
67
68
69 def findInSequence(seq, item):
70 for i in range(0, len(seq)):
71 if seq[i] == item:
72 return i
73 return -1
74
75
76 def removeItemsFromList(list, items):
77 numItemsRemoved = 0
78 for item in items:
79 if item in list:
80 idx = list.index(item)
81 del list[idx]
82 numItemsRemoved = numItemsRemoved + 1
83 return numItemsRemoved
84
85
86 ######################################
87 # String helper functions
88 ######################################
89
90
91 def stripWhitespaceFromStringList(stringList):
92 return [x.strip() for x in stringList]
93
94
95 def isSubstrInMultiLineString(inputStr, findStr):
96 return inputStr.find(findStr) >= 0
97
98
99 def getStrUnderlineStr(width):
100 underlineStr = ""
101 for i in range(width):
102 underlineStr += "-"
103 return underlineStr
104
105
106 def arrayToFormattedString(array_in, offsetStr = ""):
107 sout = ""
108 sout += offsetStr + "[\n"
109 for i in range(0, len(array_in)):
110 if i != len(array_in)-1:
111 commaChar = ","
112 else:
113 commaChar = ""
114 sout += (offsetStr + " \'" + str(array_in[i]) + "\'"+commaChar+"\n")
115 sout += offsetStr + "]\n"
116 return sout
117
118
119 def extractLinesAfterRegex(string_in, regex_in):
120 #print("regex_in = " + regex_in)
121 reMatch = re.compile(regex_in)
122 linesExtracted = ""
123 foundRegex = False
124 for line in string_in.strip().splitlines():
125 #print("line = '" + line + "'")
126 if not foundRegex:
127 matchObj = reMatch.match(line)
128 #print("matchObj = " + matchObj)
129 if matchObj:
130 foundRegex = True
131 if foundRegex:
132 linesExtracted += line + "\n"
133 return linesExtracted
134
135
136 def extractLinesMatchingRegex(string_in, regex_in):
137 #print("regex_in = " + regex_in)
138 string_in = s(string_in)
139 reMatch = re.compile(regex_in)
140 linesExtracted = ""
141 for line in string_in.strip().splitlines():
142 #print("line = '" + line + "'")
143 matchObj = reMatch.match(line)
144 #print("matchObj = " + matchObj)
145 if matchObj:
146 linesExtracted += line + "\n"
147 return linesExtracted
148 # NOTE: Above is *NOT* using tested!
149
150
151 def extractLinesMatchingSubstr(string_in, substr_in):
152 #print("substr_in = '" + substr_in + "'")
153 string_in = s(string_in)
154 linesExtracted = ""
155 for line in string_in.strip().splitlines():
156 #print("line = '" + line + "'")
157 if substr_in in line:
158 #print("matched '" + substr_in + "'")
159 linesExtracted += line + "\n"
160 return linesExtracted
161 # NOTE: Above is *NOT* unit tested!
162
163
164 # Convert a dictionary to a string, using a sorted set of keys.
165 #
166 # This is needed to provide a portable string representation across various
167 # versions of Python and platforms (see TriBITS GitHub Issue #119).
168 def sorted_dict_str(d):
169 items = []
170 keys = list(d.keys())
171 keys.sort()
172 for key in keys:
173 if isinstance(d[key], dict):
174 value = sorted_dict_str(d[key])
175 else:
176 value = repr(d[key])
177 items.append(repr(key) + ": " + value)
178 return "{" + ", ".join(items) + "}"
179
180
181
182 ##############################################
183 # System command unit testing utilities
184 ##############################################
185
186
187 class InterceptedCmndStruct:
188
189 def __init__(self, cmndRegex, cmndReturn, cmndOutput):
190 self.cmndRegex = cmndRegex
191 self.cmndReturn = cmndReturn
192 self.cmndOutput = cmndOutput
193
194 def __str__(self):
195 return "{cmndRegex='"+self.cmndRegex+"'," \
196 " cmndReturn="+str(self.cmndReturn)+"," \
197 " cmndOutput='"+str(self.cmndOutput)+"'}"
198
199 #
200 # Class that is used to record a set of commands that will be used to
201 # intercept commands
202 #
203
204 class SysCmndInterceptor:
205
206 def __init__(self):
207 self.__fallThroughCmndRegexList = []
208 self.__interceptedCmndStructList = []
209 self.__allowExtraCmnds = True
210
211 def setFallThroughCmndRegex(self, cmndRegex):
212 self.__fallThroughCmndRegexList.append(cmndRegex)
213
214 def setInterceptedCmnd(self, cmndRegex, cmndReturn, cmndOutput=None):
215 self.__interceptedCmndStructList.append(
216 InterceptedCmndStruct(cmndRegex, cmndReturn, cmndOutput) )
217
218 def setAllowExtraCmnds(self, allowExtraCmnds):
219 self.__allowExtraCmnds = allowExtraCmnds
220
221 def hasInterceptedCmnds(self):
222 return len(self.__interceptedCmndStructList) > 0
223
224 def getFallThroughCmndRegexList(self):
225 return self.__fallThroughCmndRegexList[:]
226
227 def getInterceptedCmndStructList(self):
228 return self.__interceptedCmndStructList[:]
229
230 def doProcessInterceptedCmnd(self, cmnd):
231 #print("doProcessInterceptedCmnd(): cmnd='" + cmnd + "'")
232 if self.isFallThroughCmnd(cmnd):
233 return False
234 if len(self.__interceptedCmndStructList) > 0:
235 return True
236 if not self.__allowExtraCmnds:
237 return True
238 return False
239
240 def isFallThroughCmnd(self, cmnd):
241 for interceptCmndStruct in self.__interceptedCmndStructList:
242 if re.match(interceptCmndStruct.cmndRegex, cmnd):
243 return False
244 for cmndRegex in self.__fallThroughCmndRegexList:
245 if re.match(cmndRegex, cmnd):
246 return True
247 return False
248
249 def nextInterceptedCmndStruct(self, cmnd):
250 assert(not self.isFallThroughCmnd(cmnd))
251 if len(self.__interceptedCmndStructList) == 0:
252 raise Exception("Error, cmnd='"+cmnd+"' is past the last expected command!")
253 ics = self.__interceptedCmndStructList[0]
254 if not re.match(ics.cmndRegex, cmnd):
255 raise Exception("Error, cmnd='" + cmnd + "' did not match the" \
256 " expected regex='" + ics.cmndRegex + "'!")
257 self.__interceptedCmndStructList.pop(0)
258 return (ics.cmndReturn, ics.cmndOutput)
259
260 def clear(self):
261 self.__fallThroughCmndRegexList = []
262 self.__interceptedCmndStructList = []
263 self.__allowExtraCmnds = True
264
265 def readCommandsFromStr(self, cmndsStr):
266 lines = cmndsStr.splitlines()
267 for line in lines:
268 #print("line: '" + line + "'")
269 if line == "":
270 continue
271 splitArray = line.split(':')
272 (tag, entry) = (splitArray[0], ':'.join(splitArray[1:]))
273 #(tag, entry) = line.split(':')
274 #print("(tag, entry) = " + str((tag, entry)))
275 if tag == "FT":
276 self.__fallThroughCmndRegexList.append(entry.strip())
277 elif tag == "IT":
278 entryArray = entry.split(';')
279 if len(entryArray) < 3:
280 raise Exception("Error, invalid line {"+line+"}")
281 cmndRegex = entryArray[0]
282 cmndReturn = entryArray[1]
283 cmndOutput = ""
284 for cmndOutputEntry in entryArray[2:]:
285 #print("cmndOutputEntry = {" + cmndOutputEntry + "}")
286 cmndOutput += cmndOutputEntry.strip()[1:-1]+"\n"
287 #print("(cmndRegex, cmndReturn, cmndOutput) = " +
288 # str((cmndRegex, cmndReturn, cmndOutput)))
289 self.__interceptedCmndStructList.append(
290 InterceptedCmndStruct(cmndRegex.strip(), int(cmndReturn), cmndOutput)
291 )
292 else:
293 raise Exception("Error, invalid tag = '"+tag+"'!")
294
295 def assertAllCommandsRun(self):
296 if len(self.__interceptedCmndStructList) > 0:
297 raise Exception("Error, all of the commands have not been run starting with" \
298 " the command " + str(self.__interceptedCmndStructList[0])
299 + "!")
300
301
302 g_sysCmndInterceptor = SysCmndInterceptor()
303
304
305 # Read the command interepts from a file?
306 cmndInterceptsFile = os.environ.get(
307 "GENERAL_SCRIPT_SUPPORT_CMND_INTERCEPTS_FILE","")
308 if cmndInterceptsFile:
309 cmndInterceptsFileStr = open(cmndInterceptsFile, 'r').read()
310 print("\nReading system command intercepts from file '" +
311 cmndInterceptsFile +
312 "' with contents:\n" +
313 "-----------------------------------\n" +
314 cmndInterceptsFileStr +
315 "-----------------------------------\n")
316 g_sysCmndInterceptor.readCommandsFromStr(cmndInterceptsFileStr)
317 g_sysCmndInterceptor.setAllowExtraCmnds(False)
318
319
320 # Dump all commands being performed?
321 g_dumpAllSysCmnds = "GENERAL_SCRIPT_SUPPORT_DUMD_COMMANDS" in os.environ
322
323
324 # Run a command (or the mock of that command) and optionally return the stdout
325 #
326 # Returns the commandline exit code or, if 'rtnOutput=True' returns the tuple
327 # (rtnCode, cmndOutput).
328 #
329 # For Python 2, this returns stdout as a standard ASCII byte array for the
330 # output. For Python 3, the stdout output is converted to standard unicode
331 # (i.e. 'utf-8')
332 #
333 def runSysCmndInterface(cmnd, outFile=None, rtnOutput=False, extraEnv=None, \
334 workingDir="", getStdErr=False \
335 ):
336 if g_dumpAllSysCmnds:
337 print("\nDUMP SYS CMND: " + cmnd + "\n")
338 if outFile!=None and rtnOutput==True:
339 raise Exception("Error, both outFile and rtnOutput can not be true!")
340 if g_sysCmndInterceptor.doProcessInterceptedCmnd(cmnd):
341 (cmndReturn, cmndOutput) = g_sysCmndInterceptor.nextInterceptedCmndStruct(cmnd)
342 if rtnOutput:
343 if cmndOutput==None:
344 raise Exception("Error, the command '"+cmnd+"' gave None output when" \
345 " non-null output was expected!")
346 return (cmndOutput, cmndReturn)
347 if outFile:
348 writeStrToFile(outFile, cmndOutput)
349 return cmndReturn
350 # Else, fall through
351 if extraEnv:
352 fullEnv = os.environ.copy()
353 fullEnv.update(extraEnv)
354 else:
355 fullEnv = None
356 pwd = None
357 if workingDir:
358 pwd = os.getcwd()
359 os.chdir(workingDir)
360 rtnObject = None
361 try:
362 if rtnOutput:
363 if getStdErr:
364 child = subprocess.Popen(cmnd, shell=True, stdout=subprocess.PIPE,
365 stderr = subprocess.STDOUT, env=fullEnv)
366 else:
367 child = subprocess.Popen(cmnd, shell=True, stdout=subprocess.PIPE,
368 env=fullEnv)
369 data = child.stdout.read()
370 if sys.version_info >= (3,):
371 data = data.decode('utf-8')
372 child.stdout.close()
373 child.wait()
374 rtnCode = child.returncode
375 rtnObject = (data, rtnCode)
376 else:
377 outFileHandle = None
378 if outFile:
379 outFileHandle = open(outFile, 'w')
380 rtnCode = subprocess.call(cmnd, shell=True, stderr=subprocess.STDOUT,
381 stdout=outFileHandle, env=fullEnv)
382 if outFileHandle: outFileHandle.close()
383 rtnObject = rtnCode
384 finally:
385 if pwd: os.chdir(pwd)
386 return rtnObject
387
388
389 ######################################
390 # System interaction utilities
391 ######################################
392
393
394 def runSysCmnd(cmnd, throwExcept=True, outFile=None, workingDir="",
395 extraEnv=None, echoCmndForDebugging=False \
396 ):
397 """Run system command and optionally throw on failure"""
398 sys.stdout.flush()
399 sys.stderr.flush()
400 try:
401 outFileHandle = None
402 if echoCmndForDebugging: print("Running: "+cmnd)
403 rtnCode = runSysCmndInterface(cmnd, outFile=outFile, extraEnv=extraEnv,
404 workingDir=workingDir)
405 except OSError as e:
406 rtnCode = 1 # Just some error code != 0 please!
407 if rtnCode != 0 and throwExcept:
408 raise RuntimeError("Error, the command '%s' failed with error code %d"
409 % (cmnd, rtnCode))
410 return rtnCode
411
412
413 def echoRunSysCmnd(cmnd, throwExcept=True, outFile=None, msg=None,
414 timeCmnd=False, verbose=True, workingDir="", returnTimeCmnd=False,
415 extraEnv=None
416 ):
417 """Echo command to be run and run command with runSysCmnd()"""
418 if verbose:
419 print("\nRunning: " + cmnd + "\n")
420 if workingDir:
421 print(" Running in working directory: " + workingDir + " ...\n")
422 if extraEnv:
423 print(" Appending environment:" + sorted_dict_str(extraEnv) + "\n")
424 if outFile:
425 print(" Writing console output to file " + outFile + " ...")
426 if msg and verbose:
427 print(" " + msg + "\n")
428 t1 = time.time()
429 totalTimeMin = -1.0
430 try:
431 rtn = runSysCmnd(cmnd, throwExcept, outFile, workingDir, extraEnv)
432 finally:
433 if timeCmnd:
434 t2 = time.time()
435 totalTimeMin = (t2-t1)/60.0
436 if verbose:
437 print("\n Runtime for command = %f minutes" % totalTimeMin)
438 if returnTimeCmnd:
439 return (rtn, totalTimeMin)
440 return rtn
441
442
443 # Return a shell command's output and optionally its return code
444 #
445 # For Python 2, this returns a standard ASCII byte array for the output. For
446 # Python 3, the output is converted to standard unicode (i.e. 'utf-8')
447 #
448 def getCmndOutput(cmnd, stripTrailingSpaces=False, throwOnError=True, workingDir="", \
449 getStdErr=False, rtnCode=False \
450 ):
451 (data, errCode) = runSysCmndInterface(cmnd, rtnOutput=True, workingDir=workingDir,
452 getStdErr=getStdErr)
453 if errCode != 0:
454 if throwOnError:
455 raise RuntimeError('%s failed w/ exit code %d:\n\n%s' % (cmnd, errCode, data))
456 dataToReturn = data
457 if stripTrailingSpaces:
458 dataToReturn = data.rstrip()
459 if rtnCode:
460 return (dataToReturn, errCode)
461 return dataToReturn
462
463
464 def pidStillRunning(pid):
465 #print("\npid = '" + pid + "'")
466 cmnd = "kill -s 0 "+pid
467 cmndReturn = runSysCmnd(cmnd, False)
468 #print("\ncmndReturn = " + cmndReturn)
469 return cmndReturn == 0
470
471
472 ######################################
473 # File/path utilities
474 ######################################
475
476
477 def getFilePathArray(filePathStr):
478 return filePathStr.split('/')
479
480
481 def joinDirs(dirArray):
482 """
483 Join directories.
484
485 2009/06/09: rabartl: We should be able to just use os.path.join(...) but I
486 found when used in at least on context that it resulted in not joining the
487 elements but instead just returning the array.
488 """
489 dirPath = ""
490 for dir in dirArray:
491 if not dirPath:
492 dirPath = dir
493 else:
494 dirPath = dirPath + "/" + dir
495 return dirPath
496
497
498 def downDirsArray(numDownDirs):
499 downDirsPathArray = []
500 for i in range(0, numDownDirs):
501 downDirsPathArray.append("..")
502 #print("\ndownDirsPathArray = " + downDirsPathArray)
503 return downDirsPathArray
504
505
506 def normalizePath(path):
507 return os.path.normpath(path)
508
509
510 def echoChDir(dirName, verbose=True):
511 if verbose:
512 print("\nChanging current directory to \'" + dirName + "\'")
513 if not os.path.isdir(dirName):
514 raise OSError("Error, the directory \'"+dirName+"\' does not exist in the" \
515 + " base directory \'"+os.getcwd()+"\"!" )
516 os.chdir(dirName)
517 if verbose:
518 print("\nCurrent directory is \'" + os.getcwd() + "\'\n")
519
520
521 def createDir(dirName, cdIntoDir=False, verbose=False):
522 """Create a directory if it does not exist"""
523 if os.path.exists(dirName):
524 if not os.path.isdir(dirName):
525 errMsg = "\nError the path '" + dirName + \
526 "'already exists but it is not a directory!"
527 if verbose: print(errMsg)
528 raise RuntimeError(errMsg)
529 if verbose: print("\nThe directory " + dirName + "already exists!")
530 else:
531 if verbose: print("\nCreating directory " + dirName + " ...")
532 os.mkdir(dirName)
533 if cdIntoDir:
534 echoChDir(dirName, verbose=verbose)
535
536
537 def createDirsFromPath(path):
538 #print("\npath = " + path)
539 pathList = path.split("/")
540 #print("\npathList = " + pathList)
541 if not pathList[0]:
542 currDir = "/"
543 for dir in pathList:
544 currDir = os.path.join(currDir, dir)
545 if currDir and not os.path.exists(currDir):
546 #print("\ncurrDir = " + currDir)
547 createDir(currDir)
548
549
550 def expandDirsDict(trilinosDirsDict_inout):
551
552 for dir in list(trilinosDirsDict_inout):
553 subdirsList = dir.split("/")
554 #print("\nsubdirsList = " + subdirsList)
555 for i in range(len(subdirsList)):
556 trilinosDirsDict_inout.update({joinDirs(subdirsList[:i+1]) : 0})
557
558
559 def removeIfExists(fileName):
560 if os.path.exists(fileName):
561 echoRunSysCmnd("rm "+fileName)
562
563
564 def removeDirIfExists(dirName, verbose=False):
565 if os.path.exists(dirName):
566 if verbose:
567 print("Removing existing directory '" + dirName + "' ...")
568 echoRunSysCmnd("rm -rf "+dirName)
569
570
571 def writeStrToFile(fileName, fileBodyStr):
572 with open(fileName, 'w') as fileHandle:
573 fileHandle.write(fileBodyStr)
574
575
576 def readStrFromFile(fileName):
577 with open(fileName, 'r') as fileHandle:
578 return fileHandle.read()
579
580
581 def getFileNamesWithFileTag( baseDir, fileTag ):
582 """Get a list of file names with a given date stamp"""
583 return getCmndOutput(
584 'cd %s && ls *%s*' % (baseDir, fileTag),
585 throwOnError=False
586 ).split()
587
588
589 def getFileNameFromGlob( baseDir, fileNameGlob ):
590 return getCmndOutput("cd "+baseDir+" && ls "+fileNameGlob, True, False)
591
592
593 def isEmptyDir( absDir ):
594 return (len(os.listdir(absDir)) == 0)
595
596
597 def getDirSizeInGb(dir):
598 sizeIn1024Kb = int(getCmndOutput("du -s "+dir).split('\t')[0])
599 #print("\nsizeIn1024Kb = " + str(sizeIn1024Kb))
600 return float(sizeIn1024Kb)/1e+6 # Size in Gb!
601
602
603 def isPathChar(char):
604 return (char.isalnum() or char == '/') and (not char == ' ')
605
606
607 # 2008/07/08: rabartl: This silly function is needed because on the sun
608 # machine (i.e. sass8000), the 'which' command returns some non-printable
609 # chars from the beginning of the output string that don't form a valid path.
610 # This was *very* hard to debug but this function is able to select the valid
611 # path string. This has been tested at least on linux and the sun and should
612 # work anywhere.
613 def cleanBadPath(inputPath):
614 cleanPath = ""
615 for i in range(len(inputPath)-1, -1, -1):
616 char = inputPath[i]
617 if not isPathChar(char):
618 break
619 cleanPath = char + cleanPath
620 return cleanPath
621
622
623 def getRelativePathFrom1to2(absPath1, absPath2):
624 #print("\nabsPath1 =" + absPath1)
625 #print("\nabsPath2 =" + absPath2)
626 absPath1_array = absPath1.split('/')
627 absPath2_array = absPath2.split('/')
628 #print("\nabsPath1_array =" + absPath1_array)
629 #print("\nabsPath2_array =" + absPath2_array)
630 absPath1_array_len = len(absPath1_array)
631 absPath2_array_len = len(absPath2_array)
632 maxBaseDirDepth = min(absPath1_array_len, absPath2_array_len)
633 baseDirDepth = 0
634 for dirIdx in range(0, maxBaseDirDepth):
635 dir1 = absPath1_array[dirIdx]
636 dir2 = absPath2_array[dirIdx]
637 if dir1 != dir2:
638 break
639 baseDirDepth = baseDirDepth + 1
640 #print("\nbaseDirDepth = %d" % baseDirDepth)
641 numDownDirs = absPath1_array_len - baseDirDepth
642 #print("\nnumDownDirs = %d" % numDownDirs)
643 if numDownDirs > 0:
644 downDirPath = joinDirs(downDirsArray(numDownDirs))
645 else:
646 downDirPath = "."
647 #print("\ndownDirPath = '" + downDirPath + "'")
648 if baseDirDepth == absPath2_array_len:
649 upDirPath = "."
650 else:
651 upDirPath = joinDirs(absPath2_array[baseDirDepth:])
652 #print("\nupDirPath = " + upDirPath )
653 #print("\nupDirPath = '" + upDirPath + "'")
654 relPath = os.path.join(downDirPath, upDirPath)
655 if relPath == "./.":
656 return "."
657 return relPath
658
659
660 def getExecBaseDir(execName):
661 whichOutput = getCmndOutput("type -p "+execName, True, False)
662 # Handle the outpue 'execName is execFullPath' output
663 execFullPath = whichOutput.split(' ')[-1]
664 #print("\nexecFullPath = " + execFullPath)
665 execNameMatchRe = r"^(.+)/"+execName
666 execNameGroups = re.findall(execNameMatchRe, execFullPath)
667 #print("\nexecNameGroups = " + execNameGroups)
668 if not execNameGroups:
669 return None
670 execBaseDir = execNameGroups[0]
671 #print("\nexecBaseDir = \"" + execBaseDir + "\"")
672 #execBaseDir = cleanBadPath(execBaseDir)
673 #print("\nexecBaseDir = \"" + execBaseDir + "\"")
674 return execBaseDir
675
676
677 def extractAppendUniqueDirsDictFromFileNames(filenamesArray, dirsDict):
678 for filename in filenamesArray:
679 dirsDict.update( { normalizePath(os.path.dirname(filename)) : 0 } )
680
681
682 def copyFileAndReplaceTokens( scriptsDir, inputFile, tokenReplacementList,
683 outputFile ):
684
685 """Copies an input stub file and then does a set of token replacements"""
686
687 echoRunSysCmnd("cp "+inputFile+" "+outputFile, verbose=verboseDebug)
688
689 for tokenReplacementPair in tokenReplacementList:
690 oldToken = tokenReplacementPair[0]
691 newToken = tokenReplacementPair[1]
692 echoRunSysCmnd( scriptsDir+"/token-replace.pl "+oldToken+" "+newToken\
693 +" "+outputFile+" "+outputFile, verbose=verboseDebug );
694 # ToDo: Replace above with native re commands
695
696
697 class TeeOutput(object):
698 """
699 Object that directs all calls to its write method to stdout as well
700 as a file. This is to be used as a simple replacement for the Unix
701 tee command.
702 """
703 def __init__(self, outputfile):
704 """ Constructor takes a file-like object to write output to."""
705 self._realstdout = sys.stdout
706 self._outputfile = outputfile
707
708 def _safe_outputfile_method(self, methodname, *args):
709 """
710 Calls the method specified by methodname with the given args on
711 the internal file object if it is non-null.
712 """
713 if self._outputfile is not None:
714 if hasattr(self._outputfile, methodname):
715 method = getattr(self._outputfile, methodname)
716 if method and callable(method):
717 method(*args)
718
719 def write(self, data):
720 """
721 Write the given data to stdout and to the log file.
722 """
723 self._realstdout.write(data)
724 self._safe_outputfile_method('write', data)
725
726 def flush(self):
727 """
728 Flush the internal file buffers.
729 """
730 self._realstdout.flush()
731 self._safe_outputfile_method('flush')
732
733
734 ######################################
735 # Shell argument helpers
736 ######################################
737
738
739 reCmndLineArg = re.compile(r"(--.+=)(.+)")
740
741
742 def requoteCmndLineArgs(inArgs):
743 argsStr = ""
744 for arg in inArgs:
745 splitArg = arg.split("=")
746 newArg = None
747 if len(splitArg) == 1:
748 newArg = arg
749 else:
750 newArg = splitArg[0]+"=\""+'='.join(splitArg[1:])+"\""
751 #print("\nnewArg = " + newArg)
752 argsStr = argsStr+" "+newArg
753 return argsStr
754
755
756 def commandLineOptionsToList(stringOptions):
757 """
758 Convert a string of space separated command line options to a python
759 list of the individual optionstrings.
760 TODO: Handle shell quoting.
761 """
762 return stringOptions.split()
763
764
765 class ConfigurableOptionParser(optparse.OptionParser):
766 """
767 OptionParser that accepts a python dictionary as a configuration
768 file to provide default value overrides for the options.
769 """
770 def __init__(self, configuration, **kwargs):
771 """
772 Constructor accepts a configuration dictionary with default values
773 for arguments and all of the OptionParser arguments as well.
774 """
775 optparse.OptionParser.__init__(self, **kwargs)
776 self._configuration = configuration
777
778 def add_option(self, *args, **kwargs):
779 """
780 Checks for a default override in the configuration dictionary and
781 modifies the default and help arguments before dispatching them to
782 the base class implementation.
783 """
784 if 'default' in kwargs:
785 for arg in args:
786 kwargs['default'] = self._configuration.get(arg, kwargs['default'])
787 optparse.OptionParser.add_option(self, *args, **kwargs)
788
789
790 ######################################
791 # Debugging support
792 ######################################
793
794
795 def printStackTrace():
796 sys.stdout.flush()
797 traceback.print_exc()
798
799
800 class ErrorCaptureOptionParser(optparse.OptionParser):
801 __sawError = None
802 def __init__(self, usage="%prog [options]", version=None):
803 optparse.OptionParser.__init__(self, usage, version)
804 __sawError = False
805 def error(self, msg):
806 raise Exception("Received error message: " + msg)
807
808
809 ######################################
810 # HTML directory browsing
811 ######################################
812
813
814 def createIndexHtmlBrowserList(baseDir, fileDirList = None):
815 htmlList = ""
816 # Get the fileDirList if empty
817 if not fileDirList:
818 fileDirList = os.listdir(baseDir)
819 fileDirList.sort()
820 # Create the HTML header
821 htmlList += "" \
822 + "<ul>\n" \
823 + "<li><a href=\"..\">..</a></li>\n"
824 # Fill in links to directories first
825 for fd in fileDirList:
826 absFd = baseDir+"/"+fd
827 #print("isfile(" + fd + ") = " + str(os.path.isfile(absFd)))
828 #print("isdir(" + fd + ") = " + str(os.path.isdir(absFd) ))
829 if not os.path.isfile(absFd):
830 htmlList = htmlList \
831 +"<li>dir: <a href=\""+fd+"\">"+fd+"</a></li>\n"
832 # Fill in links to regular files second
833 for fd in fileDirList:
834 absFd = baseDir+"/"+fd
835 if os.path.isfile(absFd):
836 if fd != 'index.html':
837 htmlList = htmlList \
838 +"<li>file: <a href=\""+fd+"\">"+fd+"</a></li>\n"
839 # Write the footer
840 htmlList = htmlList \
841 + "</ul>\n"
842 return htmlList
843
844
845
846 def createIndexHtmlBrowserFile(baseDir, fileDirList):
847 """Creates an HTML browser file as a returned string."""
848 htmlFile = "" \
849 + "<html>\n" \
850 + "<head>\n" \
851 + "<title>"+baseDir+"</title>\n" \
852 + "</head>\n" \
853 + "<body>\n" \
854 + "<b>"+baseDir+"</b>\n" \
855 + createIndexHtmlBrowserList(baseDir, fileDirList) \
856 + "</body>\n" \
857 + "</html>\n"
858 return htmlFile
859
860
861 def createHtmlBrowserFiles(absBaseDir, depth, verbose=False):
862
863 """Create a hierarchy of index.html files that will build a directory/file
864 browser for a web server that will not allow directory/file browsing."""
865
866 #print("\nEntering createHtmlBrowserFiles(" + absBaseDir + ",%d" % (depth) + ")")
867
868 # Get the list of all of the files/directories in absBaseDir
869 fileDirList = os.listdir(absBaseDir)
870 fileDirList.sort()
871 #print("\nfileDirList = " + str(fileDirList)
872 #sys.stdout.flush()
873
874 # Get the index.html file HTML
875 indexHtml = createIndexHtmlBrowserFile(absBaseDir, fileDirList)
876 #print("\nindexHtml:\n" + indexHtml)
877
878 # Write the index.html file
879 indexFileName = absBaseDir+"/index.html"
880 if verbose:
881 print("\nWriting " + indexFileName)
882 open(indexFileName,'w').write(indexHtml)
883
884 # Loop through all of the directories and recursively call this function
885 if depth > 0:
886 for fd in fileDirList:
887 absFd = absBaseDir+"/"+fd
888 if os.path.isdir(absFd):
889 subDir = absFd
890 #print("\nCalling createHtmlBrowserFiles(" + subDir + ",%d" % (depth-1)
891 # + ")")
892 createHtmlBrowserFiles(absBaseDir+"/"+fd,depth-1)
893
894 #print("\nLeaving createHtmlBrowserFiles(" + absBaseDir + ",%d" % (depth) +
895 # ")")