| 1 |
# @HEADER |
| 2 |
# ***************************************************************************** |
| 3 |
# TriBITS: Tribal Build, Integrate, and Test System |
| 4 |
# |
| 5 |
# Copyright 2013-2016 NTESS and the TriBITS contributors. |
| 6 |
# SPDX-License-Identifier: BSD-3-Clause |
| 7 |
# ***************************************************************************** |
| 8 |
# @HEADER |
| 9 |
|
| 10 |
""" |
| 11 |
Python module containing general support functions for creating scripts |
| 12 |
""" |
| 13 |
|
| 14 |
# |
| 15 |
# Check the python version |
| 16 |
# |
| 17 |
|
| 18 |
import sys |
| 19 |
if sys.version_info < (2,7): |
| 20 |
print("Error, Python version is " + sys.version + " < 2.7!") |
| 21 |
sys.exit(1) |
| 22 |
|
| 23 |
# |
| 24 |
# Import commands |
| 25 |
# |
| 26 |
import os |
| 27 |
import re |
| 28 |
import math |
| 29 |
import subprocess |
| 30 |
import time |
| 31 |
import datetime |
| 32 |
import optparse |
| 33 |
import traceback |
| 34 |
|
| 35 |
from Python2and3 import b, s, u |
| 36 |
|
| 37 |
verboseDebug = False |
| 38 |
|
| 39 |
|
| 40 |
# |
| 41 |
# Determine what system we are on: |
| 42 |
# |
| 43 |
|
| 44 |
rePlatformName = re.compile(r"^[a-zA-Z]+") |
| 45 |
platformStr = rePlatformName.findall(sys.platform)[0] |
| 46 |
#print("\nplatformStr = " + platformStr) |
| 47 |
|
| 48 |
|
| 49 |
###################################### |
| 50 |
# Script location functions |
| 51 |
###################################### |
| 52 |
|
| 53 |
|
| 54 |
def getScriptBaseDir(): |
| 55 |
return os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) |
| 56 |
|
| 57 |
|
| 58 |
def getScriptName(): |
| 59 |
return os.path.basename(os.path.dirname(sys.argv[0])) |
| 60 |
|
| 61 |
def getCompleteFileDirname(filename): |
| 62 |
return os.path.dirname(os.path.realpath(os.path.abspath(filename))) |
| 63 |
|
| 64 |
###################################### |
| 65 |
# List helper functions |
| 66 |
###################################### |
| 67 |
|
| 68 |
|
| 69 |
def findInSequence(seq, item): |
| 70 |
for i in range(0, len(seq)): |
| 71 |
if seq[i] == item: |
| 72 |
return i |
| 73 |
return -1 |
| 74 |
|
| 75 |
|
| 76 |
def removeItemsFromList(list, items): |
| 77 |
numItemsRemoved = 0 |
| 78 |
for item in items: |
| 79 |
if item in list: |
| 80 |
idx = list.index(item) |
| 81 |
del list[idx] |
| 82 |
numItemsRemoved = numItemsRemoved + 1 |
| 83 |
return numItemsRemoved |
| 84 |
|
| 85 |
|
| 86 |
###################################### |
| 87 |
# String helper functions |
| 88 |
###################################### |
| 89 |
|
| 90 |
|
| 91 |
def stripWhitespaceFromStringList(stringList): |
| 92 |
return [x.strip() for x in stringList] |
| 93 |
|
| 94 |
|
| 95 |
def isSubstrInMultiLineString(inputStr, findStr): |
| 96 |
return inputStr.find(findStr) >= 0 |
| 97 |
|
| 98 |
|
| 99 |
def getStrUnderlineStr(width): |
| 100 |
underlineStr = "" |
| 101 |
for i in range(width): |
| 102 |
underlineStr += "-" |
| 103 |
return underlineStr |
| 104 |
|
| 105 |
|
| 106 |
def arrayToFormattedString(array_in, offsetStr = ""): |
| 107 |
sout = "" |
| 108 |
sout += offsetStr + "[\n" |
| 109 |
for i in range(0, len(array_in)): |
| 110 |
if i != len(array_in)-1: |
| 111 |
commaChar = "," |
| 112 |
else: |
| 113 |
commaChar = "" |
| 114 |
sout += (offsetStr + " \'" + str(array_in[i]) + "\'"+commaChar+"\n") |
| 115 |
sout += offsetStr + "]\n" |
| 116 |
return sout |
| 117 |
|
| 118 |
|
| 119 |
def extractLinesAfterRegex(string_in, regex_in): |
| 120 |
#print("regex_in = " + regex_in) |
| 121 |
reMatch = re.compile(regex_in) |
| 122 |
linesExtracted = "" |
| 123 |
foundRegex = False |
| 124 |
for line in string_in.strip().splitlines(): |
| 125 |
#print("line = '" + line + "'") |
| 126 |
if not foundRegex: |
| 127 |
matchObj = reMatch.match(line) |
| 128 |
#print("matchObj = " + matchObj) |
| 129 |
if matchObj: |
| 130 |
foundRegex = True |
| 131 |
if foundRegex: |
| 132 |
linesExtracted += line + "\n" |
| 133 |
return linesExtracted |
| 134 |
|
| 135 |
|
| 136 |
def extractLinesMatchingRegex(string_in, regex_in): |
| 137 |
#print("regex_in = " + regex_in) |
| 138 |
string_in = s(string_in) |
| 139 |
reMatch = re.compile(regex_in) |
| 140 |
linesExtracted = "" |
| 141 |
for line in string_in.strip().splitlines(): |
| 142 |
#print("line = '" + line + "'") |
| 143 |
matchObj = reMatch.match(line) |
| 144 |
#print("matchObj = " + matchObj) |
| 145 |
if matchObj: |
| 146 |
linesExtracted += line + "\n" |
| 147 |
return linesExtracted |
| 148 |
# NOTE: Above is *NOT* using tested! |
| 149 |
|
| 150 |
|
| 151 |
def extractLinesMatchingSubstr(string_in, substr_in): |
| 152 |
#print("substr_in = '" + substr_in + "'") |
| 153 |
string_in = s(string_in) |
| 154 |
linesExtracted = "" |
| 155 |
for line in string_in.strip().splitlines(): |
| 156 |
#print("line = '" + line + "'") |
| 157 |
if substr_in in line: |
| 158 |
#print("matched '" + substr_in + "'") |
| 159 |
linesExtracted += line + "\n" |
| 160 |
return linesExtracted |
| 161 |
# NOTE: Above is *NOT* unit tested! |
| 162 |
|
| 163 |
|
| 164 |
# Convert a dictionary to a string, using a sorted set of keys. |
| 165 |
# |
| 166 |
# This is needed to provide a portable string representation across various |
| 167 |
# versions of Python and platforms (see TriBITS GitHub Issue #119). |
| 168 |
def sorted_dict_str(d): |
| 169 |
items = [] |
| 170 |
keys = list(d.keys()) |
| 171 |
keys.sort() |
| 172 |
for key in keys: |
| 173 |
if isinstance(d[key], dict): |
| 174 |
value = sorted_dict_str(d[key]) |
| 175 |
else: |
| 176 |
value = repr(d[key]) |
| 177 |
items.append(repr(key) + ": " + value) |
| 178 |
return "{" + ", ".join(items) + "}" |
| 179 |
|
| 180 |
|
| 181 |
|
| 182 |
############################################## |
| 183 |
# System command unit testing utilities |
| 184 |
############################################## |
| 185 |
|
| 186 |
|
| 187 |
class InterceptedCmndStruct: |
| 188 |
|
| 189 |
def __init__(self, cmndRegex, cmndReturn, cmndOutput): |
| 190 |
self.cmndRegex = cmndRegex |
| 191 |
self.cmndReturn = cmndReturn |
| 192 |
self.cmndOutput = cmndOutput |
| 193 |
|
| 194 |
def __str__(self): |
| 195 |
return "{cmndRegex='"+self.cmndRegex+"'," \ |
| 196 |
" cmndReturn="+str(self.cmndReturn)+"," \ |
| 197 |
" cmndOutput='"+str(self.cmndOutput)+"'}" |
| 198 |
|
| 199 |
# |
| 200 |
# Class that is used to record a set of commands that will be used to |
| 201 |
# intercept commands |
| 202 |
# |
| 203 |
|
| 204 |
class SysCmndInterceptor: |
| 205 |
|
| 206 |
def __init__(self): |
| 207 |
self.__fallThroughCmndRegexList = [] |
| 208 |
self.__interceptedCmndStructList = [] |
| 209 |
self.__allowExtraCmnds = True |
| 210 |
|
| 211 |
def setFallThroughCmndRegex(self, cmndRegex): |
| 212 |
self.__fallThroughCmndRegexList.append(cmndRegex) |
| 213 |
|
| 214 |
def setInterceptedCmnd(self, cmndRegex, cmndReturn, cmndOutput=None): |
| 215 |
self.__interceptedCmndStructList.append( |
| 216 |
InterceptedCmndStruct(cmndRegex, cmndReturn, cmndOutput) ) |
| 217 |
|
| 218 |
def setAllowExtraCmnds(self, allowExtraCmnds): |
| 219 |
self.__allowExtraCmnds = allowExtraCmnds |
| 220 |
|
| 221 |
def hasInterceptedCmnds(self): |
| 222 |
return len(self.__interceptedCmndStructList) > 0 |
| 223 |
|
| 224 |
def getFallThroughCmndRegexList(self): |
| 225 |
return self.__fallThroughCmndRegexList[:] |
| 226 |
|
| 227 |
def getInterceptedCmndStructList(self): |
| 228 |
return self.__interceptedCmndStructList[:] |
| 229 |
|
| 230 |
def doProcessInterceptedCmnd(self, cmnd): |
| 231 |
#print("doProcessInterceptedCmnd(): cmnd='" + cmnd + "'") |
| 232 |
if self.isFallThroughCmnd(cmnd): |
| 233 |
return False |
| 234 |
if len(self.__interceptedCmndStructList) > 0: |
| 235 |
return True |
| 236 |
if not self.__allowExtraCmnds: |
| 237 |
return True |
| 238 |
return False |
| 239 |
|
| 240 |
def isFallThroughCmnd(self, cmnd): |
| 241 |
for interceptCmndStruct in self.__interceptedCmndStructList: |
| 242 |
if re.match(interceptCmndStruct.cmndRegex, cmnd): |
| 243 |
return False |
| 244 |
for cmndRegex in self.__fallThroughCmndRegexList: |
| 245 |
if re.match(cmndRegex, cmnd): |
| 246 |
return True |
| 247 |
return False |
| 248 |
|
| 249 |
def nextInterceptedCmndStruct(self, cmnd): |
| 250 |
assert(not self.isFallThroughCmnd(cmnd)) |
| 251 |
if len(self.__interceptedCmndStructList) == 0: |
| 252 |
raise Exception("Error, cmnd='"+cmnd+"' is past the last expected command!") |
| 253 |
ics = self.__interceptedCmndStructList[0] |
| 254 |
if not re.match(ics.cmndRegex, cmnd): |
| 255 |
raise Exception("Error, cmnd='" + cmnd + "' did not match the" \ |
| 256 |
" expected regex='" + ics.cmndRegex + "'!") |
| 257 |
self.__interceptedCmndStructList.pop(0) |
| 258 |
return (ics.cmndReturn, ics.cmndOutput) |
| 259 |
|
| 260 |
def clear(self): |
| 261 |
self.__fallThroughCmndRegexList = [] |
| 262 |
self.__interceptedCmndStructList = [] |
| 263 |
self.__allowExtraCmnds = True |
| 264 |
|
| 265 |
def readCommandsFromStr(self, cmndsStr): |
| 266 |
lines = cmndsStr.splitlines() |
| 267 |
for line in lines: |
| 268 |
#print("line: '" + line + "'") |
| 269 |
if line == "": |
| 270 |
continue |
| 271 |
splitArray = line.split(':') |
| 272 |
(tag, entry) = (splitArray[0], ':'.join(splitArray[1:])) |
| 273 |
#(tag, entry) = line.split(':') |
| 274 |
#print("(tag, entry) = " + str((tag, entry))) |
| 275 |
if tag == "FT": |
| 276 |
self.__fallThroughCmndRegexList.append(entry.strip()) |
| 277 |
elif tag == "IT": |
| 278 |
entryArray = entry.split(';') |
| 279 |
if len(entryArray) < 3: |
| 280 |
raise Exception("Error, invalid line {"+line+"}") |
| 281 |
cmndRegex = entryArray[0] |
| 282 |
cmndReturn = entryArray[1] |
| 283 |
cmndOutput = "" |
| 284 |
for cmndOutputEntry in entryArray[2:]: |
| 285 |
#print("cmndOutputEntry = {" + cmndOutputEntry + "}") |
| 286 |
cmndOutput += cmndOutputEntry.strip()[1:-1]+"\n" |
| 287 |
#print("(cmndRegex, cmndReturn, cmndOutput) = " + |
| 288 |
# str((cmndRegex, cmndReturn, cmndOutput))) |
| 289 |
self.__interceptedCmndStructList.append( |
| 290 |
InterceptedCmndStruct(cmndRegex.strip(), int(cmndReturn), cmndOutput) |
| 291 |
) |
| 292 |
else: |
| 293 |
raise Exception("Error, invalid tag = '"+tag+"'!") |
| 294 |
|
| 295 |
def assertAllCommandsRun(self): |
| 296 |
if len(self.__interceptedCmndStructList) > 0: |
| 297 |
raise Exception("Error, all of the commands have not been run starting with" \ |
| 298 |
" the command " + str(self.__interceptedCmndStructList[0]) |
| 299 |
+ "!") |
| 300 |
|
| 301 |
|
| 302 |
g_sysCmndInterceptor = SysCmndInterceptor() |
| 303 |
|
| 304 |
|
| 305 |
# Read the command interepts from a file? |
| 306 |
cmndInterceptsFile = os.environ.get( |
| 307 |
"GENERAL_SCRIPT_SUPPORT_CMND_INTERCEPTS_FILE","") |
| 308 |
if cmndInterceptsFile: |
| 309 |
cmndInterceptsFileStr = open(cmndInterceptsFile, 'r').read() |
| 310 |
print("\nReading system command intercepts from file '" + |
| 311 |
cmndInterceptsFile + |
| 312 |
"' with contents:\n" + |
| 313 |
"-----------------------------------\n" + |
| 314 |
cmndInterceptsFileStr + |
| 315 |
"-----------------------------------\n") |
| 316 |
g_sysCmndInterceptor.readCommandsFromStr(cmndInterceptsFileStr) |
| 317 |
g_sysCmndInterceptor.setAllowExtraCmnds(False) |
| 318 |
|
| 319 |
|
| 320 |
# Dump all commands being performed? |
| 321 |
g_dumpAllSysCmnds = "GENERAL_SCRIPT_SUPPORT_DUMD_COMMANDS" in os.environ |
| 322 |
|
| 323 |
|
| 324 |
# Run a command (or the mock of that command) and optionally return the stdout |
| 325 |
# |
| 326 |
# Returns the commandline exit code or, if 'rtnOutput=True' returns the tuple |
| 327 |
# (rtnCode, cmndOutput). |
| 328 |
# |
| 329 |
# For Python 2, this returns stdout as a standard ASCII byte array for the |
| 330 |
# output. For Python 3, the stdout output is converted to standard unicode |
| 331 |
# (i.e. 'utf-8') |
| 332 |
# |
| 333 |
def runSysCmndInterface(cmnd, outFile=None, rtnOutput=False, extraEnv=None, \ |
| 334 |
workingDir="", getStdErr=False \ |
| 335 |
): |
| 336 |
if g_dumpAllSysCmnds: |
| 337 |
print("\nDUMP SYS CMND: " + cmnd + "\n") |
| 338 |
if outFile!=None and rtnOutput==True: |
| 339 |
raise Exception("Error, both outFile and rtnOutput can not be true!") |
| 340 |
if g_sysCmndInterceptor.doProcessInterceptedCmnd(cmnd): |
| 341 |
(cmndReturn, cmndOutput) = g_sysCmndInterceptor.nextInterceptedCmndStruct(cmnd) |
| 342 |
if rtnOutput: |
| 343 |
if cmndOutput==None: |
| 344 |
raise Exception("Error, the command '"+cmnd+"' gave None output when" \ |
| 345 |
" non-null output was expected!") |
| 346 |
return (cmndOutput, cmndReturn) |
| 347 |
if outFile: |
| 348 |
writeStrToFile(outFile, cmndOutput) |
| 349 |
return cmndReturn |
| 350 |
# Else, fall through |
| 351 |
if extraEnv: |
| 352 |
fullEnv = os.environ.copy() |
| 353 |
fullEnv.update(extraEnv) |
| 354 |
else: |
| 355 |
fullEnv = None |
| 356 |
pwd = None |
| 357 |
if workingDir: |
| 358 |
pwd = os.getcwd() |
| 359 |
os.chdir(workingDir) |
| 360 |
rtnObject = None |
| 361 |
try: |
| 362 |
if rtnOutput: |
| 363 |
if getStdErr: |
| 364 |
child = subprocess.Popen(cmnd, shell=True, stdout=subprocess.PIPE, |
| 365 |
stderr = subprocess.STDOUT, env=fullEnv) |
| 366 |
else: |
| 367 |
child = subprocess.Popen(cmnd, shell=True, stdout=subprocess.PIPE, |
| 368 |
env=fullEnv) |
| 369 |
data = child.stdout.read() |
| 370 |
if sys.version_info >= (3,): |
| 371 |
data = data.decode('utf-8') |
| 372 |
child.stdout.close() |
| 373 |
child.wait() |
| 374 |
rtnCode = child.returncode |
| 375 |
rtnObject = (data, rtnCode) |
| 376 |
else: |
| 377 |
outFileHandle = None |
| 378 |
if outFile: |
| 379 |
outFileHandle = open(outFile, 'w') |
| 380 |
rtnCode = subprocess.call(cmnd, shell=True, stderr=subprocess.STDOUT, |
| 381 |
stdout=outFileHandle, env=fullEnv) |
| 382 |
if outFileHandle: outFileHandle.close() |
| 383 |
rtnObject = rtnCode |
| 384 |
finally: |
| 385 |
if pwd: os.chdir(pwd) |
| 386 |
return rtnObject |
| 387 |
|
| 388 |
|
| 389 |
###################################### |
| 390 |
# System interaction utilities |
| 391 |
###################################### |
| 392 |
|
| 393 |
|
| 394 |
def runSysCmnd(cmnd, throwExcept=True, outFile=None, workingDir="", |
| 395 |
extraEnv=None, echoCmndForDebugging=False \ |
| 396 |
): |
| 397 |
"""Run system command and optionally throw on failure""" |
| 398 |
sys.stdout.flush() |
| 399 |
sys.stderr.flush() |
| 400 |
try: |
| 401 |
outFileHandle = None |
| 402 |
if echoCmndForDebugging: print("Running: "+cmnd) |
| 403 |
rtnCode = runSysCmndInterface(cmnd, outFile=outFile, extraEnv=extraEnv, |
| 404 |
workingDir=workingDir) |
| 405 |
except OSError as e: |
| 406 |
rtnCode = 1 # Just some error code != 0 please! |
| 407 |
if rtnCode != 0 and throwExcept: |
| 408 |
raise RuntimeError("Error, the command '%s' failed with error code %d" |
| 409 |
% (cmnd, rtnCode)) |
| 410 |
return rtnCode |
| 411 |
|
| 412 |
|
| 413 |
def echoRunSysCmnd(cmnd, throwExcept=True, outFile=None, msg=None, |
| 414 |
timeCmnd=False, verbose=True, workingDir="", returnTimeCmnd=False, |
| 415 |
extraEnv=None |
| 416 |
): |
| 417 |
"""Echo command to be run and run command with runSysCmnd()""" |
| 418 |
if verbose: |
| 419 |
print("\nRunning: " + cmnd + "\n") |
| 420 |
if workingDir: |
| 421 |
print(" Running in working directory: " + workingDir + " ...\n") |
| 422 |
if extraEnv: |
| 423 |
print(" Appending environment:" + sorted_dict_str(extraEnv) + "\n") |
| 424 |
if outFile: |
| 425 |
print(" Writing console output to file " + outFile + " ...") |
| 426 |
if msg and verbose: |
| 427 |
print(" " + msg + "\n") |
| 428 |
t1 = time.time() |
| 429 |
totalTimeMin = -1.0 |
| 430 |
try: |
| 431 |
rtn = runSysCmnd(cmnd, throwExcept, outFile, workingDir, extraEnv) |
| 432 |
finally: |
| 433 |
if timeCmnd: |
| 434 |
t2 = time.time() |
| 435 |
totalTimeMin = (t2-t1)/60.0 |
| 436 |
if verbose: |
| 437 |
print("\n Runtime for command = %f minutes" % totalTimeMin) |
| 438 |
if returnTimeCmnd: |
| 439 |
return (rtn, totalTimeMin) |
| 440 |
return rtn |
| 441 |
|
| 442 |
|
| 443 |
# Return a shell command's output and optionally its return code |
| 444 |
# |
| 445 |
# For Python 2, this returns a standard ASCII byte array for the output. For |
| 446 |
# Python 3, the output is converted to standard unicode (i.e. 'utf-8') |
| 447 |
# |
| 448 |
def getCmndOutput(cmnd, stripTrailingSpaces=False, throwOnError=True, workingDir="", \ |
| 449 |
getStdErr=False, rtnCode=False \ |
| 450 |
): |
| 451 |
(data, errCode) = runSysCmndInterface(cmnd, rtnOutput=True, workingDir=workingDir, |
| 452 |
getStdErr=getStdErr) |
| 453 |
if errCode != 0: |
| 454 |
if throwOnError: |
| 455 |
raise RuntimeError('%s failed w/ exit code %d:\n\n%s' % (cmnd, errCode, data)) |
| 456 |
dataToReturn = data |
| 457 |
if stripTrailingSpaces: |
| 458 |
dataToReturn = data.rstrip() |
| 459 |
if rtnCode: |
| 460 |
return (dataToReturn, errCode) |
| 461 |
return dataToReturn |
| 462 |
|
| 463 |
|
| 464 |
def pidStillRunning(pid): |
| 465 |
#print("\npid = '" + pid + "'") |
| 466 |
cmnd = "kill -s 0 "+pid |
| 467 |
cmndReturn = runSysCmnd(cmnd, False) |
| 468 |
#print("\ncmndReturn = " + cmndReturn) |
| 469 |
return cmndReturn == 0 |
| 470 |
|
| 471 |
|
| 472 |
###################################### |
| 473 |
# File/path utilities |
| 474 |
###################################### |
| 475 |
|
| 476 |
|
| 477 |
def getFilePathArray(filePathStr): |
| 478 |
return filePathStr.split('/') |
| 479 |
|
| 480 |
|
| 481 |
def joinDirs(dirArray): |
| 482 |
""" |
| 483 |
Join directories. |
| 484 |
|
| 485 |
2009/06/09: rabartl: We should be able to just use os.path.join(...) but I |
| 486 |
found when used in at least on context that it resulted in not joining the |
| 487 |
elements but instead just returning the array. |
| 488 |
""" |
| 489 |
dirPath = "" |
| 490 |
for dir in dirArray: |
| 491 |
if not dirPath: |
| 492 |
dirPath = dir |
| 493 |
else: |
| 494 |
dirPath = dirPath + "/" + dir |
| 495 |
return dirPath |
| 496 |
|
| 497 |
|
| 498 |
def downDirsArray(numDownDirs): |
| 499 |
downDirsPathArray = [] |
| 500 |
for i in range(0, numDownDirs): |
| 501 |
downDirsPathArray.append("..") |
| 502 |
#print("\ndownDirsPathArray = " + downDirsPathArray) |
| 503 |
return downDirsPathArray |
| 504 |
|
| 505 |
|
| 506 |
def normalizePath(path): |
| 507 |
return os.path.normpath(path) |
| 508 |
|
| 509 |
|
| 510 |
def echoChDir(dirName, verbose=True): |
| 511 |
if verbose: |
| 512 |
print("\nChanging current directory to \'" + dirName + "\'") |
| 513 |
if not os.path.isdir(dirName): |
| 514 |
raise OSError("Error, the directory \'"+dirName+"\' does not exist in the" \ |
| 515 |
+ " base directory \'"+os.getcwd()+"\"!" ) |
| 516 |
os.chdir(dirName) |
| 517 |
if verbose: |
| 518 |
print("\nCurrent directory is \'" + os.getcwd() + "\'\n") |
| 519 |
|
| 520 |
|
| 521 |
def createDir(dirName, cdIntoDir=False, verbose=False): |
| 522 |
"""Create a directory if it does not exist""" |
| 523 |
if os.path.exists(dirName): |
| 524 |
if not os.path.isdir(dirName): |
| 525 |
errMsg = "\nError the path '" + dirName + \ |
| 526 |
"'already exists but it is not a directory!" |
| 527 |
if verbose: print(errMsg) |
| 528 |
raise RuntimeError(errMsg) |
| 529 |
if verbose: print("\nThe directory " + dirName + "already exists!") |
| 530 |
else: |
| 531 |
if verbose: print("\nCreating directory " + dirName + " ...") |
| 532 |
os.mkdir(dirName) |
| 533 |
if cdIntoDir: |
| 534 |
echoChDir(dirName, verbose=verbose) |
| 535 |
|
| 536 |
|
| 537 |
def createDirsFromPath(path): |
| 538 |
#print("\npath = " + path) |
| 539 |
pathList = path.split("/") |
| 540 |
#print("\npathList = " + pathList) |
| 541 |
if not pathList[0]: |
| 542 |
currDir = "/" |
| 543 |
for dir in pathList: |
| 544 |
currDir = os.path.join(currDir, dir) |
| 545 |
if currDir and not os.path.exists(currDir): |
| 546 |
#print("\ncurrDir = " + currDir) |
| 547 |
createDir(currDir) |
| 548 |
|
| 549 |
|
| 550 |
def expandDirsDict(trilinosDirsDict_inout): |
| 551 |
|
| 552 |
for dir in list(trilinosDirsDict_inout): |
| 553 |
subdirsList = dir.split("/") |
| 554 |
#print("\nsubdirsList = " + subdirsList) |
| 555 |
for i in range(len(subdirsList)): |
| 556 |
trilinosDirsDict_inout.update({joinDirs(subdirsList[:i+1]) : 0}) |
| 557 |
|
| 558 |
|
| 559 |
def removeIfExists(fileName): |
| 560 |
if os.path.exists(fileName): |
| 561 |
echoRunSysCmnd("rm "+fileName) |
| 562 |
|
| 563 |
|
| 564 |
def removeDirIfExists(dirName, verbose=False): |
| 565 |
if os.path.exists(dirName): |
| 566 |
if verbose: |
| 567 |
print("Removing existing directory '" + dirName + "' ...") |
| 568 |
echoRunSysCmnd("rm -rf "+dirName) |
| 569 |
|
| 570 |
|
| 571 |
def writeStrToFile(fileName, fileBodyStr): |
| 572 |
with open(fileName, 'w') as fileHandle: |
| 573 |
fileHandle.write(fileBodyStr) |
| 574 |
|
| 575 |
|
| 576 |
def readStrFromFile(fileName): |
| 577 |
with open(fileName, 'r') as fileHandle: |
| 578 |
return fileHandle.read() |
| 579 |
|
| 580 |
|
| 581 |
def getFileNamesWithFileTag( baseDir, fileTag ): |
| 582 |
"""Get a list of file names with a given date stamp""" |
| 583 |
return getCmndOutput( |
| 584 |
'cd %s && ls *%s*' % (baseDir, fileTag), |
| 585 |
throwOnError=False |
| 586 |
).split() |
| 587 |
|
| 588 |
|
| 589 |
def getFileNameFromGlob( baseDir, fileNameGlob ): |
| 590 |
return getCmndOutput("cd "+baseDir+" && ls "+fileNameGlob, True, False) |
| 591 |
|
| 592 |
|
| 593 |
def isEmptyDir( absDir ): |
| 594 |
return (len(os.listdir(absDir)) == 0) |
| 595 |
|
| 596 |
|
| 597 |
def getDirSizeInGb(dir): |
| 598 |
sizeIn1024Kb = int(getCmndOutput("du -s "+dir).split('\t')[0]) |
| 599 |
#print("\nsizeIn1024Kb = " + str(sizeIn1024Kb)) |
| 600 |
return float(sizeIn1024Kb)/1e+6 # Size in Gb! |
| 601 |
|
| 602 |
|
| 603 |
def isPathChar(char): |
| 604 |
return (char.isalnum() or char == '/') and (not char == ' ') |
| 605 |
|
| 606 |
|
| 607 |
# 2008/07/08: rabartl: This silly function is needed because on the sun |
| 608 |
# machine (i.e. sass8000), the 'which' command returns some non-printable |
| 609 |
# chars from the beginning of the output string that don't form a valid path. |
| 610 |
# This was *very* hard to debug but this function is able to select the valid |
| 611 |
# path string. This has been tested at least on linux and the sun and should |
| 612 |
# work anywhere. |
| 613 |
def cleanBadPath(inputPath): |
| 614 |
cleanPath = "" |
| 615 |
for i in range(len(inputPath)-1, -1, -1): |
| 616 |
char = inputPath[i] |
| 617 |
if not isPathChar(char): |
| 618 |
break |
| 619 |
cleanPath = char + cleanPath |
| 620 |
return cleanPath |
| 621 |
|
| 622 |
|
| 623 |
def getRelativePathFrom1to2(absPath1, absPath2): |
| 624 |
#print("\nabsPath1 =" + absPath1) |
| 625 |
#print("\nabsPath2 =" + absPath2) |
| 626 |
absPath1_array = absPath1.split('/') |
| 627 |
absPath2_array = absPath2.split('/') |
| 628 |
#print("\nabsPath1_array =" + absPath1_array) |
| 629 |
#print("\nabsPath2_array =" + absPath2_array) |
| 630 |
absPath1_array_len = len(absPath1_array) |
| 631 |
absPath2_array_len = len(absPath2_array) |
| 632 |
maxBaseDirDepth = min(absPath1_array_len, absPath2_array_len) |
| 633 |
baseDirDepth = 0 |
| 634 |
for dirIdx in range(0, maxBaseDirDepth): |
| 635 |
dir1 = absPath1_array[dirIdx] |
| 636 |
dir2 = absPath2_array[dirIdx] |
| 637 |
if dir1 != dir2: |
| 638 |
break |
| 639 |
baseDirDepth = baseDirDepth + 1 |
| 640 |
#print("\nbaseDirDepth = %d" % baseDirDepth) |
| 641 |
numDownDirs = absPath1_array_len - baseDirDepth |
| 642 |
#print("\nnumDownDirs = %d" % numDownDirs) |
| 643 |
if numDownDirs > 0: |
| 644 |
downDirPath = joinDirs(downDirsArray(numDownDirs)) |
| 645 |
else: |
| 646 |
downDirPath = "." |
| 647 |
#print("\ndownDirPath = '" + downDirPath + "'") |
| 648 |
if baseDirDepth == absPath2_array_len: |
| 649 |
upDirPath = "." |
| 650 |
else: |
| 651 |
upDirPath = joinDirs(absPath2_array[baseDirDepth:]) |
| 652 |
#print("\nupDirPath = " + upDirPath ) |
| 653 |
#print("\nupDirPath = '" + upDirPath + "'") |
| 654 |
relPath = os.path.join(downDirPath, upDirPath) |
| 655 |
if relPath == "./.": |
| 656 |
return "." |
| 657 |
return relPath |
| 658 |
|
| 659 |
|
| 660 |
def getExecBaseDir(execName): |
| 661 |
whichOutput = getCmndOutput("type -p "+execName, True, False) |
| 662 |
# Handle the outpue 'execName is execFullPath' output |
| 663 |
execFullPath = whichOutput.split(' ')[-1] |
| 664 |
#print("\nexecFullPath = " + execFullPath) |
| 665 |
execNameMatchRe = r"^(.+)/"+execName |
| 666 |
execNameGroups = re.findall(execNameMatchRe, execFullPath) |
| 667 |
#print("\nexecNameGroups = " + execNameGroups) |
| 668 |
if not execNameGroups: |
| 669 |
return None |
| 670 |
execBaseDir = execNameGroups[0] |
| 671 |
#print("\nexecBaseDir = \"" + execBaseDir + "\"") |
| 672 |
#execBaseDir = cleanBadPath(execBaseDir) |
| 673 |
#print("\nexecBaseDir = \"" + execBaseDir + "\"") |
| 674 |
return execBaseDir |
| 675 |
|
| 676 |
|
| 677 |
def extractAppendUniqueDirsDictFromFileNames(filenamesArray, dirsDict): |
| 678 |
for filename in filenamesArray: |
| 679 |
dirsDict.update( { normalizePath(os.path.dirname(filename)) : 0 } ) |
| 680 |
|
| 681 |
|
| 682 |
def copyFileAndReplaceTokens( scriptsDir, inputFile, tokenReplacementList, |
| 683 |
outputFile ): |
| 684 |
|
| 685 |
"""Copies an input stub file and then does a set of token replacements""" |
| 686 |
|
| 687 |
echoRunSysCmnd("cp "+inputFile+" "+outputFile, verbose=verboseDebug) |
| 688 |
|
| 689 |
for tokenReplacementPair in tokenReplacementList: |
| 690 |
oldToken = tokenReplacementPair[0] |
| 691 |
newToken = tokenReplacementPair[1] |
| 692 |
echoRunSysCmnd( scriptsDir+"/token-replace.pl "+oldToken+" "+newToken\ |
| 693 |
+" "+outputFile+" "+outputFile, verbose=verboseDebug ); |
| 694 |
# ToDo: Replace above with native re commands |
| 695 |
|
| 696 |
|
| 697 |
class TeeOutput(object): |
| 698 |
""" |
| 699 |
Object that directs all calls to its write method to stdout as well |
| 700 |
as a file. This is to be used as a simple replacement for the Unix |
| 701 |
tee command. |
| 702 |
""" |
| 703 |
def __init__(self, outputfile): |
| 704 |
""" Constructor takes a file-like object to write output to.""" |
| 705 |
self._realstdout = sys.stdout |
| 706 |
self._outputfile = outputfile |
| 707 |
|
| 708 |
def _safe_outputfile_method(self, methodname, *args): |
| 709 |
""" |
| 710 |
Calls the method specified by methodname with the given args on |
| 711 |
the internal file object if it is non-null. |
| 712 |
""" |
| 713 |
if self._outputfile is not None: |
| 714 |
if hasattr(self._outputfile, methodname): |
| 715 |
method = getattr(self._outputfile, methodname) |
| 716 |
if method and callable(method): |
| 717 |
method(*args) |
| 718 |
|
| 719 |
def write(self, data): |
| 720 |
""" |
| 721 |
Write the given data to stdout and to the log file. |
| 722 |
""" |
| 723 |
self._realstdout.write(data) |
| 724 |
self._safe_outputfile_method('write', data) |
| 725 |
|
| 726 |
def flush(self): |
| 727 |
""" |
| 728 |
Flush the internal file buffers. |
| 729 |
""" |
| 730 |
self._realstdout.flush() |
| 731 |
self._safe_outputfile_method('flush') |
| 732 |
|
| 733 |
|
| 734 |
###################################### |
| 735 |
# Shell argument helpers |
| 736 |
###################################### |
| 737 |
|
| 738 |
|
| 739 |
reCmndLineArg = re.compile(r"(--.+=)(.+)") |
| 740 |
|
| 741 |
|
| 742 |
def requoteCmndLineArgs(inArgs): |
| 743 |
argsStr = "" |
| 744 |
for arg in inArgs: |
| 745 |
splitArg = arg.split("=") |
| 746 |
newArg = None |
| 747 |
if len(splitArg) == 1: |
| 748 |
newArg = arg |
| 749 |
else: |
| 750 |
newArg = splitArg[0]+"=\""+'='.join(splitArg[1:])+"\"" |
| 751 |
#print("\nnewArg = " + newArg) |
| 752 |
argsStr = argsStr+" "+newArg |
| 753 |
return argsStr |
| 754 |
|
| 755 |
|
| 756 |
def commandLineOptionsToList(stringOptions): |
| 757 |
""" |
| 758 |
Convert a string of space separated command line options to a python |
| 759 |
list of the individual optionstrings. |
| 760 |
TODO: Handle shell quoting. |
| 761 |
""" |
| 762 |
return stringOptions.split() |
| 763 |
|
| 764 |
|
| 765 |
class ConfigurableOptionParser(optparse.OptionParser): |
| 766 |
""" |
| 767 |
OptionParser that accepts a python dictionary as a configuration |
| 768 |
file to provide default value overrides for the options. |
| 769 |
""" |
| 770 |
def __init__(self, configuration, **kwargs): |
| 771 |
""" |
| 772 |
Constructor accepts a configuration dictionary with default values |
| 773 |
for arguments and all of the OptionParser arguments as well. |
| 774 |
""" |
| 775 |
optparse.OptionParser.__init__(self, **kwargs) |
| 776 |
self._configuration = configuration |
| 777 |
|
| 778 |
def add_option(self, *args, **kwargs): |
| 779 |
""" |
| 780 |
Checks for a default override in the configuration dictionary and |
| 781 |
modifies the default and help arguments before dispatching them to |
| 782 |
the base class implementation. |
| 783 |
""" |
| 784 |
if 'default' in kwargs: |
| 785 |
for arg in args: |
| 786 |
kwargs['default'] = self._configuration.get(arg, kwargs['default']) |
| 787 |
optparse.OptionParser.add_option(self, *args, **kwargs) |
| 788 |
|
| 789 |
|
| 790 |
###################################### |
| 791 |
# Debugging support |
| 792 |
###################################### |
| 793 |
|
| 794 |
|
| 795 |
def printStackTrace(): |
| 796 |
sys.stdout.flush() |
| 797 |
traceback.print_exc() |
| 798 |
|
| 799 |
|
| 800 |
class ErrorCaptureOptionParser(optparse.OptionParser): |
| 801 |
__sawError = None |
| 802 |
def __init__(self, usage="%prog [options]", version=None): |
| 803 |
optparse.OptionParser.__init__(self, usage, version) |
| 804 |
__sawError = False |
| 805 |
def error(self, msg): |
| 806 |
raise Exception("Received error message: " + msg) |
| 807 |
|
| 808 |
|
| 809 |
###################################### |
| 810 |
# HTML directory browsing |
| 811 |
###################################### |
| 812 |
|
| 813 |
|
| 814 |
def createIndexHtmlBrowserList(baseDir, fileDirList = None): |
| 815 |
htmlList = "" |
| 816 |
# Get the fileDirList if empty |
| 817 |
if not fileDirList: |
| 818 |
fileDirList = os.listdir(baseDir) |
| 819 |
fileDirList.sort() |
| 820 |
# Create the HTML header |
| 821 |
htmlList += "" \ |
| 822 |
+ "<ul>\n" \ |
| 823 |
+ "<li><a href=\"..\">..</a></li>\n" |
| 824 |
# Fill in links to directories first |
| 825 |
for fd in fileDirList: |
| 826 |
absFd = baseDir+"/"+fd |
| 827 |
#print("isfile(" + fd + ") = " + str(os.path.isfile(absFd))) |
| 828 |
#print("isdir(" + fd + ") = " + str(os.path.isdir(absFd) )) |
| 829 |
if not os.path.isfile(absFd): |
| 830 |
htmlList = htmlList \ |
| 831 |
+"<li>dir: <a href=\""+fd+"\">"+fd+"</a></li>\n" |
| 832 |
# Fill in links to regular files second |
| 833 |
for fd in fileDirList: |
| 834 |
absFd = baseDir+"/"+fd |
| 835 |
if os.path.isfile(absFd): |
| 836 |
if fd != 'index.html': |
| 837 |
htmlList = htmlList \ |
| 838 |
+"<li>file: <a href=\""+fd+"\">"+fd+"</a></li>\n" |
| 839 |
# Write the footer |
| 840 |
htmlList = htmlList \ |
| 841 |
+ "</ul>\n" |
| 842 |
return htmlList |
| 843 |
|
| 844 |
|
| 845 |
|
| 846 |
def createIndexHtmlBrowserFile(baseDir, fileDirList): |
| 847 |
"""Creates an HTML browser file as a returned string.""" |
| 848 |
htmlFile = "" \ |
| 849 |
+ "<html>\n" \ |
| 850 |
+ "<head>\n" \ |
| 851 |
+ "<title>"+baseDir+"</title>\n" \ |
| 852 |
+ "</head>\n" \ |
| 853 |
+ "<body>\n" \ |
| 854 |
+ "<b>"+baseDir+"</b>\n" \ |
| 855 |
+ createIndexHtmlBrowserList(baseDir, fileDirList) \ |
| 856 |
+ "</body>\n" \ |
| 857 |
+ "</html>\n" |
| 858 |
return htmlFile |
| 859 |
|
| 860 |
|
| 861 |
def createHtmlBrowserFiles(absBaseDir, depth, verbose=False): |
| 862 |
|
| 863 |
"""Create a hierarchy of index.html files that will build a directory/file |
| 864 |
browser for a web server that will not allow directory/file browsing.""" |
| 865 |
|
| 866 |
#print("\nEntering createHtmlBrowserFiles(" + absBaseDir + ",%d" % (depth) + ")") |
| 867 |
|
| 868 |
# Get the list of all of the files/directories in absBaseDir |
| 869 |
fileDirList = os.listdir(absBaseDir) |
| 870 |
fileDirList.sort() |
| 871 |
#print("\nfileDirList = " + str(fileDirList) |
| 872 |
#sys.stdout.flush() |
| 873 |
|
| 874 |
# Get the index.html file HTML |
| 875 |
indexHtml = createIndexHtmlBrowserFile(absBaseDir, fileDirList) |
| 876 |
#print("\nindexHtml:\n" + indexHtml) |
| 877 |
|
| 878 |
# Write the index.html file |
| 879 |
indexFileName = absBaseDir+"/index.html" |
| 880 |
if verbose: |
| 881 |
print("\nWriting " + indexFileName) |
| 882 |
open(indexFileName,'w').write(indexHtml) |
| 883 |
|
| 884 |
# Loop through all of the directories and recursively call this function |
| 885 |
if depth > 0: |
| 886 |
for fd in fileDirList: |
| 887 |
absFd = absBaseDir+"/"+fd |
| 888 |
if os.path.isdir(absFd): |
| 889 |
subDir = absFd |
| 890 |
#print("\nCalling createHtmlBrowserFiles(" + subDir + ",%d" % (depth-1) |
| 891 |
# + ")") |
| 892 |
createHtmlBrowserFiles(absBaseDir+"/"+fd,depth-1) |
| 893 |
|
| 894 |
#print("\nLeaving createHtmlBrowserFiles(" + absBaseDir + ",%d" % (depth) + |
| 895 |
# ")") |