| 1 |
|
| 2 |
import os |
| 3 |
import argparse |
| 4 |
import datetime |
| 5 |
|
| 6 |
from FindGeneralScriptSupport import * |
| 7 |
import CDashQueryAnalyzeReport as CDQAR |
| 8 |
import cdash_build_testing_date as CBTD |
| 9 |
|
| 10 |
|
| 11 |
class CDashAnalyzeReportRandomFailuresDriver: |
| 12 |
|
| 13 |
def __init__(self, versionInfoStrategy, extractBuildNameStrategy, |
| 14 |
usageHelp=""): |
| 15 |
self.versionInfoStrategy = versionInfoStrategy |
| 16 |
self.extractBuildNameStrategy = extractBuildNameStrategy |
| 17 |
self.args = None |
| 18 |
self.usageHelp = usageHelp |
| 19 |
|
| 20 |
def runDriver(self): |
| 21 |
|
| 22 |
self.getCmndLineArgs() |
| 23 |
|
| 24 |
cdashProjectTestingDayStartTime = self.args.cdash_testing_day_start_time |
| 25 |
cdashSiteUrl = self.args.cdash_site_url |
| 26 |
cdashProjectName = self.args.cdash_project_name |
| 27 |
initialNonpassingTestFilters = self.args.initial_nonpassing_test_filters |
| 28 |
date = self.args.reference_date |
| 29 |
groupName = self.args.group_name |
| 30 |
daysOfHistory = self.args.days_of_history |
| 31 |
printUrlMode = self.args.print_url_mode |
| 32 |
writeEmailToFile = self.args.write_email_to_file |
| 33 |
sendEmailFrom = self.args.send_email_from |
| 34 |
sendEmailTo = self.args.send_email_to |
| 35 |
emailSubjectPrefix = self.args.email_subject_prefix |
| 36 |
|
| 37 |
randomFailureSummaries = [] |
| 38 |
|
| 39 |
# A.1) Set up date range and directories |
| 40 |
|
| 41 |
# Construct date range for queryTests filter string |
| 42 |
referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD( |
| 43 |
cdashProjectTestingDayStartTime, date) |
| 44 |
dateRangeStart, dateRangeEnd = self.getDateRangeTuple(referenceDateDT, daysOfHistory) |
| 45 |
dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd |
| 46 |
dateRangeStr = dateRangeStart+" to "+dateRangeEnd |
| 47 |
|
| 48 |
testQueriesCacheDir = os.getcwd()+"/test_queries_cache" |
| 49 |
testHistoryCacheDir = testQueriesCacheDir+"/test_history_cache" |
| 50 |
createDirsFromPath(testQueriesCacheDir) |
| 51 |
createDirsFromPath(testHistoryCacheDir) |
| 52 |
|
| 53 |
# Construct queryTest.php filter for a date range |
| 54 |
initialNonpassingTestQueryFilters = \ |
| 55 |
dateUrlField+"&"+initialNonpassingTestFilters |
| 56 |
|
| 57 |
# A.2) Create starting email body and html string aggregation var |
| 58 |
|
| 59 |
cdashReportData = CDQAR.CDashReportData() |
| 60 |
|
| 61 |
cdashReportData.htmlEmailBodyTop += \ |
| 62 |
"<h2>Random test failure scan results for "+cdashProjectName\ |
| 63 |
+" from "+dateRangeStr+"</h2>\n\n" |
| 64 |
|
| 65 |
|
| 66 |
# B.1) Get all failing test result for past daysOfHistory |
| 67 |
|
| 68 |
# Beginning of scanned details and links paragraph |
| 69 |
cdashReportData.htmlEmailBodyTop +="<p>\n" |
| 70 |
|
| 71 |
print("\nGetting list of initial nonpassing tests from CDash from "+dateRangeStr) |
| 72 |
|
| 73 |
initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( |
| 74 |
cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) |
| 75 |
initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( |
| 76 |
cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) |
| 77 |
|
| 78 |
if printUrlMode == 'initial' or printUrlMode == 'all': |
| 79 |
print("\nCDash nonpassing tests browser URL:\n\n"+\ |
| 80 |
" "+initialNonpassingTestBrowserUrl+"\n") |
| 81 |
print("\nCDash nonpassing tests query URL:\n\n"+\ |
| 82 |
" "+initialNonpassingTestsQueryUrl+"\n") |
| 83 |
|
| 84 |
initialNonpassingTestsQueryCacheFile = testQueriesCacheDir +\ |
| 85 |
"/initialCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json" |
| 86 |
|
| 87 |
# List of dictionaries containing the cdash results in rows |
| 88 |
initialNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( |
| 89 |
initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\ |
| 90 |
alwaysUseCacheFileIfExists=True) |
| 91 |
|
| 92 |
cdashReportData.htmlEmailBodyTop += \ |
| 93 |
"<a href=\""+initialNonpassingTestBrowserUrl+"\">" +\ |
| 94 |
"Nonpassing tests scanned on CDash</a>=" +\ |
| 95 |
str(len(initialNonpassingTestsLOD))+"<br>\n" |
| 96 |
|
| 97 |
# Ending of scanned details and links paragraph |
| 98 |
# and start of scanning summaries and table |
| 99 |
cdashReportData.htmlEmailBodyTop +="</p>\n\n<p>\n" |
| 100 |
|
| 101 |
# B.2) Get each nonpassing test's testing history |
| 102 |
for nonpassingTest in initialNonpassingTestsLOD: |
| 103 |
|
| 104 |
# Remove unique jenkins run ID from build name |
| 105 |
correctedBuildName = self.extractBuildNameStrategy.getCoreBuildName(nonpassingTest['buildName']) |
| 106 |
|
| 107 |
testNameBuildName = nonpassingTest['testname']+"_"+nonpassingTest['buildName'] |
| 108 |
testNameBuildName = CDQAR.getCompressedFileNameIfTooLong(testNameBuildName) |
| 109 |
|
| 110 |
print("\n Getting history from "+dateRangeStr+" for\n"+\ |
| 111 |
" Test name: "+nonpassingTest['testname']+"\n"+\ |
| 112 |
" Build name: "+correctedBuildName) |
| 113 |
|
| 114 |
groupNameNormUrl, = CDQAR.normalizeUrlStrings(groupName) |
| 115 |
|
| 116 |
testHistoryFilters = \ |
| 117 |
"filtercount=3&showfilters=1&filtercombine=and"+\ |
| 118 |
"&field1=testname&compare1=63&value1="+nonpassingTest['testname']+\ |
| 119 |
"&field2=groupname&compare2=63&value2="+groupNameNormUrl+\ |
| 120 |
"&field3=buildname&compare3=63&value3="+correctedBuildName |
| 121 |
|
| 122 |
testHistoryQueryFilters = dateUrlField+"&"+testHistoryFilters |
| 123 |
|
| 124 |
testHistoryCacheFile = testHistoryCacheDir+"/"+\ |
| 125 |
CDQAR.getCompressedFileNameIfTooLong(testNameBuildName+".json", ext=".json") |
| 126 |
|
| 127 |
print("\n Creating file to write test history:\n "+testHistoryCacheFile) |
| 128 |
|
| 129 |
testHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( |
| 130 |
cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) |
| 131 |
testHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( |
| 132 |
cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) |
| 133 |
|
| 134 |
testHistoryLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( |
| 135 |
testHistoryQueryUrl, testHistoryCacheFile, alwaysUseCacheFileIfExists=True) |
| 136 |
|
| 137 |
print("\n Size of test history: "+str(len(testHistoryLOD))) |
| 138 |
|
| 139 |
if printUrlMode == 'all': |
| 140 |
print("\n CDash test history browser URL for "+nonpassingTest['testname']+" "+\ |
| 141 |
correctedBuildName+":\n\n"+" "+testHistoryBrowserUrl) |
| 142 |
print("\n CDash test history query URL for "+nonpassingTest['testname']+" "+\ |
| 143 |
correctedBuildName+":\n\n"+" "+testHistoryQueryUrl) |
| 144 |
|
| 145 |
if len(testHistoryLOD) < 2: |
| 146 |
print("\n Size of test history too small for any comparisons, skipping ...\n") |
| 147 |
continue |
| 148 |
|
| 149 |
# B.3) Split full testing history to passed and nonpassed lists of dicts |
| 150 |
passingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Passed"] |
| 151 |
nonpassingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Failed"] |
| 152 |
nonpassingSha1Pairs = set() |
| 153 |
|
| 154 |
print("\n Num of passing tests in test history: "+str(len(passingTestHistoryLOD))) |
| 155 |
print("\n Num of nonpassing tests in test history: "+str(len(nonpassingTestHistoryLOD))) |
| 156 |
|
| 157 |
buildSummaryCacheDir = testQueriesCacheDir+"/build_summary_cache/"+testNameBuildName |
| 158 |
createDirsFromPath(buildSummaryCacheDir) |
| 159 |
# NOTE: There is an argument to be made that test histories should get their own directory |
| 160 |
# instead of build summaries and that build summaries should live inside of there |
| 161 |
|
| 162 |
# C.1) Get all nonpassing tests' sha1s into a set |
| 163 |
for test in nonpassingTestHistoryLOD: |
| 164 |
|
| 165 |
buildId = self.getBuildIdFromTest(test) |
| 166 |
|
| 167 |
buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId |
| 168 |
buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId) |
| 169 |
buildConfigOutput = self.downloadBuildSummaryOffCDash( |
| 170 |
buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all', |
| 171 |
alwaysUseCacheFileIfExists=True)['configure']['output'] |
| 172 |
|
| 173 |
nonpassingSha1Pairs.add( |
| 174 |
self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput)) |
| 175 |
|
| 176 |
print("\n Test history failing sha1s: "+str(nonpassingSha1Pairs)) |
| 177 |
|
| 178 |
# C.2) Check if passing tests' sha1s exist in nonpassing sha1s set |
| 179 |
for test in passingTestHistoryLOD: |
| 180 |
|
| 181 |
buildId = self.getBuildIdFromTest(test) |
| 182 |
|
| 183 |
buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId |
| 184 |
buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId) |
| 185 |
buildConfigOutput = self.downloadBuildSummaryOffCDash( |
| 186 |
buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all', |
| 187 |
alwaysUseCacheFileIfExists=True)['configure']['output'] |
| 188 |
|
| 189 |
passingSha1Pair = \ |
| 190 |
self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput) |
| 191 |
|
| 192 |
if self.versionInfoStrategy.checkTargetTopicRandomFailure(passingSha1Pair, nonpassingSha1Pairs): |
| 193 |
print("\n Found passing sha1 pair, " + str(passingSha1Pair)+\ |
| 194 |
" in set of nonpassing sha1 pairs: \n"+str(nonpassingSha1Pairs)) |
| 195 |
|
| 196 |
randomFailureSummaries.append( |
| 197 |
RandomFailureSummary(test['buildName'], test['testname'], |
| 198 |
testHistoryBrowserUrl, passingSha1Pair)) |
| 199 |
|
| 200 |
|
| 201 |
print("\n*** CDash random failure analysis for " +\ |
| 202 |
cdashProjectName+" "+groupName+" from " +dateRangeStr) |
| 203 |
|
| 204 |
print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n") |
| 205 |
|
| 206 |
print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n") |
| 207 |
|
| 208 |
cdashReportData.htmlEmailBodyTop += \ |
| 209 |
"Found random failing tests: "+str(len(randomFailureSummaries))+"<br>\n" |
| 210 |
|
| 211 |
if len(randomFailureSummaries) > 0: |
| 212 |
cdashReportData.globalPass = False |
| 213 |
|
| 214 |
# Add number of random failing tests 'rft' found to summary data list |
| 215 |
cdashReportData.summaryLineDataNumbersList.append( |
| 216 |
"rft="+str(len(randomFailureSummaries))) |
| 217 |
|
| 218 |
# Add number of initial failing tests 'ift' scanned to summary |
| 219 |
# data list |
| 220 |
cdashReportData.summaryLineDataNumbersList.append( |
| 221 |
"ift="+str(len(initialNonpassingTestsLOD))) |
| 222 |
|
| 223 |
for summary in randomFailureSummaries: |
| 224 |
print(str(summary)) |
| 225 |
summary.singleSummaryReporter(cdashReportData) |
| 226 |
|
| 227 |
summaryLine = CDQAR.getOverallCDashReportSummaryLine( |
| 228 |
cdashReportData, cdashProjectName+" "+groupName, dateRangeStr) |
| 229 |
print("\n"+summaryLine) |
| 230 |
|
| 231 |
# Finish HTML body paragraph |
| 232 |
cdashReportData.htmlEmailBodyTop += "\n</p>" |
| 233 |
|
| 234 |
defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr() |
| 235 |
subjectLine = emailSubjectPrefix+summaryLine |
| 236 |
|
| 237 |
if writeEmailToFile: |
| 238 |
print("\nWriting HTML to file: "+writeEmailToFile+" ...") |
| 239 |
htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, |
| 240 |
pageTitle=subjectLine, pageStyle=defaultPageStyle) |
| 241 |
# print(htmlStr) |
| 242 |
with open(writeEmailToFile, 'w') as outFile: |
| 243 |
outFile.write(htmlStr) |
| 244 |
|
| 245 |
if sendEmailTo: |
| 246 |
htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, |
| 247 |
pageStyle=defaultPageStyle) |
| 248 |
for emailAddress in sendEmailTo.split(','): |
| 249 |
emailAddress = emailAddress.strip() |
| 250 |
print("\nSending email to '"+emailAddress+"' ...") |
| 251 |
msg=CDQAR.createHtmlMimeEmail( |
| 252 |
sendEmailFrom, emailAddress, subjectLine, "", htmlStr) |
| 253 |
CDQAR.sendMineEmail(msg) |
| 254 |
|
| 255 |
def getCmndLineArgs(self): |
| 256 |
parser = argparse.ArgumentParser(description=self.usageHelp) |
| 257 |
|
| 258 |
parser.add_argument("--cdash-site-url", default="", required=True, |
| 259 |
help="Base CDash site (e.g. 'https://testing.sandia.gov/cdash')."+\ |
| 260 |
" [REQUIRED]") |
| 261 |
parser.add_argument("--cdash-project-name", default="", required=True, |
| 262 |
help="CDash project name (e.g. TriBITS)."+\ |
| 263 |
" [REQUIRED]") |
| 264 |
parser.add_argument("--initial-nonpassing-test-filters", default="", required=True, |
| 265 |
help="Partial URL fragment containing CDash filters for the initial set"+\ |
| 266 |
" of nonpassing tests (e.g. 'filtercount=2&showfilters=1&filtercombine=and&field1=status&compare1=63&"+\ |
| 267 |
"value1=Failed&field2=groupname&compare2=63&value2=Pull%%20Request')."+\ |
| 268 |
" [REQUIRED]") |
| 269 |
parser.add_argument("--group-name", default="", required=True, |
| 270 |
help="Name of the build group index for filtering a set of test (e.g. 'Pull Request')."+\ |
| 271 |
" [REQUIRED]") |
| 272 |
parser.add_argument("--cdash-testing-day-start-time", default="00:00", |
| 273 |
help="CDash project's testing day start time in UTC format '<hh>:<mm>'."+\ |
| 274 |
" [default='00:00']") |
| 275 |
parser.add_argument("--reference-date", default="yesterday", |
| 276 |
help="Starting reference date for querying failing tests as <YYYY-MM-DD> or"+\ |
| 277 |
" special values 'today' or 'yesterday.'"+\ |
| 278 |
" [default='yesterday']") |
| 279 |
parser.add_argument("--days-of-history", default=1, type=int, |
| 280 |
help="Number of days of testing history to query starting from the reference and going"+\ |
| 281 |
" backwards in time."+\ |
| 282 |
" [default=1]") |
| 283 |
parser.add_argument("--print-url-mode", choices=['none','initial','all'], default='none', |
| 284 |
help="Mode of url printing."+\ |
| 285 |
" [default=none]") |
| 286 |
parser.add_argument("--write-email-to-file", default="", |
| 287 |
help="Name of file to write built email HTML to."+\ |
| 288 |
" [default='']") |
| 289 |
parser.add_argument("--send-email-to", default="", |
| 290 |
help="Target email address to send script summary results to."+\ |
| 291 |
" [default='']") |
| 292 |
parser.add_argument("--send-email-from", default="[email protected]", |
| 293 |
help="Addressed sender of the script summary results email."+\ |
| 294 |
" [default='[email protected]']") |
| 295 |
parser.add_argument("--email-subject-prefix", default="", |
| 296 |
help="Prefix string added to the email subject line."+\ |
| 297 |
" [default='']") |
| 298 |
|
| 299 |
self.args = parser.parse_args() |
| 300 |
|
| 301 |
def getDateRangeTuple(self, referenceDateTime, dayTimeDelta): |
| 302 |
beginDateTime = referenceDateTime - datetime.timedelta(days=(dayTimeDelta-1)) |
| 303 |
beginDateTimeStr = CBTD.getDateStrFromDateTime(beginDateTime) |
| 304 |
endDateTimeStr = CBTD.getDateStrFromDateTime(referenceDateTime) |
| 305 |
return (beginDateTimeStr, endDateTimeStr) |
| 306 |
|
| 307 |
def getBuildIdFromTest(self, test): |
| 308 |
return test['buildSummaryLink'].split("/")[-1] |
| 309 |
|
| 310 |
def downloadBuildSummaryOffCDash(self, |
| 311 |
cdashBuildSummaryQueryUrl, buildSummaryCacheFile=None, |
| 312 |
useCachedCDashData=False, alwaysUseCacheFileIfExists=False, |
| 313 |
verbose='False' |
| 314 |
): |
| 315 |
verbose = verbose == 'all' |
| 316 |
buildSummaryJson = CDQAR.getAndCacheCDashQueryDataOrReadFromCache( |
| 317 |
cdashBuildSummaryQueryUrl, buildSummaryCacheFile, useCachedCDashData, |
| 318 |
alwaysUseCacheFileIfExists, verbose) |
| 319 |
return buildSummaryJson |
| 320 |
|
| 321 |
|
| 322 |
class RandomFailureSummary(object): |
| 323 |
|
| 324 |
def __init__(self, buildName, testName, testHistoryUrl, sha1Pair): |
| 325 |
self.buildName = buildName |
| 326 |
self.testName = testName |
| 327 |
self.testHistoryUrl = testHistoryUrl |
| 328 |
self.sha1Pair = sha1Pair |
| 329 |
|
| 330 |
def __str__(self): |
| 331 |
myStr = "Test name: "+self.testName +\ |
| 332 |
"\nBuild name: "+self.buildName +\ |
| 333 |
"\nIdentical sha1 pairs: "+str(self.sha1Pair) +\ |
| 334 |
"\nTest history browser URL: " +\ |
| 335 |
"\n "+self.testHistoryUrl+"\n" |
| 336 |
return myStr |
| 337 |
|
| 338 |
def singleSummaryReporter(self, cdashReportData): |
| 339 |
cdashReportData.htmlEmailBodyTop += \ |
| 340 |
"\n<br>Build name: "+ self.buildName +\ |
| 341 |
"\n<br>Test name: "+ self.testName +\ |
| 342 |
"\n<br>Test history URL: <a href=\""+self.testHistoryUrl+"\">Link</a>" +\ |
| 343 |
"\n<br>Sha1 Pair : "+ str(self.sha1Pair) |
| 344 |
|