import os import argparse import datetime from FindGeneralScriptSupport import * import CDashQueryAnalyzeReport as CDQAR import cdash_build_testing_date as CBTD class CDashAnalyzeReportRandomFailuresDriver: def __init__(self, versionInfoStrategy, extractBuildNameStrategy, usageHelp=""): self.versionInfoStrategy = versionInfoStrategy self.extractBuildNameStrategy = extractBuildNameStrategy self.args = None self.usageHelp = usageHelp def runDriver(self): self.getCmndLineArgs() cdashProjectTestingDayStartTime = self.args.cdash_testing_day_start_time cdashSiteUrl = self.args.cdash_site_url cdashProjectName = self.args.cdash_project_name initialNonpassingTestFilters = self.args.initial_nonpassing_test_filters date = self.args.reference_date groupName = self.args.group_name daysOfHistory = self.args.days_of_history printUrlMode = self.args.print_url_mode writeEmailToFile = self.args.write_email_to_file sendEmailFrom = self.args.send_email_from sendEmailTo = self.args.send_email_to emailSubjectPrefix = self.args.email_subject_prefix randomFailureSummaries = [] # A.1) Set up date range and directories # Construct date range for queryTests filter string referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD( cdashProjectTestingDayStartTime, date) dateRangeStart, dateRangeEnd = self.getDateRangeTuple(referenceDateDT, daysOfHistory) dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd dateRangeStr = dateRangeStart+" to "+dateRangeEnd testQueriesCacheDir = os.getcwd()+"/test_queries_cache" testHistoryCacheDir = testQueriesCacheDir+"/test_history_cache" createDirsFromPath(testQueriesCacheDir) createDirsFromPath(testHistoryCacheDir) # Construct queryTest.php filter for a date range initialNonpassingTestQueryFilters = \ dateUrlField+"&"+initialNonpassingTestFilters # A.2) Create starting email body and html string aggregation var cdashReportData = CDQAR.CDashReportData() cdashReportData.htmlEmailBodyTop += \ "
\n"
print("\nGetting list of initial nonpassing tests from CDash from "+dateRangeStr)
initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl(
cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
if printUrlMode == 'initial' or printUrlMode == 'all':
print("\nCDash nonpassing tests browser URL:\n\n"+\
" "+initialNonpassingTestBrowserUrl+"\n")
print("\nCDash nonpassing tests query URL:\n\n"+\
" "+initialNonpassingTestsQueryUrl+"\n")
initialNonpassingTestsQueryCacheFile = testQueriesCacheDir +\
"/initialCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json"
# List of dictionaries containing the cdash results in rows
initialNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten(
initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\
alwaysUseCacheFileIfExists=True)
cdashReportData.htmlEmailBodyTop += \
"" +\
"Nonpassing tests scanned on CDash=" +\
str(len(initialNonpassingTestsLOD))+"
\n"
# Ending of scanned details and links paragraph
# and start of scanning summaries and table
cdashReportData.htmlEmailBodyTop +="
\n"
# B.2) Get each nonpassing test's testing history
for nonpassingTest in initialNonpassingTestsLOD:
# Remove unique jenkins run ID from build name
correctedBuildName = self.extractBuildNameStrategy.getCoreBuildName(nonpassingTest['buildName'])
testNameBuildName = nonpassingTest['testname']+"_"+nonpassingTest['buildName']
testNameBuildName = CDQAR.getCompressedFileNameIfTooLong(testNameBuildName)
print("\n Getting history from "+dateRangeStr+" for\n"+\
" Test name: "+nonpassingTest['testname']+"\n"+\
" Build name: "+correctedBuildName)
groupNameNormUrl, = CDQAR.normalizeUrlStrings(groupName)
testHistoryFilters = \
"filtercount=3&showfilters=1&filtercombine=and"+\
"&field1=testname&compare1=63&value1="+nonpassingTest['testname']+\
"&field2=groupname&compare2=63&value2="+groupNameNormUrl+\
"&field3=buildname&compare3=63&value3="+correctedBuildName
testHistoryQueryFilters = dateUrlField+"&"+testHistoryFilters
testHistoryCacheFile = testHistoryCacheDir+"/"+\
CDQAR.getCompressedFileNameIfTooLong(testNameBuildName+".json", ext=".json")
print("\n Creating file to write test history:\n "+testHistoryCacheFile)
testHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl(
cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters)
testHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters)
testHistoryLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten(
testHistoryQueryUrl, testHistoryCacheFile, alwaysUseCacheFileIfExists=True)
print("\n Size of test history: "+str(len(testHistoryLOD)))
if printUrlMode == 'all':
print("\n CDash test history browser URL for "+nonpassingTest['testname']+" "+\
correctedBuildName+":\n\n"+" "+testHistoryBrowserUrl)
print("\n CDash test history query URL for "+nonpassingTest['testname']+" "+\
correctedBuildName+":\n\n"+" "+testHistoryQueryUrl)
if len(testHistoryLOD) < 2:
print("\n Size of test history too small for any comparisons, skipping ...\n")
continue
# B.3) Split full testing history to passed and nonpassed lists of dicts
passingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Passed"]
nonpassingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Failed"]
nonpassingSha1Pairs = set()
print("\n Num of passing tests in test history: "+str(len(passingTestHistoryLOD)))
print("\n Num of nonpassing tests in test history: "+str(len(nonpassingTestHistoryLOD)))
buildSummaryCacheDir = testQueriesCacheDir+"/build_summary_cache/"+testNameBuildName
createDirsFromPath(buildSummaryCacheDir)
# NOTE: There is an argument to be made that test histories should get their own directory
# instead of build summaries and that build summaries should live inside of there
# C.1) Get all nonpassing tests' sha1s into a set
for test in nonpassingTestHistoryLOD:
buildId = self.getBuildIdFromTest(test)
buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId
buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId)
buildConfigOutput = self.downloadBuildSummaryOffCDash(
buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all',
alwaysUseCacheFileIfExists=True)['configure']['output']
nonpassingSha1Pairs.add(
self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput))
print("\n Test history failing sha1s: "+str(nonpassingSha1Pairs))
# C.2) Check if passing tests' sha1s exist in nonpassing sha1s set
for test in passingTestHistoryLOD:
buildId = self.getBuildIdFromTest(test)
buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId
buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId)
buildConfigOutput = self.downloadBuildSummaryOffCDash(
buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all',
alwaysUseCacheFileIfExists=True)['configure']['output']
passingSha1Pair = \
self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput)
if self.versionInfoStrategy.checkTargetTopicRandomFailure(passingSha1Pair, nonpassingSha1Pairs):
print("\n Found passing sha1 pair, " + str(passingSha1Pair)+\
" in set of nonpassing sha1 pairs: \n"+str(nonpassingSha1Pairs))
randomFailureSummaries.append(
RandomFailureSummary(test['buildName'], test['testname'],
testHistoryBrowserUrl, passingSha1Pair))
print("\n*** CDash random failure analysis for " +\
cdashProjectName+" "+groupName+" from " +dateRangeStr)
print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n")
print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n")
cdashReportData.htmlEmailBodyTop += \
"Found random failing tests: "+str(len(randomFailureSummaries))+"
\n"
if len(randomFailureSummaries) > 0:
cdashReportData.globalPass = False
# Add number of random failing tests 'rft' found to summary data list
cdashReportData.summaryLineDataNumbersList.append(
"rft="+str(len(randomFailureSummaries)))
# Add number of initial failing tests 'ift' scanned to summary
# data list
cdashReportData.summaryLineDataNumbersList.append(
"ift="+str(len(initialNonpassingTestsLOD)))
for summary in randomFailureSummaries:
print(str(summary))
summary.singleSummaryReporter(cdashReportData)
summaryLine = CDQAR.getOverallCDashReportSummaryLine(
cdashReportData, cdashProjectName+" "+groupName, dateRangeStr)
print("\n"+summaryLine)
# Finish HTML body paragraph
cdashReportData.htmlEmailBodyTop += "\n