Debian Codegraph @

cmake/tribits/ci_support/CDashAnalyzeReportRandomFailures.py

raw
1
2 import os
3 import argparse
4 import datetime
5
6 from FindGeneralScriptSupport import *
7 import CDashQueryAnalyzeReport as CDQAR
8 import cdash_build_testing_date as CBTD
9
10
11 class CDashAnalyzeReportRandomFailuresDriver:
12
13 def __init__(self, versionInfoStrategy, extractBuildNameStrategy,
14 usageHelp=""):
15 self.versionInfoStrategy = versionInfoStrategy
16 self.extractBuildNameStrategy = extractBuildNameStrategy
17 self.args = None
18 self.usageHelp = usageHelp
19
20 def runDriver(self):
21
22 self.getCmndLineArgs()
23
24 cdashProjectTestingDayStartTime = self.args.cdash_testing_day_start_time
25 cdashSiteUrl = self.args.cdash_site_url
26 cdashProjectName = self.args.cdash_project_name
27 initialNonpassingTestFilters = self.args.initial_nonpassing_test_filters
28 date = self.args.reference_date
29 groupName = self.args.group_name
30 daysOfHistory = self.args.days_of_history
31 printUrlMode = self.args.print_url_mode
32 writeEmailToFile = self.args.write_email_to_file
33 sendEmailFrom = self.args.send_email_from
34 sendEmailTo = self.args.send_email_to
35 emailSubjectPrefix = self.args.email_subject_prefix
36
37 randomFailureSummaries = []
38
39 # A.1) Set up date range and directories
40
41 # Construct date range for queryTests filter string
42 referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD(
43 cdashProjectTestingDayStartTime, date)
44 dateRangeStart, dateRangeEnd = self.getDateRangeTuple(referenceDateDT, daysOfHistory)
45 dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd
46 dateRangeStr = dateRangeStart+" to "+dateRangeEnd
47
48 testQueriesCacheDir = os.getcwd()+"/test_queries_cache"
49 testHistoryCacheDir = testQueriesCacheDir+"/test_history_cache"
50 createDirsFromPath(testQueriesCacheDir)
51 createDirsFromPath(testHistoryCacheDir)
52
53 # Construct queryTest.php filter for a date range
54 initialNonpassingTestQueryFilters = \
55 dateUrlField+"&"+initialNonpassingTestFilters
56
57 # A.2) Create starting email body and html string aggregation var
58
59 cdashReportData = CDQAR.CDashReportData()
60
61 cdashReportData.htmlEmailBodyTop += \
62 "<h2>Random test failure scan results for "+cdashProjectName\
63 +" from "+dateRangeStr+"</h2>\n\n"
64
65
66 # B.1) Get all failing test result for past daysOfHistory
67
68 # Beginning of scanned details and links paragraph
69 cdashReportData.htmlEmailBodyTop +="<p>\n"
70
71 print("\nGetting list of initial nonpassing tests from CDash from "+dateRangeStr)
72
73 initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl(
74 cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
75 initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
76 cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
77
78 if printUrlMode == 'initial' or printUrlMode == 'all':
79 print("\nCDash nonpassing tests browser URL:\n\n"+\
80 " "+initialNonpassingTestBrowserUrl+"\n")
81 print("\nCDash nonpassing tests query URL:\n\n"+\
82 " "+initialNonpassingTestsQueryUrl+"\n")
83
84 initialNonpassingTestsQueryCacheFile = testQueriesCacheDir +\
85 "/initialCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json"
86
87 # List of dictionaries containing the cdash results in rows
88 initialNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten(
89 initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\
90 alwaysUseCacheFileIfExists=True)
91
92 cdashReportData.htmlEmailBodyTop += \
93 "<a href=\""+initialNonpassingTestBrowserUrl+"\">" +\
94 "Nonpassing tests scanned on CDash</a>=" +\
95 str(len(initialNonpassingTestsLOD))+"<br>\n"
96
97 # Ending of scanned details and links paragraph
98 # and start of scanning summaries and table
99 cdashReportData.htmlEmailBodyTop +="</p>\n\n<p>\n"
100
101 # B.2) Get each nonpassing test's testing history
102 for nonpassingTest in initialNonpassingTestsLOD:
103
104 # Remove unique jenkins run ID from build name
105 correctedBuildName = self.extractBuildNameStrategy.getCoreBuildName(nonpassingTest['buildName'])
106
107 testNameBuildName = nonpassingTest['testname']+"_"+nonpassingTest['buildName']
108 testNameBuildName = CDQAR.getCompressedFileNameIfTooLong(testNameBuildName)
109
110 print("\n Getting history from "+dateRangeStr+" for\n"+\
111 " Test name: "+nonpassingTest['testname']+"\n"+\
112 " Build name: "+correctedBuildName)
113
114 groupNameNormUrl, = CDQAR.normalizeUrlStrings(groupName)
115
116 testHistoryFilters = \
117 "filtercount=3&showfilters=1&filtercombine=and"+\
118 "&field1=testname&compare1=63&value1="+nonpassingTest['testname']+\
119 "&field2=groupname&compare2=63&value2="+groupNameNormUrl+\
120 "&field3=buildname&compare3=63&value3="+correctedBuildName
121
122 testHistoryQueryFilters = dateUrlField+"&"+testHistoryFilters
123
124 testHistoryCacheFile = testHistoryCacheDir+"/"+\
125 CDQAR.getCompressedFileNameIfTooLong(testNameBuildName+".json", ext=".json")
126
127 print("\n Creating file to write test history:\n "+testHistoryCacheFile)
128
129 testHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl(
130 cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters)
131 testHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
132 cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters)
133
134 testHistoryLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten(
135 testHistoryQueryUrl, testHistoryCacheFile, alwaysUseCacheFileIfExists=True)
136
137 print("\n Size of test history: "+str(len(testHistoryLOD)))
138
139 if printUrlMode == 'all':
140 print("\n CDash test history browser URL for "+nonpassingTest['testname']+" "+\
141 correctedBuildName+":\n\n"+" "+testHistoryBrowserUrl)
142 print("\n CDash test history query URL for "+nonpassingTest['testname']+" "+\
143 correctedBuildName+":\n\n"+" "+testHistoryQueryUrl)
144
145 if len(testHistoryLOD) < 2:
146 print("\n Size of test history too small for any comparisons, skipping ...\n")
147 continue
148
149 # B.3) Split full testing history to passed and nonpassed lists of dicts
150 passingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Passed"]
151 nonpassingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Failed"]
152 nonpassingSha1Pairs = set()
153
154 print("\n Num of passing tests in test history: "+str(len(passingTestHistoryLOD)))
155 print("\n Num of nonpassing tests in test history: "+str(len(nonpassingTestHistoryLOD)))
156
157 buildSummaryCacheDir = testQueriesCacheDir+"/build_summary_cache/"+testNameBuildName
158 createDirsFromPath(buildSummaryCacheDir)
159 # NOTE: There is an argument to be made that test histories should get their own directory
160 # instead of build summaries and that build summaries should live inside of there
161
162 # C.1) Get all nonpassing tests' sha1s into a set
163 for test in nonpassingTestHistoryLOD:
164
165 buildId = self.getBuildIdFromTest(test)
166
167 buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId
168 buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId)
169 buildConfigOutput = self.downloadBuildSummaryOffCDash(
170 buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all',
171 alwaysUseCacheFileIfExists=True)['configure']['output']
172
173 nonpassingSha1Pairs.add(
174 self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput))
175
176 print("\n Test history failing sha1s: "+str(nonpassingSha1Pairs))
177
178 # C.2) Check if passing tests' sha1s exist in nonpassing sha1s set
179 for test in passingTestHistoryLOD:
180
181 buildId = self.getBuildIdFromTest(test)
182
183 buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId
184 buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId)
185 buildConfigOutput = self.downloadBuildSummaryOffCDash(
186 buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all',
187 alwaysUseCacheFileIfExists=True)['configure']['output']
188
189 passingSha1Pair = \
190 self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput)
191
192 if self.versionInfoStrategy.checkTargetTopicRandomFailure(passingSha1Pair, nonpassingSha1Pairs):
193 print("\n Found passing sha1 pair, " + str(passingSha1Pair)+\
194 " in set of nonpassing sha1 pairs: \n"+str(nonpassingSha1Pairs))
195
196 randomFailureSummaries.append(
197 RandomFailureSummary(test['buildName'], test['testname'],
198 testHistoryBrowserUrl, passingSha1Pair))
199
200
201 print("\n*** CDash random failure analysis for " +\
202 cdashProjectName+" "+groupName+" from " +dateRangeStr)
203
204 print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n")
205
206 print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n")
207
208 cdashReportData.htmlEmailBodyTop += \
209 "Found random failing tests: "+str(len(randomFailureSummaries))+"<br>\n"
210
211 if len(randomFailureSummaries) > 0:
212 cdashReportData.globalPass = False
213
214 # Add number of random failing tests 'rft' found to summary data list
215 cdashReportData.summaryLineDataNumbersList.append(
216 "rft="+str(len(randomFailureSummaries)))
217
218 # Add number of initial failing tests 'ift' scanned to summary
219 # data list
220 cdashReportData.summaryLineDataNumbersList.append(
221 "ift="+str(len(initialNonpassingTestsLOD)))
222
223 for summary in randomFailureSummaries:
224 print(str(summary))
225 summary.singleSummaryReporter(cdashReportData)
226
227 summaryLine = CDQAR.getOverallCDashReportSummaryLine(
228 cdashReportData, cdashProjectName+" "+groupName, dateRangeStr)
229 print("\n"+summaryLine)
230
231 # Finish HTML body paragraph
232 cdashReportData.htmlEmailBodyTop += "\n</p>"
233
234 defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr()
235 subjectLine = emailSubjectPrefix+summaryLine
236
237 if writeEmailToFile:
238 print("\nWriting HTML to file: "+writeEmailToFile+" ...")
239 htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData,
240 pageTitle=subjectLine, pageStyle=defaultPageStyle)
241 # print(htmlStr)
242 with open(writeEmailToFile, 'w') as outFile:
243 outFile.write(htmlStr)
244
245 if sendEmailTo:
246 htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData,
247 pageStyle=defaultPageStyle)
248 for emailAddress in sendEmailTo.split(','):
249 emailAddress = emailAddress.strip()
250 print("\nSending email to '"+emailAddress+"' ...")
251 msg=CDQAR.createHtmlMimeEmail(
252 sendEmailFrom, emailAddress, subjectLine, "", htmlStr)
253 CDQAR.sendMineEmail(msg)
254
255 def getCmndLineArgs(self):
256 parser = argparse.ArgumentParser(description=self.usageHelp)
257
258 parser.add_argument("--cdash-site-url", default="", required=True,
259 help="Base CDash site (e.g. 'https://testing.sandia.gov/cdash')."+\
260 " [REQUIRED]")
261 parser.add_argument("--cdash-project-name", default="", required=True,
262 help="CDash project name (e.g. TriBITS)."+\
263 " [REQUIRED]")
264 parser.add_argument("--initial-nonpassing-test-filters", default="", required=True,
265 help="Partial URL fragment containing CDash filters for the initial set"+\
266 " of nonpassing tests (e.g. 'filtercount=2&showfilters=1&filtercombine=and&field1=status&compare1=63&"+\
267 "value1=Failed&field2=groupname&compare2=63&value2=Pull%%20Request')."+\
268 " [REQUIRED]")
269 parser.add_argument("--group-name", default="", required=True,
270 help="Name of the build group index for filtering a set of test (e.g. 'Pull Request')."+\
271 " [REQUIRED]")
272 parser.add_argument("--cdash-testing-day-start-time", default="00:00",
273 help="CDash project's testing day start time in UTC format '<hh>:<mm>'."+\
274 " [default='00:00']")
275 parser.add_argument("--reference-date", default="yesterday",
276 help="Starting reference date for querying failing tests as <YYYY-MM-DD> or"+\
277 " special values 'today' or 'yesterday.'"+\
278 " [default='yesterday']")
279 parser.add_argument("--days-of-history", default=1, type=int,
280 help="Number of days of testing history to query starting from the reference and going"+\
281 " backwards in time."+\
282 " [default=1]")
283 parser.add_argument("--print-url-mode", choices=['none','initial','all'], default='none',
284 help="Mode of url printing."+\
285 " [default=none]")
286 parser.add_argument("--write-email-to-file", default="",
287 help="Name of file to write built email HTML to."+\
288 " [default='']")
289 parser.add_argument("--send-email-to", default="",
290 help="Target email address to send script summary results to."+\
291 " [default='']")
292 parser.add_argument("--send-email-from", default="[email protected]",
293 help="Addressed sender of the script summary results email."+\
294 " [default='[email protected]']")
295 parser.add_argument("--email-subject-prefix", default="",
296 help="Prefix string added to the email subject line."+\
297 " [default='']")
298
299 self.args = parser.parse_args()
300
301 def getDateRangeTuple(self, referenceDateTime, dayTimeDelta):
302 beginDateTime = referenceDateTime - datetime.timedelta(days=(dayTimeDelta-1))
303 beginDateTimeStr = CBTD.getDateStrFromDateTime(beginDateTime)
304 endDateTimeStr = CBTD.getDateStrFromDateTime(referenceDateTime)
305 return (beginDateTimeStr, endDateTimeStr)
306
307 def getBuildIdFromTest(self, test):
308 return test['buildSummaryLink'].split("/")[-1]
309
310 def downloadBuildSummaryOffCDash(self,
311 cdashBuildSummaryQueryUrl, buildSummaryCacheFile=None,
312 useCachedCDashData=False, alwaysUseCacheFileIfExists=False,
313 verbose='False'
314 ):
315 verbose = verbose == 'all'
316 buildSummaryJson = CDQAR.getAndCacheCDashQueryDataOrReadFromCache(
317 cdashBuildSummaryQueryUrl, buildSummaryCacheFile, useCachedCDashData,
318 alwaysUseCacheFileIfExists, verbose)
319 return buildSummaryJson
320
321
322 class RandomFailureSummary(object):
323
324 def __init__(self, buildName, testName, testHistoryUrl, sha1Pair):
325 self.buildName = buildName
326 self.testName = testName
327 self.testHistoryUrl = testHistoryUrl
328 self.sha1Pair = sha1Pair
329
330 def __str__(self):
331 myStr = "Test name: "+self.testName +\
332 "\nBuild name: "+self.buildName +\
333 "\nIdentical sha1 pairs: "+str(self.sha1Pair) +\
334 "\nTest history browser URL: " +\
335 "\n "+self.testHistoryUrl+"\n"
336 return myStr
337
338 def singleSummaryReporter(self, cdashReportData):
339 cdashReportData.htmlEmailBodyTop += \
340 "\n<br>Build name: "+ self.buildName +\
341 "\n<br>Test name: "+ self.testName +\
342 "\n<br>Test history URL: <a href=\""+self.testHistoryUrl+"\">Link</a>" +\
343 "\n<br>Sha1 Pair : "+ str(self.sha1Pair)
344