Debian Codegraph @

cmake/tribits/ci_support/cdash_analyze_and_report.py

raw
1 #!/usr/bin/env python3
2
3 # @HEADER
4 # *****************************************************************************
5 # TriBITS: Tribal Build, Integrate, and Test System
6 #
7 # Copyright 2013-2016 NTESS and the TriBITS contributors.
8 # SPDX-License-Identifier: BSD-3-Clause
9 # *****************************************************************************
10 # @HEADER
11
12 import sys
13 import pprint
14 import datetime
15
16 from FindGeneralScriptSupport import *
17 from GeneralScriptSupport import *
18 import CDashQueryAnalyzeReport as CDQAR
19 import cdash_build_testing_date as CBTD
20 from gitdist import addOptionParserChoiceOption
21
22 #
23 # Help message
24 #
25
26
27 usageHelp = r"""cdash_analyze_and_report.py [options]
28
29 This script takes in CDash URL information and other data as command-line
30 arguments and then analyzes it to look for missing expected builds, failed
31 tests, and various types of other failures and then reports the findings as an
32 HTML file written to disk and/or an HTML-formatted email sent to one or more
33 email addresses. (Other types of output can be produced as well in different
34 files.)
35
36 If all of the expected builds are found (and all of them have test results)
37 and there are no other failures found, then the script returns 0. Otherwise
38 the script returns non-zero. Therefore, this script can be used to drive
39 automated workflows by examining data on CDash.
40 """
41
42
43 #
44 # Helper functions
45 #
46
47
48 def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""):
49
50 clp.add_option(
51 "--date", dest="date", type="string", default='yesterday',
52 help="Date for the testing day <YYYY-MM-DD> or special values 'today'"+\
53 " or 'yesterday'. [default 'yesterday']" )
54
55 clp.add_option(
56 "--cdash-project-testing-day-start-time", dest="cdashProjectTestingDayStartTime",
57 type="string", default="00:00",
58 help="The CDash project testing day build star time in UTC in format '<hh>:<mm>'."+\
59 " [default = '00:00']" )
60
61 clp.add_option(
62 "--cdash-project-name", dest="cdashProjectName", type="string", default="",
63 help="CDash project name (e.g. 'Trilinos'). [REQUIRED]" )
64
65 clp.add_option(
66 "--build-set-name", dest="buildSetName", type="string", default="",
67 help="Name for the set of builds, (e.g. 'Trilinos Nightly Builds)."+\
68 " This used in the email summary line and in the HTML file body"+\
69 " to identify the set of builds and tests being examined."+\
70 " [REQUIRED]" )
71
72 clp.add_option(
73 "--cdash-site-url", dest="cdashSiteUrl", type="string", default="",
74 help="Base CDash site (e.g. 'https://testing.sandia.gov/cdash')."+\
75 " [REQUIRED]" )
76
77 clp.add_option(
78 "--cdash-builds-filters", dest="cdashBuildsFilters", type="string",
79 default="",
80 help="Partial URL fragment for index.php making of the filters for"+\
81 " the set of builds (e.g. 'filtercount=1&showfilters=1&field1=groupname&compare1=61&value1=ATDM')."+\
82 " [REQUIRED]" )
83
84 clp.add_option(
85 "--cdash-nonpassed-tests-filters", dest="cdashNonpassedTestsFilters", type="string",
86 default="",
87 help="Partial URL fragment for queryTests.php making of the filters for"+\
88 " the set of non-passing tests matching this set of builds (e.g."+\
89 " 'filtercombine=and&filtercount=1&showfilters=1&filtercombine=and&field1=groupname&compare1=61&value1=ATDM')."+\
90 " This set of filter fields may also filter out extra nonpassing tests"+\
91 " such for known random system failures to avoid flooding the output. In this"+\
92 " case, one should also set --require-test-history-match-nonpassing-tests=off."+\
93 " [REQUIRED]" )
94
95 clp.add_option(
96 "--expected-builds-file", dest="expectedBuildsFile", type="string",
97 default="",
98 help="Path to a CSV file that lists the expected builds. Each of these builds"+\
99 " must have unique 'site' and 'buildname' field pairs or an error will be"+\
100 " raised and the tool will abort. A list of files is also allowed that are"+\
101 " separated with ',' as <file1>,<file2>,... [default = '']" )
102
103 clp.add_option(
104 "--tests-with-issue-trackers-file", dest="testsWithIssueTrackersFile",
105 type="string", default="",
106 help="Path to CSV file that lists tests with issue trackers (and other data)."+\
107 " Each of these tests must have a unique 'site', 'buildName', and 'testname'"+\
108 " sets or an error will be raised and the tool will abort. [default = '']" )
109
110 addOptionParserChoiceOption(
111 "--filter-out-builds-and-tests-not-matching-expected-builds",
112 "filterOutBuildsAndTestsNotMatchingExpectedBuildsStr",
113 ("on", "off"), 1,
114 "Filter out build and test data not matching input list of expected builds."+\
115 " If set to 'on', this will filter out build and test data downloaded"+\
116 " from CDash that does not match the list of expected builds provided in"+\
117 " --expected-builds-file=<csv-file>. This will also filter out any tests"+\
118 " with issue trackers listed in"+\
119 " --tests-with-issue-trackers-file=<csv-file>.",
120 clp )
121
122 cdashQueriesCacheDir_default=os.getcwd()
123
124 clp.add_option(
125 "--cdash-queries-cache-dir", dest="cdashQueriesCacheDir", type="string",
126 default=cdashQueriesCacheDir_default,
127 help="Cache CDash query data this directory." \
128 +" [default = '"+cdashQueriesCacheDir_default+"']" )
129
130 clp.add_option(
131 "--cdash-base-cache-files-prefix", dest="cdashBaseCacheFilesPrefix", type="string",
132 default="",
133 help="Prefix given to the base-level cache files outside of the test_history/"+\
134 " directory. This is to allow multiple invocations of this script to share"+\
135 " the same base cache directory and share the test_history/ in case there are"+\
136 " overrlapping sets of tests where the test history cache could be reused."+\
137 " [default is derived from the --build-set-name=<build_set_name> argument where"+\
138 " spaces and punctuation in <build_set_name> are replaced with '_']" )
139
140 addOptionParserChoiceOption(
141 "--use-cached-cdash-data", "useCachedCDashDataStr",
142 ("on", "off"), 1,
143 "Use data downloaded from CDash already cached. Note that this only"+\
144 " impacts the reuse of base-level cache files and does not impact the usage"+\
145 " of test history cache in the <cacheDir>/test_history/ directory."+\
146 " If a test history file for a given testing day exists under the test_history/"+\
147 " directory it is used unconditionally.",
148 clp )
149
150 testHistoryDaysDefault= 30
151
152 clp.add_option(
153 "--limit-test-history-days", dest="testHistoryDays",
154 default=testHistoryDaysDefault, type="int",
155 help="Number of days to go back in history for each test."+\
156 " [default = '"+str(testHistoryDaysDefault)+"']" )
157
158 limitTableRows = 10
159
160 clp.add_option(
161 "--limit-table-rows", dest="limitTableRows", type="int",
162 default=limitTableRows,
163 help="Limit to the number of rows displayed in many of"+\
164 " the tables. This impacts tables like 'twoif' and 'twoinr'"+\
165 " that could have thousands of entries for some projects."+\
166 " This limits the number of tests for which detailed test history"+\
167 " is downloaded from CDash and is therefore important to ensure the"+\
168 " tool does not take too long to execute. However, this does NOT"+\
169 " limit the number of rows in many other tables that should be bounded like"+\
170 " any of the tables related to the builds or the list of tests with"+\
171 " issue trackers. (The number of those should never be extremely high.)"+\
172 " [default '"+str(limitTableRows)+"']" )
173
174 addOptionParserChoiceOption(
175 "--require-test-history-match-nonpassing-tests",
176 "requireTestHistoryMatchNonpassingTestsStr",
177 ("on", "off"), 0,
178 "Require that the status for each tracked test listed in the tests with issue"\
179 +" trackers CSV file match the status of that test returned from the test history"\
180 +" returned from CDash. In general, these should match up but these may not if extra"\
181 +" filter criteria has been added to the list on nonpassing tests in the"\
182 +" --cdash-nonpassed-tests-filters=<filters> set of filters (such as to filter out"\
183 +" a large number of random system failures). In this case, an error will be"\
184 +" returned by default and the script will crash. But this can be relaxed by"\
185 +" setting this to 'off' which will result in these tracked tests being listed in"\
186 +" the 'twim' table but typically with status 'Failed'.",
187 clp )
188
189 addOptionParserChoiceOption(
190 "--print-details", "printDetailsStr",
191 ("on", "off"), 1,
192 "Print more info like the CDash URLs for downloaded data and the cache"+\
193 " file names.",
194 clp )
195
196 addOptionParserChoiceOption(
197 "--list-unexpected-builds",
198 "listUnexpectedBuildsStr",
199 ("on", "off"), 1,
200 "List unexpected builds downloaded from CDash (i.e. not matching an expected build)'.",
201 clp )
202
203 clp.add_option(
204 "--write-unexpected-builds-to-file",
205 dest="writeUnexpectedBuildsToFile", type="string", default="",
206 help="Write a CSV file with a list of unexpected builds 'bu'." \
207 +" This is to make it easy to add new entries to the file read by" \
208 +" the option --expected-builds-file=<csv-file>. [default = '']" )
209
210 clp.add_option(
211 "--write-failing-tests-without-issue-trackers-to-file",
212 dest="writeFailingTestsWithoutIssueTrackersToFile", type="string", default="",
213 help="Write a CSV file with a list of tests with issue trackers failed 'twif'." \
214 +" This is to make it easy to add new entries to the file read by" \
215 +" the option --tests-with-issue-trackers-file=<csv-file>. [default = '']" )
216
217 clp.add_option(
218 "--write-test-data-to-file",
219 dest="writeTestDataToFile", type="string", default="",
220 help="Write pretty-printed Python list of dictionaries for tests" \
221 +" with issue trackers. This includes the history of the tests for" \
222 +" --limit-test-history-days=<days> of history. This contains all of the" \
223 +" information that appears in the generated summary tables for tests with" \
224 +" associated issue trackers. [default = '']" )
225
226 clp.add_option(
227 "--write-email-to-file", dest="writeEmailToFile", type="string", default="",
228 help="Write the body of the HTML email to this file. [default = '']" )
229
230 clp.add_option(
231 "--email-from-address=", dest="emailFromAddress", type="string", default="",
232 help="Address reported in the sent email. [default '']" )
233
234 clp.add_option(
235 "--send-email-to=", dest="sendEmailTo", type="string", default="",
236 help="Send email to 'address1, address2, ...'. [default '']" )
237
238 addOptionParserChoiceOption(
239 "--email-without-soft-hyphens",
240 "emailWithoutSoftHyphensStr",
241 ("on", "off"), 1,
242 "Remove soft hyphens from emails.",
243 clp )
244
245
246 def validateAndConvertCmndLineOptions(inOptions):
247
248 if inOptions.date == "":
249 print("Error, can't have empty --date, must pass in --date=YYYY-MM-DD"+\
250 " or special values --date=today or --date=yesterday!")
251 sys.exit(1)
252 else:
253 dateTimeObj = CDQAR.convertInputDateArgToYYYYMMDD(
254 inOptions.cdashProjectTestingDayStartTime,
255 inOptions.date)
256 inOptions.date = CBTD.getDateStrFromDateTime(dateTimeObj)
257
258 # ToDo: Assert more of the options to make sure they are correct!
259
260
261 def setExtraCmndLineOptionsAfterParse(inOptions_inout):
262
263 setattr(inOptions_inout, 'filterOutBuildsAndTestsNotMatchingExpectedBuilds',
264 inOptions_inout.filterOutBuildsAndTestsNotMatchingExpectedBuildsStr == "on")
265
266 setattr(inOptions_inout, 'useCachedCDashData',
267 inOptions_inout.useCachedCDashDataStr == "on")
268
269 setattr(inOptions_inout, 'requireTestHistoryMatchNonpassingTests',
270 inOptions_inout.requireTestHistoryMatchNonpassingTestsStr == "on")
271
272 setattr(inOptions_inout, 'printDetails',
273 inOptions_inout.printDetailsStr == "on")
274
275 setattr(inOptions_inout, 'listUnexpectedBuilds',
276 inOptions_inout.listUnexpectedBuildsStr == "on")
277
278 if inOptions_inout.cdashBaseCacheFilesPrefix == "":
279 inOptions_inout.cdashBaseCacheFilesPrefix = \
280 CDQAR.getFileNameStrFromText(inOptions_inout.buildSetName)+"_"
281
282 setattr(inOptions_inout, 'emailWithoutSoftHyphens',
283 inOptions_inout.emailWithoutSoftHyphensStr == "on")
284
285
286 def getCmndLineOptions():
287 from optparse import OptionParser
288 clp = OptionParser(usage=usageHelp)
289 injectCmndLineOptionsInParser(clp)
290 (options, args) = clp.parse_args()
291 validateAndConvertCmndLineOptions(options)
292 setExtraCmndLineOptionsAfterParse(options)
293 return options
294
295
296 def fwdCmndLineOptions(inOptions, lt=""):
297 io = inOptions
298 cmndLineOpts = \
299 " --date='"+io.date+"'"+lt+\
300 " --cdash-project-testing-day-start-time='"+io.cdashProjectTestingDayStartTime+"'"+lt+\
301 " --cdash-project-name='"+io.cdashProjectName+"'"+lt+\
302 " --build-set-name='"+io.buildSetName+"'"+lt+\
303 " --cdash-site-url='"+io.cdashSiteUrl+"'"+lt+\
304 " --cdash-builds-filters='"+io.cdashBuildsFilters+"'"+lt+\
305 " --cdash-nonpassed-tests-filters='"+io.cdashNonpassedTestsFilters+"'"+lt+\
306 " --expected-builds-file='"+io.expectedBuildsFile+"'"+lt+\
307 " --tests-with-issue-trackers-file='"+io.testsWithIssueTrackersFile+"'"+lt+\
308 " --filter-out-builds-and-tests-not-matching-expected-builds='"+\
309 io.filterOutBuildsAndTestsNotMatchingExpectedBuildsStr+"'"+lt+\
310 " --cdash-queries-cache-dir='"+io.cdashQueriesCacheDir+"'"+lt+\
311 " --cdash-base-cache-files-prefix='"+io.cdashBaseCacheFilesPrefix+"'"+lt+\
312 " --use-cached-cdash-data='"+io.useCachedCDashDataStr+"'"+lt+\
313 " --limit-test-history-days='"+str(io.testHistoryDays)+"'"+lt+\
314 " --limit-table-rows='"+str(io.limitTableRows)+"'"+lt+\
315 " --require-test-history-match-nonpassing-tests='"+io.requireTestHistoryMatchNonpassingTestsStr+"'"+lt+\
316 " --print-details='"+io.printDetailsStr+"'"+lt+\
317 " --list-unexpected-builds='"+io.listUnexpectedBuildsStr+"'"+lt+\
318 " --write-unexpected-builds-to-fileo='"+io.writeUnexpectedBuildsToFile+"'"+lt+\
319 " --write-failing-tests-without-issue-trackers-to-file='"+io.writeFailingTestsWithoutIssueTrackersToFile+"'"+lt+\
320 " --write-test-data-to-file='"+io.writeTestDataToFile+"'"+lt+\
321 " --write-email-to-file='"+io.writeEmailToFile+"'"+lt+\
322 " --email-from-address='"+io.emailFromAddress+"'"+lt+\
323 " --send-email-to='"+io.sendEmailTo+"'"+lt+\
324 " --email-without-soft-hyphens='"+io.emailWithoutSoftHyphensStr+"'"+lt
325 return cmndLineOpts
326
327
328 def echoCmndLineOptions(inOptions):
329 print(fwdCmndLineOptions(inOptions, " \\\n"))
330
331
332 def echoCmndLine(inOptions):
333
334 print("")
335 print("**************************************************************************")
336 print("cdash_analyze_and_report.py \\")
337
338 echoCmndLineOptions(inOptions)
339
340
341 # Strategy class that can get test history for a list of tests and set them in
342 # the test dicts taking input from the cdash_analyze_and_report.py commandline
343 # arguments.
344 #
345 class AddTestHistoryStrategy(object):
346
347
348 def __init__(self, inOptions, testHistoryCacheDir):
349 self.inOptions = inOptions
350 self.testHistoryCacheDir = testHistoryCacheDir
351
352
353 def getTestHistory(self, testLOD):
354
355 sio = self.inOptions
356
357 CDQAR.foreachTransform(
358 testLOD,
359 CDQAR.AddTestHistoryToTestDictFunctor(
360 cdashUrl=sio.cdashSiteUrl,
361 projectName=sio.cdashProjectName,
362 date=sio.date,
363 testingDayStartTimeUtc=sio.cdashProjectTestingDayStartTime,
364 daysOfHistory=sio.testHistoryDays,
365 testCacheDir=self.testHistoryCacheDir,
366 useCachedCDashData=sio.useCachedCDashData,
367 alwaysUseCacheFileIfExists=True,
368 verbose=True,
369 printDetails=sio.printDetails,
370 requireMatchTestTopTestHistory=sio.requireTestHistoryMatchNonpassingTests,
371 )
372 )
373
374
375 #
376 # Run the script
377 #
378
379 if __name__ == '__main__':
380
381 #
382 # Get commandline options
383 #
384
385 inOptions = getCmndLineOptions()
386 echoCmndLine(inOptions)
387
388 cacheDirAndBaseFilePrefix = \
389 inOptions.cdashQueriesCacheDir+"/"+inOptions.cdashBaseCacheFilesPrefix
390
391 #
392 # A) Define common data, etc
393 #
394
395 tcd = CDQAR.TableColumnData
396 pp = pprint.PrettyPrinter(indent=2)
397
398 groupSiteBuildNameSortOrder = ['group', 'site', 'buildname']
399
400 #
401 # B) Sound off
402 #
403
404 print("***")
405 print("*** Query and analyze CDash results for "+inOptions.buildSetName+\
406 " for testing day "+inOptions.date)
407 print("***")
408
409 #
410 # C) Create beginning of email body (that does not require getting any data off CDash)
411 #
412
413 # Aggregation of vars that get updated in this main() body and by functions
414 # called.
415 cdashReportData = CDQAR.CDashReportData()
416
417 cdashReportData.htmlEmailBodyTop += \
418 "<h2>Build and Test results for "+inOptions.buildSetName \
419 +" on "+inOptions.date+"</h2>\n\n"
420
421 #
422 # D) Read data files, get data off of CDash, do analysis, and construct HTML
423 # body parts
424 #
425
426 try:
427
428 # Beginning of top full build and tests CDash links paragraph
429 cdashReportData.htmlEmailBodyTop += "<p>\n"
430
431 #
432 # D.1) Read data from input files, set up cache directories
433 #
434 # Assert this data is correct and abort if there is an error before we run
435 # expensive CDash queries!
436 #
437
438 # Get list of expected builds from input CSV file
439 expectedBuildsLOD = CDQAR.getExpectedBuildsListOfDictsFromCsvFileArg(
440 inOptions.expectedBuildsFile)
441 print("\nNum expected builds = "+str(len(expectedBuildsLOD)))
442
443 # Create a SearchableListOfDict object to help look up expected builds
444 # given a build dict by key/value pairs 'group', 'site', and 'buildname'
445 # (requires unique builds with these key/value pairs)
446 expectedBuildsSLOD = CDQAR.createSearchableListOfBuilds(expectedBuildsLOD)
447
448 # Create a SearchableListOfDicts that will look up an expected build given
449 # just a test dict fields ['site', 'buildName']. (The list of tests with
450 # issue trackers does not have 'group' since cdash/queryTests.php does not
451 # give the 'group' associated with each test. Also, note that we need
452 # this special SearchableListOfDicts since the Build Name key name
453 # different for a cdash/queryTests.php test dict 'buildName' and a
454 # cdash/index.php build dict 'buildname'.)
455 testsToExpectedBuildsSLOD = \
456 CDQAR.createTestToBuildSearchableListOfDicts(expectedBuildsLOD)
457 # ToDo: Put in try/except to print about error in duplicate rows in the
458 # list of expected builds.
459
460 # Get list of tests with issue trackers from the input CSV file
461 if inOptions.testsWithIssueTrackersFile:
462 fullTestsWithIssueTrackersLOD = CDQAR.getTestsWtihIssueTrackersListFromCsvFile(
463 inOptions.testsWithIssueTrackersFile)
464 else:
465 fullTestsWithIssueTrackersLOD = []
466 print("\nNum tests with issue trackers read from CSV file = "+\
467 str(len(fullTestsWithIssueTrackersLOD)))
468
469 if inOptions.filterOutBuildsAndTestsNotMatchingExpectedBuilds:
470 (testsWithIssueTrackersLOD, testsWithIssueTrackersNotExpectedLOD) = \
471 CDQAR.splitTestsOnMatchExpectedBuilds(fullTestsWithIssueTrackersLOD,
472 testsToExpectedBuildsSLOD)
473 print("Num tests with issue trackers matching expected builds = "+\
474 str(len(testsWithIssueTrackersLOD)))
475 else:
476 testsWithIssueTrackersLOD = fullTestsWithIssueTrackersLOD
477 print("Num tests with issue trackers = "+\
478 str(len(testsWithIssueTrackersLOD)))
479
480 # Get a SearchableListOfDicts for the tests with issue trackers to allow
481 # them to be looked up based on matching ['site', 'buildName', 'testname']
482 # key/value pairs.
483 testsWithIssueTrackersSLOD = \
484 CDQAR.createSearchableListOfTests(testsWithIssueTrackersLOD)
485 # ToDo: Put in try/except to print about error in duplicate rows in the
486 # list of tests with issue trackers.
487
488 # Get a functor that will return True if a passed-in test dict matches a
489 # test with an issue tracker for the test key/value pairs ['site',
490 # 'buildName', and 'testname'].
491 testsWithIssueTrackerMatchFunctor = \
492 CDQAR.MatchDictKeysValuesFunctor(testsWithIssueTrackersSLOD)
493
494 # Assert that the list of tests with issue trackers matches the list of
495 # expected builds
496 (allTestsMatch, errMsg) = CDQAR.doTestsWithIssueTrackersMatchExpectedBuilds(
497 testsWithIssueTrackersLOD, testsToExpectedBuildsSLOD)
498 if not allTestsMatch:
499 raise Exception(errMsg)
500
501 # Test history cache dir
502 testHistoryCacheDir = inOptions.cdashQueriesCacheDir+"/test_history"
503 if not os.path.exists(testHistoryCacheDir):
504 print("\nCreating new test cache directory '"+testHistoryCacheDir+"'")
505 os.mkdir(testHistoryCacheDir)
506
507 #
508 # D.2) Get top-level lists of build and nonpassing tests off CDash
509 #
510
511 #
512 # D.2.a) Get list of dicts of builds off cdash/index.phpp
513 #
514
515 # @arghdos: note, we do not have to normalize the URLs from the input
516 # options because they are currently taken from the cdash site already
517 # (i.e., they are already in normalized form).
518
519 cdashIndexBuildsBrowserUrl = CDQAR.getCDashIndexBrowserUrl(
520 inOptions.cdashSiteUrl, inOptions.cdashProjectName, inOptions.date,
521 inOptions.cdashBuildsFilters)
522
523 print("\nCDash builds browser URL:\n\n "+cdashIndexBuildsBrowserUrl+"\n")
524
525 cdashIndexBuildsQueryUrl = CDQAR.getCDashIndexQueryUrl(
526 inOptions.cdashSiteUrl,
527 inOptions.cdashProjectName,
528 inOptions.date,
529 inOptions.cdashBuildsFilters )
530
531 fullCDashIndexBuildsJsonCacheFile = \
532 cacheDirAndBaseFilePrefix+"fullCDashIndexBuilds.json"
533
534 fullBuildsLOD = CDQAR.downloadBuildsOffCDashAndFlatten(
535 cdashIndexBuildsQueryUrl,
536 fullCDashIndexBuildsJsonCacheFile,
537 inOptions.useCachedCDashData )
538
539 print("\nNum builds downloaded from CDash = "+str(len(fullBuildsLOD)))
540
541 (buildsExpectedLOD, buildsUnexpectedLOD) = \
542 CDQAR.splitTestsOnMatchExpectedBuilds(fullBuildsLOD, expectedBuildsSLOD)
543
544 if inOptions.filterOutBuildsAndTestsNotMatchingExpectedBuilds:
545 print("Num builds matching expected builds = "+str(len(buildsExpectedLOD)))
546 buildsLOD = buildsExpectedLOD
547 else:
548 buildsLOD = fullBuildsLOD
549
550 if inOptions.listUnexpectedBuilds:
551 print("Num builds unexpected = "+str(len(buildsUnexpectedLOD)))
552
553 print("Num builds = "+str(len(buildsLOD)))
554
555 # HTML line "Builds on CDash"
556 cdashReportData.htmlEmailBodyTop += \
557 "<a href=\""+cdashIndexBuildsBrowserUrl+"\">"+\
558 "Builds on CDash</a> (num/expected="+\
559 str(len(buildsLOD))+"/"+str(len(expectedBuildsLOD))+")<br>\n"
560
561 # Create a SearchableListOfDict object to help look up builds given a
562 # build dict by key/value pairs 'group', 'site', and 'buildname' (requires
563 # unique builds with these key/value pairs)
564 buildsSLOD = CDQAR.createSearchableListOfBuilds(buildsLOD)
565 # ToDo: Add try/except to report duplicate builds in case this raises an
566 # exception.
567
568 #
569 # D.2.b) Get list of dicts of all nonpassing tests off
570 # cdash/queryTests.php
571 #
572
573 cdashNonpassingTestsBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
574 inOptions.cdashSiteUrl, inOptions.cdashProjectName, inOptions.date,
575 inOptions.cdashNonpassedTestsFilters)
576
577 print("\nGetting list of nonpassing tests from CDash ...\n")
578
579 print("\nCDash nonpassing tests browser URL:\n\n"+\
580 " "+cdashNonpassingTestsBrowserUrl+"\n")
581
582 cdashNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl(
583 inOptions.cdashSiteUrl, inOptions.cdashProjectName, inOptions.date,
584 inOptions.cdashNonpassedTestsFilters)
585
586 cdashNonpassingTestsQueryJsonCacheFile = \
587 cacheDirAndBaseFilePrefix+"fullCDashNonpassingTests.json"
588
589 fullNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten(
590 cdashNonpassingTestsQueryUrl, cdashNonpassingTestsQueryJsonCacheFile,
591 inOptions.useCachedCDashData )
592
593 print("\nNum nonpassing tests direct from CDash query = "+\
594 str(len(fullNonpassingTestsLOD)))
595
596 if inOptions.filterOutBuildsAndTestsNotMatchingExpectedBuilds:
597 (nonpassingTestsLOD, nonpassingTestsNotExpectedLOD) = \
598 CDQAR.splitTestsOnMatchExpectedBuilds(fullNonpassingTestsLOD,
599 testsToExpectedBuildsSLOD)
600 print("Num nonpassing tests matching expected builds = "+\
601 str(len(nonpassingTestsLOD)))
602 else:
603 nonpassingTestsLOD = fullNonpassingTestsLOD
604
605 print("Num nonpassing tests = "+\
606 str(len(nonpassingTestsLOD)))
607
608 # HTML line "Nonpassing Tests on CDash"
609 cdashReportData.htmlEmailBodyTop += \
610 "<a href=\""+cdashNonpassingTestsBrowserUrl+"\">"+\
611 "Non-passing Tests on CDash</a> (num="+str(len(nonpassingTestsLOD))+")<br>\n"
612
613 # End of full build and test link paragraph and start the next paragraph
614 # for the summary of failures and other tables
615 cdashReportData.htmlEmailBodyTop += \
616 "</p>\n\n"+\
617 "<p>\n"
618
619 # Create a SearchableListOfDicts object for looking up a nonpassing test
620 # given the test dict fields 'site', 'buildName', and 'testname'.
621 nonpassingTestsSLOD = CDQAR.createSearchableListOfTests(
622 nonpassingTestsLOD, removeExactDuplicateElements=True,
623 checkDictsAreSame_in=CDQAR.checkCDashTestDictsAreSame )
624 # NOTE: Above we add the option to remove exact duplicate tests since
625 # cdash/queryTests.php can return duplicate tests (i.e. all test dict
626 # fields are the same except and has the same buildid but could have
627 # different testids!)
628 # ToDo: Add try/except for above code in order to report duplicate tests
629 # where the buildid (and other fields) not match.
630
631 print("Num nonpassing tests after removing duplicate tests = "+\
632 str(len(nonpassingTestsLOD)))
633
634 # Create a functor to to see if a test dict matches one of the nonpassing
635 # tests downloaded from cdash/queryTests.php.
636 nonpassingTestsMatchFunctor = \
637 CDQAR.MatchDictKeysValuesFunctor(nonpassingTestsSLOD)
638
639 #
640 # D.3) Partition the various list of tests into different sets that will
641 # be displayed in different tables.
642 #
643
644 # Add issue tracker info for all nonpassing tests (including adding empty
645 # issue tracker fields for tests that don't have issue trackers)
646 CDQAR.foreachTransform( nonpassingTestsLOD,
647 CDQAR.AddIssueTrackerInfoToTestDictFunctor(testsWithIssueTrackersSLOD))
648
649 # Split the list of nonpassing tests into those with and without issue
650 # trackers
651 (nonpassingTestsWithIssueTrackersLOD,nonpassingTestsWithoutIssueTrackersLOD)=\
652 CDQAR.splitListOnMatch(nonpassingTestsLOD, testsWithIssueTrackerMatchFunctor)
653 print("Num nonpassing tests without issue trackers = "+\
654 str(len(nonpassingTestsWithoutIssueTrackersLOD)))
655 print("Num nonpassing tests with issue trackers = "+\
656 str(len(nonpassingTestsWithIssueTrackersLOD)))
657
658 # Split the list nonpassing tests without issue trackers into 'twoif' and
659 # 'twoinp'
660 (twoifLOD, twoinrLOD) = CDQAR.splitListOnMatch(
661 nonpassingTestsWithoutIssueTrackersLOD, CDQAR.isTestFailed)
662 print("Num nonpassing tests without issue trackers Failed = "+str(len(twoifLOD)))
663 print("Num nonpassing tests without issue trackers Not Run = "+str(len(twoinrLOD)))
664
665 # Split the list nonpassing tests with issue trackers into 'twif' and
666 # 'twinp'
667 (twifLOD, twinrLOD) = CDQAR.splitListOnMatch(
668 nonpassingTestsWithIssueTrackersLOD, CDQAR.isTestFailed)
669 print("Num nonpassing tests with issue trackers Failed = "+str(len(twifLOD)))
670 print("Num nonpassing tests with issue trackers Not Run = "+str(len(twinrLOD)))
671
672 # Get list of tests with issue trackers that are not in the list of
673 # nonpassing tests (and therefore these are passing or missing)
674 testsWithIssueTrackersGrossPassingOrMissingLOD = CDQAR.getFilteredList(
675 testsWithIssueTrackersSLOD,
676 CDQAR.NotMatchFunctor(nonpassingTestsMatchFunctor) )
677 print("Num tests with issue trackers gross passing or missing = "+\
678 str(len(testsWithIssueTrackersGrossPassingOrMissingLOD)))
679
680 #
681 # D.4) Process and tabulate lists of builds
682 #
683
684 buildsetReporter = CDQAR.SingleBuildsetReporter(cdashReportData)
685
686 #
687 # 'bm'
688 #
689
690 print("\nSearch for any missing expected builds ...\n")
691
692 missingExpectedBuildsLOD = CDQAR.getMissingExpectedBuildsList(
693 buildsSLOD, expectedBuildsLOD)
694
695 buildsetReporter.reportSingleBuildset("Builds Missing", "bm",
696 missingExpectedBuildsLOD,
697 buildsetGlobalPass=False,
698 buildsetColor=CDQAR.cdashColorFailed(),
699 buildsetColDataList=[
700 tcd("Group", 'group'),
701 tcd("Site", 'site'),
702 tcd("Build Name", 'buildname'),
703 tcd("Missing Status", 'status'),
704 ],
705 )
706
707 #
708 # 'cf'
709 #
710
711 print("\nSearch for any builds with configure failures ...\n")
712
713 buildsWithConfigureFailuresLOD = \
714 CDQAR.getFilteredList(buildsSLOD, CDQAR.buildHasConfigureFailures)
715
716 buildsetReporter.reportSingleBuildset("Builds with Configure Failures", "cf",
717 buildsWithConfigureFailuresLOD,
718 buildsetGlobalPass=False,
719 buildsetColor=CDQAR.cdashColorFailed(),
720 )
721
722 #
723 # 'bf'
724 #
725
726 print("\nSearch for any builds with compilation (build) failures ...\n")
727
728 buildsWithBuildFailuresLOD = \
729 CDQAR.getFilteredList(buildsSLOD, CDQAR.buildHasBuildFailures)
730
731 buildsetReporter.reportSingleBuildset("Builds with Build Failures", "bf",
732 buildsWithBuildFailuresLOD,
733 buildsetGlobalPass=False,
734 buildsetColor=CDQAR.cdashColorFailed(),
735 )
736
737 #
738 # 'bu'
739 #
740
741 if inOptions.listUnexpectedBuilds:
742 buildsetReporter.reportSingleBuildset("Builds Unexpected", "bu",
743 buildsUnexpectedLOD,
744 buildsetGlobalPass=True,
745 buildsetColor=None,
746 )
747
748 #
749 # D.5) Analyaize and report the different sets of tests
750 #
751
752 #
753 # D.5.a) Final processing of lists of tests and splitting into the
754 # different tests sets to report
755 #
756
757 # Object to make it easy to process the different test sets
758 addTestHistoryStrategy = AddTestHistoryStrategy(inOptions, testHistoryCacheDir)
759 testsetReporter = CDQAR.SingleTestsetReporter(cdashReportData,
760 addTestHistoryStrategy=addTestHistoryStrategy)
761
762 # Special functor to look up missing expected build given a test dict
763 testsToMissingExpectedBuildsSLOD = \
764 CDQAR.createTestToBuildSearchableListOfDicts(missingExpectedBuildsLOD)
765
766 # Functor for matching an missing expected build given a test dict
767 testMatchesMissingExpectedBuildsFunctor = CDQAR.MatchDictKeysValuesFunctor(
768 testsToMissingExpectedBuildsSLOD)
769
770 # Get list of tests with issue trackers that are not in the list of
771 # nonpassing tests and don't match expected builds (and therefore will be
772 # included in the sets 'twip' and 'twim').
773 ( testsWithIssueTrackersMatchingMissingExpectedBuildsLOD,
774 testsWithIssueTrackersPassingOrMissingLOD ) \
775 = \
776 CDQAR.splitListOnMatch( testsWithIssueTrackersGrossPassingOrMissingLOD,
777 testMatchesMissingExpectedBuildsFunctor )
778 print("\nNum tests with issue trackers passing or missing matching"+\
779 " posted builds = "+str(len(testsWithIssueTrackersPassingOrMissingLOD)))
780 print("\nTests with issue trackers missing that match"+\
781 " missing expected builds: num="+\
782 str(len(testsWithIssueTrackersMatchingMissingExpectedBuildsLOD)))
783 if len(testsWithIssueTrackersMatchingMissingExpectedBuildsLOD) > 0:
784 for testDict in testsWithIssueTrackersMatchingMissingExpectedBuildsLOD:
785 print(" "+sorted_dict_str(testDict))
786 print("\nNOTE: The above tests will NOT be listed in the set 'twim'!")
787
788 # Get test history for all of the tests with issue trackers that are not
789 # passing or missing. These will either be tests that are passing today
790 # (and therefore have history) or they will be tests that are missing.
791 # (But don't get test history or list out tests with issue trackers that
792 # match missing expected builds that did not submit any test data today.)
793
794 twipLOD = []
795 twimLOD = []
796
797 if testsWithIssueTrackersPassingOrMissingLOD:
798
799 print("\nGetting test history for tests with issue trackers"+\
800 " passing or missing: num="+str(len(testsWithIssueTrackersPassingOrMissingLOD)))
801
802 CDQAR.foreachTransform(
803 testsWithIssueTrackersPassingOrMissingLOD,
804 CDQAR.AddTestHistoryToTestDictFunctor(
805 inOptions.cdashSiteUrl,
806 inOptions.cdashProjectName,
807 inOptions.date,
808 inOptions.cdashProjectTestingDayStartTime,
809 inOptions.testHistoryDays,
810 testHistoryCacheDir,
811 useCachedCDashData=inOptions.useCachedCDashData,
812 alwaysUseCacheFileIfExists=True,
813 verbose=True,
814 printDetails=inOptions.printDetails,
815 requireMatchTestTopTestHistory=inOptions.requireTestHistoryMatchNonpassingTests,
816
817 )
818 )
819
820 # Split into lists for 'twip' and 'twim'
821 (twipLOD, twimLOD) = CDQAR.splitListOnMatch(
822 testsWithIssueTrackersPassingOrMissingLOD, CDQAR.isTestPassed )
823
824 print("\nNum tests with issue trackers Passed = "+str(len(twipLOD)))
825 print("Num tests with issue trackers Missing = "+str(len(twimLOD)))
826
827 #
828 # D.5.b) Report the different sets of tests
829 #
830 # NOTE: The order of these is chosen so those that require action of the
831 # person doing the triaging are sorted to the top.
832 #
833
834 testsetReporter.reportSingleTestset(
835 CDQAR.getStandardTestsetTypeInfo('twoif'),
836 len(twoifLOD), twoifLOD,
837 limitTableRows=inOptions.limitTableRows,
838 getTestHistory=True,
839 )
840
841 testsetReporter.reportSingleTestset(
842 CDQAR.getStandardTestsetTypeInfo('twoinr'),
843 len(twoinrLOD), twoinrLOD,
844 limitTableRows=inOptions.limitTableRows,
845 getTestHistory=True,
846 )
847
848 testsetReporter.reportSingleTestset(
849 CDQAR.getStandardTestsetTypeInfo('twip'),
850 len(twipLOD), twipLOD,
851 limitTableRows=None,
852 getTestHistory=False, # Already got it above!
853 )
854
855 testsetReporter.reportSingleTestset(
856 CDQAR.getStandardTestsetTypeInfo('twim', ""),
857 len(twimLOD), twimLOD,
858 limitTableRows=None,
859 getTestHistory=False, # Already got it above!
860 )
861
862 testsetReporter.reportSingleTestset(
863 CDQAR.getStandardTestsetTypeInfo('twif', ""),
864 len(twifLOD), twifLOD,
865 limitTableRows=None,
866 getTestHistory=True,
867 )
868
869 testsetReporter.reportSingleTestset(
870 CDQAR.getStandardTestsetTypeInfo('twinr', ""),
871 len(twinrLOD), twinrLOD,
872 limitTableRows=None,
873 getTestHistory=True,
874 )
875
876 #
877 # D.6) Write out list of unexpected builds to CSV file
878 #
879
880 if inOptions.writeUnexpectedBuildsToFile:
881 unexpectedBuildsCsvFileName = inOptions.writeUnexpectedBuildsToFile
882 print("\nWriting list of unexpected builds to file "\
883 +unexpectedBuildsCsvFileName+" ...")
884 CDQAR.writeExpectedBuildsListOfDictsToCsvFile(buildsUnexpectedLOD,
885 unexpectedBuildsCsvFileName)
886
887 #
888 # D.7) Write out list twiof to CSV file
889 #
890
891 if inOptions.writeFailingTestsWithoutIssueTrackersToFile:
892 twoifCsvFileName = inOptions.writeFailingTestsWithoutIssueTrackersToFile
893 print("\nWriting list of 'twiof' to file "+twoifCsvFileName+" ...")
894 CDQAR.writeTestsListOfDictsToCsvFile(twoifLOD, twoifCsvFileName)
895
896 #
897 # D.8) Write out test data to CSV file
898 #
899
900 if inOptions.writeTestDataToFile:
901 testDataFileName = inOptions.writeTestDataToFile
902 print("\nWriting out gathered test data to file "+testDataFileName+" ...")
903 testDataLOD = twipLOD + twimLOD + twifLOD + twinrLOD
904 CDQAR.foreachTransform(testDataLOD,
905 CDQAR.AddCDashTestingDayFunctor(inOptions.date))
906 # ToDo: Add the first inOptions.limitTableRows elements of twiofLOD and twoinrLOD?
907 CDQAR.pprintPythonDataToFile(testDataLOD, testDataFileName)
908
909 except Exception:
910 # Traceback!
911 print("")
912 sys.stdout.flush()
913 traceback.print_exc()
914 # Report the error
915 cdashReportData.htmlEmailBodyBottom += "\n<pre><code>\n"+\
916 traceback.format_exc()+"\n</code></pre>\n"
917 print("\nError, could not compute the analysis due to"+\
918 " above error so return failed!")
919 cdashReportData.globalPass = False
920 cdashReportData.summaryLineDataNumbersList.append("SCRIPT CRASHED")
921
922 #
923 # E) Put together final email summary line
924 #
925
926 summaryLine = CDQAR.getOverallCDashReportSummaryLine(cdashReportData,
927 inOptions.buildSetName, inOptions.date)
928
929 #
930 # F) Finish off HTML body guts and define overall HTML body style
931 #
932
933 # Finish off the top paragraph of the summary lines
934 cdashReportData.htmlEmailBodyTop += \
935 "</p>\n"
936
937 #
938 # G) Write HTML body file and/or send HTML email(s)
939 #
940
941 defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr()
942
943 if inOptions.writeEmailToFile:
944 print("\nWriting HTML file '"+inOptions.writeEmailToFile+"' ...")
945 fullCDashHtmlReportPageStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData,
946 pageTitle=summaryLine, pageStyle=defaultPageStyle)
947 with open(inOptions.writeEmailToFile, 'w') as outFile:
948 outFile.write(fullCDashHtmlReportPageStr)
949
950 if inOptions.sendEmailTo:
951 htmlEmailBodyStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData,
952 pageStyle=defaultPageStyle)
953 for emailAddress in inOptions.sendEmailTo.split(','):
954 emailAddress = emailAddress.strip()
955 print("\nSending email to '"+emailAddress+"' ...")
956 msg=CDQAR.createHtmlMimeEmail(
957 inOptions.emailFromAddress, emailAddress, summaryLine, "",
958 htmlEmailBodyStr, inOptions.emailWithoutSoftHyphens)
959 CDQAR.sendMineEmail(msg)
960
961 #
962 # H) Return final global pass/fail
963 #
964
965 print("\n"+summaryLine+"\n")
966
967 if cdashReportData.globalPass:
968 sys.exit(0)
969 else:
970 sys.exit(1)