| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
# @HEADER |
| 4 |
# ***************************************************************************** |
| 5 |
# TriBITS: Tribal Build, Integrate, and Test System |
| 6 |
# |
| 7 |
# Copyright 2013-2016 NTESS and the TriBITS contributors. |
| 8 |
# SPDX-License-Identifier: BSD-3-Clause |
| 9 |
# ***************************************************************************** |
| 10 |
# @HEADER |
| 11 |
|
| 12 |
import sys |
| 13 |
import pprint |
| 14 |
import datetime |
| 15 |
|
| 16 |
from FindGeneralScriptSupport import * |
| 17 |
from GeneralScriptSupport import * |
| 18 |
import CDashQueryAnalyzeReport as CDQAR |
| 19 |
import cdash_build_testing_date as CBTD |
| 20 |
from gitdist import addOptionParserChoiceOption |
| 21 |
|
| 22 |
# |
| 23 |
# Help message |
| 24 |
# |
| 25 |
|
| 26 |
|
| 27 |
usageHelp = r"""cdash_analyze_and_report.py [options] |
| 28 |
|
| 29 |
This script takes in CDash URL information and other data as command-line |
| 30 |
arguments and then analyzes it to look for missing expected builds, failed |
| 31 |
tests, and various types of other failures and then reports the findings as an |
| 32 |
HTML file written to disk and/or an HTML-formatted email sent to one or more |
| 33 |
email addresses. (Other types of output can be produced as well in different |
| 34 |
files.) |
| 35 |
|
| 36 |
If all of the expected builds are found (and all of them have test results) |
| 37 |
and there are no other failures found, then the script returns 0. Otherwise |
| 38 |
the script returns non-zero. Therefore, this script can be used to drive |
| 39 |
automated workflows by examining data on CDash. |
| 40 |
""" |
| 41 |
|
| 42 |
|
| 43 |
# |
| 44 |
# Helper functions |
| 45 |
# |
| 46 |
|
| 47 |
|
| 48 |
def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""): |
| 49 |
|
| 50 |
clp.add_option( |
| 51 |
"--date", dest="date", type="string", default='yesterday', |
| 52 |
help="Date for the testing day <YYYY-MM-DD> or special values 'today'"+\ |
| 53 |
" or 'yesterday'. [default 'yesterday']" ) |
| 54 |
|
| 55 |
clp.add_option( |
| 56 |
"--cdash-project-testing-day-start-time", dest="cdashProjectTestingDayStartTime", |
| 57 |
type="string", default="00:00", |
| 58 |
help="The CDash project testing day build star time in UTC in format '<hh>:<mm>'."+\ |
| 59 |
" [default = '00:00']" ) |
| 60 |
|
| 61 |
clp.add_option( |
| 62 |
"--cdash-project-name", dest="cdashProjectName", type="string", default="", |
| 63 |
help="CDash project name (e.g. 'Trilinos'). [REQUIRED]" ) |
| 64 |
|
| 65 |
clp.add_option( |
| 66 |
"--build-set-name", dest="buildSetName", type="string", default="", |
| 67 |
help="Name for the set of builds, (e.g. 'Trilinos Nightly Builds)."+\ |
| 68 |
" This used in the email summary line and in the HTML file body"+\ |
| 69 |
" to identify the set of builds and tests being examined."+\ |
| 70 |
" [REQUIRED]" ) |
| 71 |
|
| 72 |
clp.add_option( |
| 73 |
"--cdash-site-url", dest="cdashSiteUrl", type="string", default="", |
| 74 |
help="Base CDash site (e.g. 'https://testing.sandia.gov/cdash')."+\ |
| 75 |
" [REQUIRED]" ) |
| 76 |
|
| 77 |
clp.add_option( |
| 78 |
"--cdash-builds-filters", dest="cdashBuildsFilters", type="string", |
| 79 |
default="", |
| 80 |
help="Partial URL fragment for index.php making of the filters for"+\ |
| 81 |
" the set of builds (e.g. 'filtercount=1&showfilters=1&field1=groupname&compare1=61&value1=ATDM')."+\ |
| 82 |
" [REQUIRED]" ) |
| 83 |
|
| 84 |
clp.add_option( |
| 85 |
"--cdash-nonpassed-tests-filters", dest="cdashNonpassedTestsFilters", type="string", |
| 86 |
default="", |
| 87 |
help="Partial URL fragment for queryTests.php making of the filters for"+\ |
| 88 |
" the set of non-passing tests matching this set of builds (e.g."+\ |
| 89 |
" 'filtercombine=and&filtercount=1&showfilters=1&filtercombine=and&field1=groupname&compare1=61&value1=ATDM')."+\ |
| 90 |
" This set of filter fields may also filter out extra nonpassing tests"+\ |
| 91 |
" such for known random system failures to avoid flooding the output. In this"+\ |
| 92 |
" case, one should also set --require-test-history-match-nonpassing-tests=off."+\ |
| 93 |
" [REQUIRED]" ) |
| 94 |
|
| 95 |
clp.add_option( |
| 96 |
"--expected-builds-file", dest="expectedBuildsFile", type="string", |
| 97 |
default="", |
| 98 |
help="Path to a CSV file that lists the expected builds. Each of these builds"+\ |
| 99 |
" must have unique 'site' and 'buildname' field pairs or an error will be"+\ |
| 100 |
" raised and the tool will abort. A list of files is also allowed that are"+\ |
| 101 |
" separated with ',' as <file1>,<file2>,... [default = '']" ) |
| 102 |
|
| 103 |
clp.add_option( |
| 104 |
"--tests-with-issue-trackers-file", dest="testsWithIssueTrackersFile", |
| 105 |
type="string", default="", |
| 106 |
help="Path to CSV file that lists tests with issue trackers (and other data)."+\ |
| 107 |
" Each of these tests must have a unique 'site', 'buildName', and 'testname'"+\ |
| 108 |
" sets or an error will be raised and the tool will abort. [default = '']" ) |
| 109 |
|
| 110 |
addOptionParserChoiceOption( |
| 111 |
"--filter-out-builds-and-tests-not-matching-expected-builds", |
| 112 |
"filterOutBuildsAndTestsNotMatchingExpectedBuildsStr", |
| 113 |
("on", "off"), 1, |
| 114 |
"Filter out build and test data not matching input list of expected builds."+\ |
| 115 |
" If set to 'on', this will filter out build and test data downloaded"+\ |
| 116 |
" from CDash that does not match the list of expected builds provided in"+\ |
| 117 |
" --expected-builds-file=<csv-file>. This will also filter out any tests"+\ |
| 118 |
" with issue trackers listed in"+\ |
| 119 |
" --tests-with-issue-trackers-file=<csv-file>.", |
| 120 |
clp ) |
| 121 |
|
| 122 |
cdashQueriesCacheDir_default=os.getcwd() |
| 123 |
|
| 124 |
clp.add_option( |
| 125 |
"--cdash-queries-cache-dir", dest="cdashQueriesCacheDir", type="string", |
| 126 |
default=cdashQueriesCacheDir_default, |
| 127 |
help="Cache CDash query data this directory." \ |
| 128 |
+" [default = '"+cdashQueriesCacheDir_default+"']" ) |
| 129 |
|
| 130 |
clp.add_option( |
| 131 |
"--cdash-base-cache-files-prefix", dest="cdashBaseCacheFilesPrefix", type="string", |
| 132 |
default="", |
| 133 |
help="Prefix given to the base-level cache files outside of the test_history/"+\ |
| 134 |
" directory. This is to allow multiple invocations of this script to share"+\ |
| 135 |
" the same base cache directory and share the test_history/ in case there are"+\ |
| 136 |
" overrlapping sets of tests where the test history cache could be reused."+\ |
| 137 |
" [default is derived from the --build-set-name=<build_set_name> argument where"+\ |
| 138 |
" spaces and punctuation in <build_set_name> are replaced with '_']" ) |
| 139 |
|
| 140 |
addOptionParserChoiceOption( |
| 141 |
"--use-cached-cdash-data", "useCachedCDashDataStr", |
| 142 |
("on", "off"), 1, |
| 143 |
"Use data downloaded from CDash already cached. Note that this only"+\ |
| 144 |
" impacts the reuse of base-level cache files and does not impact the usage"+\ |
| 145 |
" of test history cache in the <cacheDir>/test_history/ directory."+\ |
| 146 |
" If a test history file for a given testing day exists under the test_history/"+\ |
| 147 |
" directory it is used unconditionally.", |
| 148 |
clp ) |
| 149 |
|
| 150 |
testHistoryDaysDefault= 30 |
| 151 |
|
| 152 |
clp.add_option( |
| 153 |
"--limit-test-history-days", dest="testHistoryDays", |
| 154 |
default=testHistoryDaysDefault, type="int", |
| 155 |
help="Number of days to go back in history for each test."+\ |
| 156 |
" [default = '"+str(testHistoryDaysDefault)+"']" ) |
| 157 |
|
| 158 |
limitTableRows = 10 |
| 159 |
|
| 160 |
clp.add_option( |
| 161 |
"--limit-table-rows", dest="limitTableRows", type="int", |
| 162 |
default=limitTableRows, |
| 163 |
help="Limit to the number of rows displayed in many of"+\ |
| 164 |
" the tables. This impacts tables like 'twoif' and 'twoinr'"+\ |
| 165 |
" that could have thousands of entries for some projects."+\ |
| 166 |
" This limits the number of tests for which detailed test history"+\ |
| 167 |
" is downloaded from CDash and is therefore important to ensure the"+\ |
| 168 |
" tool does not take too long to execute. However, this does NOT"+\ |
| 169 |
" limit the number of rows in many other tables that should be bounded like"+\ |
| 170 |
" any of the tables related to the builds or the list of tests with"+\ |
| 171 |
" issue trackers. (The number of those should never be extremely high.)"+\ |
| 172 |
" [default '"+str(limitTableRows)+"']" ) |
| 173 |
|
| 174 |
addOptionParserChoiceOption( |
| 175 |
"--require-test-history-match-nonpassing-tests", |
| 176 |
"requireTestHistoryMatchNonpassingTestsStr", |
| 177 |
("on", "off"), 0, |
| 178 |
"Require that the status for each tracked test listed in the tests with issue"\ |
| 179 |
+" trackers CSV file match the status of that test returned from the test history"\ |
| 180 |
+" returned from CDash. In general, these should match up but these may not if extra"\ |
| 181 |
+" filter criteria has been added to the list on nonpassing tests in the"\ |
| 182 |
+" --cdash-nonpassed-tests-filters=<filters> set of filters (such as to filter out"\ |
| 183 |
+" a large number of random system failures). In this case, an error will be"\ |
| 184 |
+" returned by default and the script will crash. But this can be relaxed by"\ |
| 185 |
+" setting this to 'off' which will result in these tracked tests being listed in"\ |
| 186 |
+" the 'twim' table but typically with status 'Failed'.", |
| 187 |
clp ) |
| 188 |
|
| 189 |
addOptionParserChoiceOption( |
| 190 |
"--print-details", "printDetailsStr", |
| 191 |
("on", "off"), 1, |
| 192 |
"Print more info like the CDash URLs for downloaded data and the cache"+\ |
| 193 |
" file names.", |
| 194 |
clp ) |
| 195 |
|
| 196 |
addOptionParserChoiceOption( |
| 197 |
"--list-unexpected-builds", |
| 198 |
"listUnexpectedBuildsStr", |
| 199 |
("on", "off"), 1, |
| 200 |
"List unexpected builds downloaded from CDash (i.e. not matching an expected build)'.", |
| 201 |
clp ) |
| 202 |
|
| 203 |
clp.add_option( |
| 204 |
"--write-unexpected-builds-to-file", |
| 205 |
dest="writeUnexpectedBuildsToFile", type="string", default="", |
| 206 |
help="Write a CSV file with a list of unexpected builds 'bu'." \ |
| 207 |
+" This is to make it easy to add new entries to the file read by" \ |
| 208 |
+" the option --expected-builds-file=<csv-file>. [default = '']" ) |
| 209 |
|
| 210 |
clp.add_option( |
| 211 |
"--write-failing-tests-without-issue-trackers-to-file", |
| 212 |
dest="writeFailingTestsWithoutIssueTrackersToFile", type="string", default="", |
| 213 |
help="Write a CSV file with a list of tests with issue trackers failed 'twif'." \ |
| 214 |
+" This is to make it easy to add new entries to the file read by" \ |
| 215 |
+" the option --tests-with-issue-trackers-file=<csv-file>. [default = '']" ) |
| 216 |
|
| 217 |
clp.add_option( |
| 218 |
"--write-test-data-to-file", |
| 219 |
dest="writeTestDataToFile", type="string", default="", |
| 220 |
help="Write pretty-printed Python list of dictionaries for tests" \ |
| 221 |
+" with issue trackers. This includes the history of the tests for" \ |
| 222 |
+" --limit-test-history-days=<days> of history. This contains all of the" \ |
| 223 |
+" information that appears in the generated summary tables for tests with" \ |
| 224 |
+" associated issue trackers. [default = '']" ) |
| 225 |
|
| 226 |
clp.add_option( |
| 227 |
"--write-email-to-file", dest="writeEmailToFile", type="string", default="", |
| 228 |
help="Write the body of the HTML email to this file. [default = '']" ) |
| 229 |
|
| 230 |
clp.add_option( |
| 231 |
"--email-from-address=", dest="emailFromAddress", type="string", default="", |
| 232 |
help="Address reported in the sent email. [default '']" ) |
| 233 |
|
| 234 |
clp.add_option( |
| 235 |
"--send-email-to=", dest="sendEmailTo", type="string", default="", |
| 236 |
help="Send email to 'address1, address2, ...'. [default '']" ) |
| 237 |
|
| 238 |
addOptionParserChoiceOption( |
| 239 |
"--email-without-soft-hyphens", |
| 240 |
"emailWithoutSoftHyphensStr", |
| 241 |
("on", "off"), 1, |
| 242 |
"Remove soft hyphens from emails.", |
| 243 |
clp ) |
| 244 |
|
| 245 |
|
| 246 |
def validateAndConvertCmndLineOptions(inOptions): |
| 247 |
|
| 248 |
if inOptions.date == "": |
| 249 |
print("Error, can't have empty --date, must pass in --date=YYYY-MM-DD"+\ |
| 250 |
" or special values --date=today or --date=yesterday!") |
| 251 |
sys.exit(1) |
| 252 |
else: |
| 253 |
dateTimeObj = CDQAR.convertInputDateArgToYYYYMMDD( |
| 254 |
inOptions.cdashProjectTestingDayStartTime, |
| 255 |
inOptions.date) |
| 256 |
inOptions.date = CBTD.getDateStrFromDateTime(dateTimeObj) |
| 257 |
|
| 258 |
# ToDo: Assert more of the options to make sure they are correct! |
| 259 |
|
| 260 |
|
| 261 |
def setExtraCmndLineOptionsAfterParse(inOptions_inout): |
| 262 |
|
| 263 |
setattr(inOptions_inout, 'filterOutBuildsAndTestsNotMatchingExpectedBuilds', |
| 264 |
inOptions_inout.filterOutBuildsAndTestsNotMatchingExpectedBuildsStr == "on") |
| 265 |
|
| 266 |
setattr(inOptions_inout, 'useCachedCDashData', |
| 267 |
inOptions_inout.useCachedCDashDataStr == "on") |
| 268 |
|
| 269 |
setattr(inOptions_inout, 'requireTestHistoryMatchNonpassingTests', |
| 270 |
inOptions_inout.requireTestHistoryMatchNonpassingTestsStr == "on") |
| 271 |
|
| 272 |
setattr(inOptions_inout, 'printDetails', |
| 273 |
inOptions_inout.printDetailsStr == "on") |
| 274 |
|
| 275 |
setattr(inOptions_inout, 'listUnexpectedBuilds', |
| 276 |
inOptions_inout.listUnexpectedBuildsStr == "on") |
| 277 |
|
| 278 |
if inOptions_inout.cdashBaseCacheFilesPrefix == "": |
| 279 |
inOptions_inout.cdashBaseCacheFilesPrefix = \ |
| 280 |
CDQAR.getFileNameStrFromText(inOptions_inout.buildSetName)+"_" |
| 281 |
|
| 282 |
setattr(inOptions_inout, 'emailWithoutSoftHyphens', |
| 283 |
inOptions_inout.emailWithoutSoftHyphensStr == "on") |
| 284 |
|
| 285 |
|
| 286 |
def getCmndLineOptions(): |
| 287 |
from optparse import OptionParser |
| 288 |
clp = OptionParser(usage=usageHelp) |
| 289 |
injectCmndLineOptionsInParser(clp) |
| 290 |
(options, args) = clp.parse_args() |
| 291 |
validateAndConvertCmndLineOptions(options) |
| 292 |
setExtraCmndLineOptionsAfterParse(options) |
| 293 |
return options |
| 294 |
|
| 295 |
|
| 296 |
def fwdCmndLineOptions(inOptions, lt=""): |
| 297 |
io = inOptions |
| 298 |
cmndLineOpts = \ |
| 299 |
" --date='"+io.date+"'"+lt+\ |
| 300 |
" --cdash-project-testing-day-start-time='"+io.cdashProjectTestingDayStartTime+"'"+lt+\ |
| 301 |
" --cdash-project-name='"+io.cdashProjectName+"'"+lt+\ |
| 302 |
" --build-set-name='"+io.buildSetName+"'"+lt+\ |
| 303 |
" --cdash-site-url='"+io.cdashSiteUrl+"'"+lt+\ |
| 304 |
" --cdash-builds-filters='"+io.cdashBuildsFilters+"'"+lt+\ |
| 305 |
" --cdash-nonpassed-tests-filters='"+io.cdashNonpassedTestsFilters+"'"+lt+\ |
| 306 |
" --expected-builds-file='"+io.expectedBuildsFile+"'"+lt+\ |
| 307 |
" --tests-with-issue-trackers-file='"+io.testsWithIssueTrackersFile+"'"+lt+\ |
| 308 |
" --filter-out-builds-and-tests-not-matching-expected-builds='"+\ |
| 309 |
io.filterOutBuildsAndTestsNotMatchingExpectedBuildsStr+"'"+lt+\ |
| 310 |
" --cdash-queries-cache-dir='"+io.cdashQueriesCacheDir+"'"+lt+\ |
| 311 |
" --cdash-base-cache-files-prefix='"+io.cdashBaseCacheFilesPrefix+"'"+lt+\ |
| 312 |
" --use-cached-cdash-data='"+io.useCachedCDashDataStr+"'"+lt+\ |
| 313 |
" --limit-test-history-days='"+str(io.testHistoryDays)+"'"+lt+\ |
| 314 |
" --limit-table-rows='"+str(io.limitTableRows)+"'"+lt+\ |
| 315 |
" --require-test-history-match-nonpassing-tests='"+io.requireTestHistoryMatchNonpassingTestsStr+"'"+lt+\ |
| 316 |
" --print-details='"+io.printDetailsStr+"'"+lt+\ |
| 317 |
" --list-unexpected-builds='"+io.listUnexpectedBuildsStr+"'"+lt+\ |
| 318 |
" --write-unexpected-builds-to-fileo='"+io.writeUnexpectedBuildsToFile+"'"+lt+\ |
| 319 |
" --write-failing-tests-without-issue-trackers-to-file='"+io.writeFailingTestsWithoutIssueTrackersToFile+"'"+lt+\ |
| 320 |
" --write-test-data-to-file='"+io.writeTestDataToFile+"'"+lt+\ |
| 321 |
" --write-email-to-file='"+io.writeEmailToFile+"'"+lt+\ |
| 322 |
" --email-from-address='"+io.emailFromAddress+"'"+lt+\ |
| 323 |
" --send-email-to='"+io.sendEmailTo+"'"+lt+\ |
| 324 |
" --email-without-soft-hyphens='"+io.emailWithoutSoftHyphensStr+"'"+lt |
| 325 |
return cmndLineOpts |
| 326 |
|
| 327 |
|
| 328 |
def echoCmndLineOptions(inOptions): |
| 329 |
print(fwdCmndLineOptions(inOptions, " \\\n")) |
| 330 |
|
| 331 |
|
| 332 |
def echoCmndLine(inOptions): |
| 333 |
|
| 334 |
print("") |
| 335 |
print("**************************************************************************") |
| 336 |
print("cdash_analyze_and_report.py \\") |
| 337 |
|
| 338 |
echoCmndLineOptions(inOptions) |
| 339 |
|
| 340 |
|
| 341 |
# Strategy class that can get test history for a list of tests and set them in |
| 342 |
# the test dicts taking input from the cdash_analyze_and_report.py commandline |
| 343 |
# arguments. |
| 344 |
# |
| 345 |
class AddTestHistoryStrategy(object): |
| 346 |
|
| 347 |
|
| 348 |
def __init__(self, inOptions, testHistoryCacheDir): |
| 349 |
self.inOptions = inOptions |
| 350 |
self.testHistoryCacheDir = testHistoryCacheDir |
| 351 |
|
| 352 |
|
| 353 |
def getTestHistory(self, testLOD): |
| 354 |
|
| 355 |
sio = self.inOptions |
| 356 |
|
| 357 |
CDQAR.foreachTransform( |
| 358 |
testLOD, |
| 359 |
CDQAR.AddTestHistoryToTestDictFunctor( |
| 360 |
cdashUrl=sio.cdashSiteUrl, |
| 361 |
projectName=sio.cdashProjectName, |
| 362 |
date=sio.date, |
| 363 |
testingDayStartTimeUtc=sio.cdashProjectTestingDayStartTime, |
| 364 |
daysOfHistory=sio.testHistoryDays, |
| 365 |
testCacheDir=self.testHistoryCacheDir, |
| 366 |
useCachedCDashData=sio.useCachedCDashData, |
| 367 |
alwaysUseCacheFileIfExists=True, |
| 368 |
verbose=True, |
| 369 |
printDetails=sio.printDetails, |
| 370 |
requireMatchTestTopTestHistory=sio.requireTestHistoryMatchNonpassingTests, |
| 371 |
) |
| 372 |
) |
| 373 |
|
| 374 |
|
| 375 |
# |
| 376 |
# Run the script |
| 377 |
# |
| 378 |
|
| 379 |
if __name__ == '__main__': |
| 380 |
|
| 381 |
# |
| 382 |
# Get commandline options |
| 383 |
# |
| 384 |
|
| 385 |
inOptions = getCmndLineOptions() |
| 386 |
echoCmndLine(inOptions) |
| 387 |
|
| 388 |
cacheDirAndBaseFilePrefix = \ |
| 389 |
inOptions.cdashQueriesCacheDir+"/"+inOptions.cdashBaseCacheFilesPrefix |
| 390 |
|
| 391 |
# |
| 392 |
# A) Define common data, etc |
| 393 |
# |
| 394 |
|
| 395 |
tcd = CDQAR.TableColumnData |
| 396 |
pp = pprint.PrettyPrinter(indent=2) |
| 397 |
|
| 398 |
groupSiteBuildNameSortOrder = ['group', 'site', 'buildname'] |
| 399 |
|
| 400 |
# |
| 401 |
# B) Sound off |
| 402 |
# |
| 403 |
|
| 404 |
print("***") |
| 405 |
print("*** Query and analyze CDash results for "+inOptions.buildSetName+\ |
| 406 |
" for testing day "+inOptions.date) |
| 407 |
print("***") |
| 408 |
|
| 409 |
# |
| 410 |
# C) Create beginning of email body (that does not require getting any data off CDash) |
| 411 |
# |
| 412 |
|
| 413 |
# Aggregation of vars that get updated in this main() body and by functions |
| 414 |
# called. |
| 415 |
cdashReportData = CDQAR.CDashReportData() |
| 416 |
|
| 417 |
cdashReportData.htmlEmailBodyTop += \ |
| 418 |
"<h2>Build and Test results for "+inOptions.buildSetName \ |
| 419 |
+" on "+inOptions.date+"</h2>\n\n" |
| 420 |
|
| 421 |
# |
| 422 |
# D) Read data files, get data off of CDash, do analysis, and construct HTML |
| 423 |
# body parts |
| 424 |
# |
| 425 |
|
| 426 |
try: |
| 427 |
|
| 428 |
# Beginning of top full build and tests CDash links paragraph |
| 429 |
cdashReportData.htmlEmailBodyTop += "<p>\n" |
| 430 |
|
| 431 |
# |
| 432 |
# D.1) Read data from input files, set up cache directories |
| 433 |
# |
| 434 |
# Assert this data is correct and abort if there is an error before we run |
| 435 |
# expensive CDash queries! |
| 436 |
# |
| 437 |
|
| 438 |
# Get list of expected builds from input CSV file |
| 439 |
expectedBuildsLOD = CDQAR.getExpectedBuildsListOfDictsFromCsvFileArg( |
| 440 |
inOptions.expectedBuildsFile) |
| 441 |
print("\nNum expected builds = "+str(len(expectedBuildsLOD))) |
| 442 |
|
| 443 |
# Create a SearchableListOfDict object to help look up expected builds |
| 444 |
# given a build dict by key/value pairs 'group', 'site', and 'buildname' |
| 445 |
# (requires unique builds with these key/value pairs) |
| 446 |
expectedBuildsSLOD = CDQAR.createSearchableListOfBuilds(expectedBuildsLOD) |
| 447 |
|
| 448 |
# Create a SearchableListOfDicts that will look up an expected build given |
| 449 |
# just a test dict fields ['site', 'buildName']. (The list of tests with |
| 450 |
# issue trackers does not have 'group' since cdash/queryTests.php does not |
| 451 |
# give the 'group' associated with each test. Also, note that we need |
| 452 |
# this special SearchableListOfDicts since the Build Name key name |
| 453 |
# different for a cdash/queryTests.php test dict 'buildName' and a |
| 454 |
# cdash/index.php build dict 'buildname'.) |
| 455 |
testsToExpectedBuildsSLOD = \ |
| 456 |
CDQAR.createTestToBuildSearchableListOfDicts(expectedBuildsLOD) |
| 457 |
# ToDo: Put in try/except to print about error in duplicate rows in the |
| 458 |
# list of expected builds. |
| 459 |
|
| 460 |
# Get list of tests with issue trackers from the input CSV file |
| 461 |
if inOptions.testsWithIssueTrackersFile: |
| 462 |
fullTestsWithIssueTrackersLOD = CDQAR.getTestsWtihIssueTrackersListFromCsvFile( |
| 463 |
inOptions.testsWithIssueTrackersFile) |
| 464 |
else: |
| 465 |
fullTestsWithIssueTrackersLOD = [] |
| 466 |
print("\nNum tests with issue trackers read from CSV file = "+\ |
| 467 |
str(len(fullTestsWithIssueTrackersLOD))) |
| 468 |
|
| 469 |
if inOptions.filterOutBuildsAndTestsNotMatchingExpectedBuilds: |
| 470 |
(testsWithIssueTrackersLOD, testsWithIssueTrackersNotExpectedLOD) = \ |
| 471 |
CDQAR.splitTestsOnMatchExpectedBuilds(fullTestsWithIssueTrackersLOD, |
| 472 |
testsToExpectedBuildsSLOD) |
| 473 |
print("Num tests with issue trackers matching expected builds = "+\ |
| 474 |
str(len(testsWithIssueTrackersLOD))) |
| 475 |
else: |
| 476 |
testsWithIssueTrackersLOD = fullTestsWithIssueTrackersLOD |
| 477 |
print("Num tests with issue trackers = "+\ |
| 478 |
str(len(testsWithIssueTrackersLOD))) |
| 479 |
|
| 480 |
# Get a SearchableListOfDicts for the tests with issue trackers to allow |
| 481 |
# them to be looked up based on matching ['site', 'buildName', 'testname'] |
| 482 |
# key/value pairs. |
| 483 |
testsWithIssueTrackersSLOD = \ |
| 484 |
CDQAR.createSearchableListOfTests(testsWithIssueTrackersLOD) |
| 485 |
# ToDo: Put in try/except to print about error in duplicate rows in the |
| 486 |
# list of tests with issue trackers. |
| 487 |
|
| 488 |
# Get a functor that will return True if a passed-in test dict matches a |
| 489 |
# test with an issue tracker for the test key/value pairs ['site', |
| 490 |
# 'buildName', and 'testname']. |
| 491 |
testsWithIssueTrackerMatchFunctor = \ |
| 492 |
CDQAR.MatchDictKeysValuesFunctor(testsWithIssueTrackersSLOD) |
| 493 |
|
| 494 |
# Assert that the list of tests with issue trackers matches the list of |
| 495 |
# expected builds |
| 496 |
(allTestsMatch, errMsg) = CDQAR.doTestsWithIssueTrackersMatchExpectedBuilds( |
| 497 |
testsWithIssueTrackersLOD, testsToExpectedBuildsSLOD) |
| 498 |
if not allTestsMatch: |
| 499 |
raise Exception(errMsg) |
| 500 |
|
| 501 |
# Test history cache dir |
| 502 |
testHistoryCacheDir = inOptions.cdashQueriesCacheDir+"/test_history" |
| 503 |
if not os.path.exists(testHistoryCacheDir): |
| 504 |
print("\nCreating new test cache directory '"+testHistoryCacheDir+"'") |
| 505 |
os.mkdir(testHistoryCacheDir) |
| 506 |
|
| 507 |
# |
| 508 |
# D.2) Get top-level lists of build and nonpassing tests off CDash |
| 509 |
# |
| 510 |
|
| 511 |
# |
| 512 |
# D.2.a) Get list of dicts of builds off cdash/index.phpp |
| 513 |
# |
| 514 |
|
| 515 |
# @arghdos: note, we do not have to normalize the URLs from the input |
| 516 |
# options because they are currently taken from the cdash site already |
| 517 |
# (i.e., they are already in normalized form). |
| 518 |
|
| 519 |
cdashIndexBuildsBrowserUrl = CDQAR.getCDashIndexBrowserUrl( |
| 520 |
inOptions.cdashSiteUrl, inOptions.cdashProjectName, inOptions.date, |
| 521 |
inOptions.cdashBuildsFilters) |
| 522 |
|
| 523 |
print("\nCDash builds browser URL:\n\n "+cdashIndexBuildsBrowserUrl+"\n") |
| 524 |
|
| 525 |
cdashIndexBuildsQueryUrl = CDQAR.getCDashIndexQueryUrl( |
| 526 |
inOptions.cdashSiteUrl, |
| 527 |
inOptions.cdashProjectName, |
| 528 |
inOptions.date, |
| 529 |
inOptions.cdashBuildsFilters ) |
| 530 |
|
| 531 |
fullCDashIndexBuildsJsonCacheFile = \ |
| 532 |
cacheDirAndBaseFilePrefix+"fullCDashIndexBuilds.json" |
| 533 |
|
| 534 |
fullBuildsLOD = CDQAR.downloadBuildsOffCDashAndFlatten( |
| 535 |
cdashIndexBuildsQueryUrl, |
| 536 |
fullCDashIndexBuildsJsonCacheFile, |
| 537 |
inOptions.useCachedCDashData ) |
| 538 |
|
| 539 |
print("\nNum builds downloaded from CDash = "+str(len(fullBuildsLOD))) |
| 540 |
|
| 541 |
(buildsExpectedLOD, buildsUnexpectedLOD) = \ |
| 542 |
CDQAR.splitTestsOnMatchExpectedBuilds(fullBuildsLOD, expectedBuildsSLOD) |
| 543 |
|
| 544 |
if inOptions.filterOutBuildsAndTestsNotMatchingExpectedBuilds: |
| 545 |
print("Num builds matching expected builds = "+str(len(buildsExpectedLOD))) |
| 546 |
buildsLOD = buildsExpectedLOD |
| 547 |
else: |
| 548 |
buildsLOD = fullBuildsLOD |
| 549 |
|
| 550 |
if inOptions.listUnexpectedBuilds: |
| 551 |
print("Num builds unexpected = "+str(len(buildsUnexpectedLOD))) |
| 552 |
|
| 553 |
print("Num builds = "+str(len(buildsLOD))) |
| 554 |
|
| 555 |
# HTML line "Builds on CDash" |
| 556 |
cdashReportData.htmlEmailBodyTop += \ |
| 557 |
"<a href=\""+cdashIndexBuildsBrowserUrl+"\">"+\ |
| 558 |
"Builds on CDash</a> (num/expected="+\ |
| 559 |
str(len(buildsLOD))+"/"+str(len(expectedBuildsLOD))+")<br>\n" |
| 560 |
|
| 561 |
# Create a SearchableListOfDict object to help look up builds given a |
| 562 |
# build dict by key/value pairs 'group', 'site', and 'buildname' (requires |
| 563 |
# unique builds with these key/value pairs) |
| 564 |
buildsSLOD = CDQAR.createSearchableListOfBuilds(buildsLOD) |
| 565 |
# ToDo: Add try/except to report duplicate builds in case this raises an |
| 566 |
# exception. |
| 567 |
|
| 568 |
# |
| 569 |
# D.2.b) Get list of dicts of all nonpassing tests off |
| 570 |
# cdash/queryTests.php |
| 571 |
# |
| 572 |
|
| 573 |
cdashNonpassingTestsBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( |
| 574 |
inOptions.cdashSiteUrl, inOptions.cdashProjectName, inOptions.date, |
| 575 |
inOptions.cdashNonpassedTestsFilters) |
| 576 |
|
| 577 |
print("\nGetting list of nonpassing tests from CDash ...\n") |
| 578 |
|
| 579 |
print("\nCDash nonpassing tests browser URL:\n\n"+\ |
| 580 |
" "+cdashNonpassingTestsBrowserUrl+"\n") |
| 581 |
|
| 582 |
cdashNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( |
| 583 |
inOptions.cdashSiteUrl, inOptions.cdashProjectName, inOptions.date, |
| 584 |
inOptions.cdashNonpassedTestsFilters) |
| 585 |
|
| 586 |
cdashNonpassingTestsQueryJsonCacheFile = \ |
| 587 |
cacheDirAndBaseFilePrefix+"fullCDashNonpassingTests.json" |
| 588 |
|
| 589 |
fullNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( |
| 590 |
cdashNonpassingTestsQueryUrl, cdashNonpassingTestsQueryJsonCacheFile, |
| 591 |
inOptions.useCachedCDashData ) |
| 592 |
|
| 593 |
print("\nNum nonpassing tests direct from CDash query = "+\ |
| 594 |
str(len(fullNonpassingTestsLOD))) |
| 595 |
|
| 596 |
if inOptions.filterOutBuildsAndTestsNotMatchingExpectedBuilds: |
| 597 |
(nonpassingTestsLOD, nonpassingTestsNotExpectedLOD) = \ |
| 598 |
CDQAR.splitTestsOnMatchExpectedBuilds(fullNonpassingTestsLOD, |
| 599 |
testsToExpectedBuildsSLOD) |
| 600 |
print("Num nonpassing tests matching expected builds = "+\ |
| 601 |
str(len(nonpassingTestsLOD))) |
| 602 |
else: |
| 603 |
nonpassingTestsLOD = fullNonpassingTestsLOD |
| 604 |
|
| 605 |
print("Num nonpassing tests = "+\ |
| 606 |
str(len(nonpassingTestsLOD))) |
| 607 |
|
| 608 |
# HTML line "Nonpassing Tests on CDash" |
| 609 |
cdashReportData.htmlEmailBodyTop += \ |
| 610 |
"<a href=\""+cdashNonpassingTestsBrowserUrl+"\">"+\ |
| 611 |
"Non-passing Tests on CDash</a> (num="+str(len(nonpassingTestsLOD))+")<br>\n" |
| 612 |
|
| 613 |
# End of full build and test link paragraph and start the next paragraph |
| 614 |
# for the summary of failures and other tables |
| 615 |
cdashReportData.htmlEmailBodyTop += \ |
| 616 |
"</p>\n\n"+\ |
| 617 |
"<p>\n" |
| 618 |
|
| 619 |
# Create a SearchableListOfDicts object for looking up a nonpassing test |
| 620 |
# given the test dict fields 'site', 'buildName', and 'testname'. |
| 621 |
nonpassingTestsSLOD = CDQAR.createSearchableListOfTests( |
| 622 |
nonpassingTestsLOD, removeExactDuplicateElements=True, |
| 623 |
checkDictsAreSame_in=CDQAR.checkCDashTestDictsAreSame ) |
| 624 |
# NOTE: Above we add the option to remove exact duplicate tests since |
| 625 |
# cdash/queryTests.php can return duplicate tests (i.e. all test dict |
| 626 |
# fields are the same except and has the same buildid but could have |
| 627 |
# different testids!) |
| 628 |
# ToDo: Add try/except for above code in order to report duplicate tests |
| 629 |
# where the buildid (and other fields) not match. |
| 630 |
|
| 631 |
print("Num nonpassing tests after removing duplicate tests = "+\ |
| 632 |
str(len(nonpassingTestsLOD))) |
| 633 |
|
| 634 |
# Create a functor to to see if a test dict matches one of the nonpassing |
| 635 |
# tests downloaded from cdash/queryTests.php. |
| 636 |
nonpassingTestsMatchFunctor = \ |
| 637 |
CDQAR.MatchDictKeysValuesFunctor(nonpassingTestsSLOD) |
| 638 |
|
| 639 |
# |
| 640 |
# D.3) Partition the various list of tests into different sets that will |
| 641 |
# be displayed in different tables. |
| 642 |
# |
| 643 |
|
| 644 |
# Add issue tracker info for all nonpassing tests (including adding empty |
| 645 |
# issue tracker fields for tests that don't have issue trackers) |
| 646 |
CDQAR.foreachTransform( nonpassingTestsLOD, |
| 647 |
CDQAR.AddIssueTrackerInfoToTestDictFunctor(testsWithIssueTrackersSLOD)) |
| 648 |
|
| 649 |
# Split the list of nonpassing tests into those with and without issue |
| 650 |
# trackers |
| 651 |
(nonpassingTestsWithIssueTrackersLOD,nonpassingTestsWithoutIssueTrackersLOD)=\ |
| 652 |
CDQAR.splitListOnMatch(nonpassingTestsLOD, testsWithIssueTrackerMatchFunctor) |
| 653 |
print("Num nonpassing tests without issue trackers = "+\ |
| 654 |
str(len(nonpassingTestsWithoutIssueTrackersLOD))) |
| 655 |
print("Num nonpassing tests with issue trackers = "+\ |
| 656 |
str(len(nonpassingTestsWithIssueTrackersLOD))) |
| 657 |
|
| 658 |
# Split the list nonpassing tests without issue trackers into 'twoif' and |
| 659 |
# 'twoinp' |
| 660 |
(twoifLOD, twoinrLOD) = CDQAR.splitListOnMatch( |
| 661 |
nonpassingTestsWithoutIssueTrackersLOD, CDQAR.isTestFailed) |
| 662 |
print("Num nonpassing tests without issue trackers Failed = "+str(len(twoifLOD))) |
| 663 |
print("Num nonpassing tests without issue trackers Not Run = "+str(len(twoinrLOD))) |
| 664 |
|
| 665 |
# Split the list nonpassing tests with issue trackers into 'twif' and |
| 666 |
# 'twinp' |
| 667 |
(twifLOD, twinrLOD) = CDQAR.splitListOnMatch( |
| 668 |
nonpassingTestsWithIssueTrackersLOD, CDQAR.isTestFailed) |
| 669 |
print("Num nonpassing tests with issue trackers Failed = "+str(len(twifLOD))) |
| 670 |
print("Num nonpassing tests with issue trackers Not Run = "+str(len(twinrLOD))) |
| 671 |
|
| 672 |
# Get list of tests with issue trackers that are not in the list of |
| 673 |
# nonpassing tests (and therefore these are passing or missing) |
| 674 |
testsWithIssueTrackersGrossPassingOrMissingLOD = CDQAR.getFilteredList( |
| 675 |
testsWithIssueTrackersSLOD, |
| 676 |
CDQAR.NotMatchFunctor(nonpassingTestsMatchFunctor) ) |
| 677 |
print("Num tests with issue trackers gross passing or missing = "+\ |
| 678 |
str(len(testsWithIssueTrackersGrossPassingOrMissingLOD))) |
| 679 |
|
| 680 |
# |
| 681 |
# D.4) Process and tabulate lists of builds |
| 682 |
# |
| 683 |
|
| 684 |
buildsetReporter = CDQAR.SingleBuildsetReporter(cdashReportData) |
| 685 |
|
| 686 |
# |
| 687 |
# 'bm' |
| 688 |
# |
| 689 |
|
| 690 |
print("\nSearch for any missing expected builds ...\n") |
| 691 |
|
| 692 |
missingExpectedBuildsLOD = CDQAR.getMissingExpectedBuildsList( |
| 693 |
buildsSLOD, expectedBuildsLOD) |
| 694 |
|
| 695 |
buildsetReporter.reportSingleBuildset("Builds Missing", "bm", |
| 696 |
missingExpectedBuildsLOD, |
| 697 |
buildsetGlobalPass=False, |
| 698 |
buildsetColor=CDQAR.cdashColorFailed(), |
| 699 |
buildsetColDataList=[ |
| 700 |
tcd("Group", 'group'), |
| 701 |
tcd("Site", 'site'), |
| 702 |
tcd("Build Name", 'buildname'), |
| 703 |
tcd("Missing Status", 'status'), |
| 704 |
], |
| 705 |
) |
| 706 |
|
| 707 |
# |
| 708 |
# 'cf' |
| 709 |
# |
| 710 |
|
| 711 |
print("\nSearch for any builds with configure failures ...\n") |
| 712 |
|
| 713 |
buildsWithConfigureFailuresLOD = \ |
| 714 |
CDQAR.getFilteredList(buildsSLOD, CDQAR.buildHasConfigureFailures) |
| 715 |
|
| 716 |
buildsetReporter.reportSingleBuildset("Builds with Configure Failures", "cf", |
| 717 |
buildsWithConfigureFailuresLOD, |
| 718 |
buildsetGlobalPass=False, |
| 719 |
buildsetColor=CDQAR.cdashColorFailed(), |
| 720 |
) |
| 721 |
|
| 722 |
# |
| 723 |
# 'bf' |
| 724 |
# |
| 725 |
|
| 726 |
print("\nSearch for any builds with compilation (build) failures ...\n") |
| 727 |
|
| 728 |
buildsWithBuildFailuresLOD = \ |
| 729 |
CDQAR.getFilteredList(buildsSLOD, CDQAR.buildHasBuildFailures) |
| 730 |
|
| 731 |
buildsetReporter.reportSingleBuildset("Builds with Build Failures", "bf", |
| 732 |
buildsWithBuildFailuresLOD, |
| 733 |
buildsetGlobalPass=False, |
| 734 |
buildsetColor=CDQAR.cdashColorFailed(), |
| 735 |
) |
| 736 |
|
| 737 |
# |
| 738 |
# 'bu' |
| 739 |
# |
| 740 |
|
| 741 |
if inOptions.listUnexpectedBuilds: |
| 742 |
buildsetReporter.reportSingleBuildset("Builds Unexpected", "bu", |
| 743 |
buildsUnexpectedLOD, |
| 744 |
buildsetGlobalPass=True, |
| 745 |
buildsetColor=None, |
| 746 |
) |
| 747 |
|
| 748 |
# |
| 749 |
# D.5) Analyaize and report the different sets of tests |
| 750 |
# |
| 751 |
|
| 752 |
# |
| 753 |
# D.5.a) Final processing of lists of tests and splitting into the |
| 754 |
# different tests sets to report |
| 755 |
# |
| 756 |
|
| 757 |
# Object to make it easy to process the different test sets |
| 758 |
addTestHistoryStrategy = AddTestHistoryStrategy(inOptions, testHistoryCacheDir) |
| 759 |
testsetReporter = CDQAR.SingleTestsetReporter(cdashReportData, |
| 760 |
addTestHistoryStrategy=addTestHistoryStrategy) |
| 761 |
|
| 762 |
# Special functor to look up missing expected build given a test dict |
| 763 |
testsToMissingExpectedBuildsSLOD = \ |
| 764 |
CDQAR.createTestToBuildSearchableListOfDicts(missingExpectedBuildsLOD) |
| 765 |
|
| 766 |
# Functor for matching an missing expected build given a test dict |
| 767 |
testMatchesMissingExpectedBuildsFunctor = CDQAR.MatchDictKeysValuesFunctor( |
| 768 |
testsToMissingExpectedBuildsSLOD) |
| 769 |
|
| 770 |
# Get list of tests with issue trackers that are not in the list of |
| 771 |
# nonpassing tests and don't match expected builds (and therefore will be |
| 772 |
# included in the sets 'twip' and 'twim'). |
| 773 |
( testsWithIssueTrackersMatchingMissingExpectedBuildsLOD, |
| 774 |
testsWithIssueTrackersPassingOrMissingLOD ) \ |
| 775 |
= \ |
| 776 |
CDQAR.splitListOnMatch( testsWithIssueTrackersGrossPassingOrMissingLOD, |
| 777 |
testMatchesMissingExpectedBuildsFunctor ) |
| 778 |
print("\nNum tests with issue trackers passing or missing matching"+\ |
| 779 |
" posted builds = "+str(len(testsWithIssueTrackersPassingOrMissingLOD))) |
| 780 |
print("\nTests with issue trackers missing that match"+\ |
| 781 |
" missing expected builds: num="+\ |
| 782 |
str(len(testsWithIssueTrackersMatchingMissingExpectedBuildsLOD))) |
| 783 |
if len(testsWithIssueTrackersMatchingMissingExpectedBuildsLOD) > 0: |
| 784 |
for testDict in testsWithIssueTrackersMatchingMissingExpectedBuildsLOD: |
| 785 |
print(" "+sorted_dict_str(testDict)) |
| 786 |
print("\nNOTE: The above tests will NOT be listed in the set 'twim'!") |
| 787 |
|
| 788 |
# Get test history for all of the tests with issue trackers that are not |
| 789 |
# passing or missing. These will either be tests that are passing today |
| 790 |
# (and therefore have history) or they will be tests that are missing. |
| 791 |
# (But don't get test history or list out tests with issue trackers that |
| 792 |
# match missing expected builds that did not submit any test data today.) |
| 793 |
|
| 794 |
twipLOD = [] |
| 795 |
twimLOD = [] |
| 796 |
|
| 797 |
if testsWithIssueTrackersPassingOrMissingLOD: |
| 798 |
|
| 799 |
print("\nGetting test history for tests with issue trackers"+\ |
| 800 |
" passing or missing: num="+str(len(testsWithIssueTrackersPassingOrMissingLOD))) |
| 801 |
|
| 802 |
CDQAR.foreachTransform( |
| 803 |
testsWithIssueTrackersPassingOrMissingLOD, |
| 804 |
CDQAR.AddTestHistoryToTestDictFunctor( |
| 805 |
inOptions.cdashSiteUrl, |
| 806 |
inOptions.cdashProjectName, |
| 807 |
inOptions.date, |
| 808 |
inOptions.cdashProjectTestingDayStartTime, |
| 809 |
inOptions.testHistoryDays, |
| 810 |
testHistoryCacheDir, |
| 811 |
useCachedCDashData=inOptions.useCachedCDashData, |
| 812 |
alwaysUseCacheFileIfExists=True, |
| 813 |
verbose=True, |
| 814 |
printDetails=inOptions.printDetails, |
| 815 |
requireMatchTestTopTestHistory=inOptions.requireTestHistoryMatchNonpassingTests, |
| 816 |
|
| 817 |
) |
| 818 |
) |
| 819 |
|
| 820 |
# Split into lists for 'twip' and 'twim' |
| 821 |
(twipLOD, twimLOD) = CDQAR.splitListOnMatch( |
| 822 |
testsWithIssueTrackersPassingOrMissingLOD, CDQAR.isTestPassed ) |
| 823 |
|
| 824 |
print("\nNum tests with issue trackers Passed = "+str(len(twipLOD))) |
| 825 |
print("Num tests with issue trackers Missing = "+str(len(twimLOD))) |
| 826 |
|
| 827 |
# |
| 828 |
# D.5.b) Report the different sets of tests |
| 829 |
# |
| 830 |
# NOTE: The order of these is chosen so those that require action of the |
| 831 |
# person doing the triaging are sorted to the top. |
| 832 |
# |
| 833 |
|
| 834 |
testsetReporter.reportSingleTestset( |
| 835 |
CDQAR.getStandardTestsetTypeInfo('twoif'), |
| 836 |
len(twoifLOD), twoifLOD, |
| 837 |
limitTableRows=inOptions.limitTableRows, |
| 838 |
getTestHistory=True, |
| 839 |
) |
| 840 |
|
| 841 |
testsetReporter.reportSingleTestset( |
| 842 |
CDQAR.getStandardTestsetTypeInfo('twoinr'), |
| 843 |
len(twoinrLOD), twoinrLOD, |
| 844 |
limitTableRows=inOptions.limitTableRows, |
| 845 |
getTestHistory=True, |
| 846 |
) |
| 847 |
|
| 848 |
testsetReporter.reportSingleTestset( |
| 849 |
CDQAR.getStandardTestsetTypeInfo('twip'), |
| 850 |
len(twipLOD), twipLOD, |
| 851 |
limitTableRows=None, |
| 852 |
getTestHistory=False, # Already got it above! |
| 853 |
) |
| 854 |
|
| 855 |
testsetReporter.reportSingleTestset( |
| 856 |
CDQAR.getStandardTestsetTypeInfo('twim', ""), |
| 857 |
len(twimLOD), twimLOD, |
| 858 |
limitTableRows=None, |
| 859 |
getTestHistory=False, # Already got it above! |
| 860 |
) |
| 861 |
|
| 862 |
testsetReporter.reportSingleTestset( |
| 863 |
CDQAR.getStandardTestsetTypeInfo('twif', ""), |
| 864 |
len(twifLOD), twifLOD, |
| 865 |
limitTableRows=None, |
| 866 |
getTestHistory=True, |
| 867 |
) |
| 868 |
|
| 869 |
testsetReporter.reportSingleTestset( |
| 870 |
CDQAR.getStandardTestsetTypeInfo('twinr', ""), |
| 871 |
len(twinrLOD), twinrLOD, |
| 872 |
limitTableRows=None, |
| 873 |
getTestHistory=True, |
| 874 |
) |
| 875 |
|
| 876 |
# |
| 877 |
# D.6) Write out list of unexpected builds to CSV file |
| 878 |
# |
| 879 |
|
| 880 |
if inOptions.writeUnexpectedBuildsToFile: |
| 881 |
unexpectedBuildsCsvFileName = inOptions.writeUnexpectedBuildsToFile |
| 882 |
print("\nWriting list of unexpected builds to file "\ |
| 883 |
+unexpectedBuildsCsvFileName+" ...") |
| 884 |
CDQAR.writeExpectedBuildsListOfDictsToCsvFile(buildsUnexpectedLOD, |
| 885 |
unexpectedBuildsCsvFileName) |
| 886 |
|
| 887 |
# |
| 888 |
# D.7) Write out list twiof to CSV file |
| 889 |
# |
| 890 |
|
| 891 |
if inOptions.writeFailingTestsWithoutIssueTrackersToFile: |
| 892 |
twoifCsvFileName = inOptions.writeFailingTestsWithoutIssueTrackersToFile |
| 893 |
print("\nWriting list of 'twiof' to file "+twoifCsvFileName+" ...") |
| 894 |
CDQAR.writeTestsListOfDictsToCsvFile(twoifLOD, twoifCsvFileName) |
| 895 |
|
| 896 |
# |
| 897 |
# D.8) Write out test data to CSV file |
| 898 |
# |
| 899 |
|
| 900 |
if inOptions.writeTestDataToFile: |
| 901 |
testDataFileName = inOptions.writeTestDataToFile |
| 902 |
print("\nWriting out gathered test data to file "+testDataFileName+" ...") |
| 903 |
testDataLOD = twipLOD + twimLOD + twifLOD + twinrLOD |
| 904 |
CDQAR.foreachTransform(testDataLOD, |
| 905 |
CDQAR.AddCDashTestingDayFunctor(inOptions.date)) |
| 906 |
# ToDo: Add the first inOptions.limitTableRows elements of twiofLOD and twoinrLOD? |
| 907 |
CDQAR.pprintPythonDataToFile(testDataLOD, testDataFileName) |
| 908 |
|
| 909 |
except Exception: |
| 910 |
# Traceback! |
| 911 |
print("") |
| 912 |
sys.stdout.flush() |
| 913 |
traceback.print_exc() |
| 914 |
# Report the error |
| 915 |
cdashReportData.htmlEmailBodyBottom += "\n<pre><code>\n"+\ |
| 916 |
traceback.format_exc()+"\n</code></pre>\n" |
| 917 |
print("\nError, could not compute the analysis due to"+\ |
| 918 |
" above error so return failed!") |
| 919 |
cdashReportData.globalPass = False |
| 920 |
cdashReportData.summaryLineDataNumbersList.append("SCRIPT CRASHED") |
| 921 |
|
| 922 |
# |
| 923 |
# E) Put together final email summary line |
| 924 |
# |
| 925 |
|
| 926 |
summaryLine = CDQAR.getOverallCDashReportSummaryLine(cdashReportData, |
| 927 |
inOptions.buildSetName, inOptions.date) |
| 928 |
|
| 929 |
# |
| 930 |
# F) Finish off HTML body guts and define overall HTML body style |
| 931 |
# |
| 932 |
|
| 933 |
# Finish off the top paragraph of the summary lines |
| 934 |
cdashReportData.htmlEmailBodyTop += \ |
| 935 |
"</p>\n" |
| 936 |
|
| 937 |
# |
| 938 |
# G) Write HTML body file and/or send HTML email(s) |
| 939 |
# |
| 940 |
|
| 941 |
defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr() |
| 942 |
|
| 943 |
if inOptions.writeEmailToFile: |
| 944 |
print("\nWriting HTML file '"+inOptions.writeEmailToFile+"' ...") |
| 945 |
fullCDashHtmlReportPageStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, |
| 946 |
pageTitle=summaryLine, pageStyle=defaultPageStyle) |
| 947 |
with open(inOptions.writeEmailToFile, 'w') as outFile: |
| 948 |
outFile.write(fullCDashHtmlReportPageStr) |
| 949 |
|
| 950 |
if inOptions.sendEmailTo: |
| 951 |
htmlEmailBodyStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, |
| 952 |
pageStyle=defaultPageStyle) |
| 953 |
for emailAddress in inOptions.sendEmailTo.split(','): |
| 954 |
emailAddress = emailAddress.strip() |
| 955 |
print("\nSending email to '"+emailAddress+"' ...") |
| 956 |
msg=CDQAR.createHtmlMimeEmail( |
| 957 |
inOptions.emailFromAddress, emailAddress, summaryLine, "", |
| 958 |
htmlEmailBodyStr, inOptions.emailWithoutSoftHyphens) |
| 959 |
CDQAR.sendMineEmail(msg) |
| 960 |
|
| 961 |
# |
| 962 |
# H) Return final global pass/fail |
| 963 |
# |
| 964 |
|
| 965 |
print("\n"+summaryLine+"\n") |
| 966 |
|
| 967 |
if cdashReportData.globalPass: |
| 968 |
sys.exit(0) |
| 969 |
else: |
| 970 |
sys.exit(1) |