Debian Codegraph @

cmake/tribits/ci_support/TribitsDependencies.py

raw
1 #!/usr/bin/env python3
2
3 # @HEADER
4 # *****************************************************************************
5 # TriBITS: Tribal Build, Integrate, and Test System
6 #
7 # Copyright 2013-2016 NTESS and the TriBITS contributors.
8 # SPDX-License-Identifier: BSD-3-Clause
9 # *****************************************************************************
10 # @HEADER
11
12
13 import xml.dom.minidom
14 import os
15 import sys
16
17 from FindGeneralScriptSupport import *
18 from GeneralScriptSupport import *
19
20 #
21 # Default file locations
22 #
23
24
25 def getDefaultDepsXmlDirectory(rootDir):
26 return os.path.join(rootDir, 'cmake', 'dependencies')
27
28
29 def getDefaultDepsXmlInFile(rootDir, projectName):
30 return os.path.join(
31 getDefaultDepsXmlDirectory(rootDir),
32 '%sPackageDependencies.xml' % projectName)
33
34
35 def getDefaultDepsHtmlOutFile(rootDir, projectName):
36 return os.path.join(
37 getDefaultDepsXmlDirectory(rootDir),
38 '%sPackageDependenciesTable.html' % projectName)
39
40
41 def getDefaultCDashDepsXmlFile(rootDir):
42 return os.path.join(
43 getDefaultDepsXmlDirectory(rootDir),
44 'CDashSubprojectDependencies.xml')
45
46
47 #
48 # Store and manipulate the dependencies
49 #
50
51
52 class PackageEmailAddresses:
53
54 def __init__(self, regression_in):
55 self.regression = regression_in
56
57 def __str__(self):
58 return "{regression="+self.regression+"}"
59
60
61 class PackageDependencies:
62
63 def __init__(self, packageName_in, packageDir_in, packageType_in,
64 libRequiredDepPackages_in, libOptionalDepPackages_in,
65 testRequiredDepPackages_in, testOptionalDepPackages_in,
66 emailAddresses_in, parentPackage_in
67 ):
68 self.packageName = packageName_in
69 self.packageDir = packageDir_in
70 self.packageType = packageType_in
71 self.packageID = -1
72 self.libRequiredDepPackages = libRequiredDepPackages_in
73 self.libOptionalDepPackages = libOptionalDepPackages_in
74 self.testRequiredDepPackages = testRequiredDepPackages_in
75 self.testOptionalDepPackages = testOptionalDepPackages_in
76 self.emailAddresses = emailAddresses_in
77 self.parentPackage = parentPackage_in
78
79 def __str__(self):
80 return "{\n"+\
81 " packageName="+self.packageName+",\n"+\
82 " packageID="+str(self.packageID)+",\n"+\
83 " packageDir="+str(self.packageDir)+",\n"+\
84 " libRequiredDepPackages="+str(self.libRequiredDepPackages)+",\n"+\
85 " libOptionalDepPackages="+str(self.libOptionalDepPackages)+",\n"+\
86 " testRequiredDepPackages="+str(self.testRequiredDepPackages)+",\n"+\
87 " testOptionalDepPackages="+str(self.testOptionalDepPackages)+" \n"+\
88 " emailAddresses="+str(self.emailAddresses)+"\n"+\
89 " parentPackage="+str(self.parentPackage)+"\n"+\
90 "}\n"
91
92
93 def isRequiredDep(dep):
94 return (dep[-1] == 'R')
95
96
97 def isDirectDep(dep):
98 return (dep[0] != 'I')
99
100
101 def isLibDep(dep):
102 return (dep[0] == 'L' or dep[1] == 'L')
103
104
105 #
106 # (dep1, dep2) => newDep
107 #
108 # (*) Required dependencies trump optional dependencies
109 # (*) Direct dependencies trump indirect dependencies
110 # (*) Library dependicnes trump test dependencies
111 #
112 def updatePackageDep(dep1, dep2):
113
114 #print("\n updatePackageDep("+dep1+", "+dep2+") ...")
115
116 dep1_required = isRequiredDep(dep1)
117 dep1_direct = isDirectDep(dep1)
118 dep1_lib = isLibDep(dep1)
119
120 dep2_required = isRequiredDep(dep2)
121 dep2_direct = isDirectDep(dep2)
122 dep2_lib = isLibDep(dep2)
123
124 selectedDep = False
125
126 if dep1 == dep2:
127 newDep = dep1
128 selectedDep = True
129
130 # Required trumps optional
131 if not selectedDep:
132 if dep1_required and not dep2_required:
133 newDep = dep1
134 selectedDep = True
135 elif not dep1_required and dep2_required:
136 newDep = dep2
137 selectedDep = True
138
139 # Direct trumps indirect
140 if not selectedDep:
141 if dep1_direct and not dep2_direct:
142 newDep = dep1
143 selectedDep = True
144 elif not dep1_direct and dep2_direct:
145 newDep = dep2
146 selectedDep = True
147
148 # Library trumps test
149 if not selectedDep:
150 if dep1_lib and not dep2_lib:
151 newDep = dep1
152 selectedDep = True
153 elif not dep1_lib and dep2_lib:
154 newDep = dep2
155 selectedDep = True
156
157 assert(selectedDep)
158
159 #print("\n newDep =", newDep)
160
161 return newDep
162
163
164 class DepStats:
165 isDirect = None
166 isRequired = None
167 isTestDepChain = None
168 def __init__(self, isDirect, isRequired, isTestDepChain):
169 self.isDirect = isDirect
170 self.isRequired = isRequired
171 self.isTestDepChain = isTestDepChain
172
173
174 class TribitsDependencies:
175
176
177 def __init__(self):
178 self.__projectName = None
179 self.__projectBaseDirName = None
180 self.__packagesList = []
181 self.__packagesNameToID = {}
182 self.__packagesDirToID = {}
183
184
185 def setProjectName(self, projectName):
186 self.__projectName = projectName
187
188
189 def getProjectName(self):
190 return self.__projectName
191
192
193 def setProjectBaseDirName(self, projectBaseDirName):
194 self.__projectBaseDirName = projectBaseDirName
195
196
197 def getProjectBaseDirName(self):
198 return self.__projectBaseDirName
199
200
201 def addPackageDependencies(self, packageDeps):
202 packageName = packageDeps.packageName
203 packageDir = packageDeps.packageDir
204 self.__packagesList.append(packageDeps)
205 packageDeps.packageID = len(self.__packagesList)-1
206 self.__packagesNameToID.update( { packageName : packageDeps.packageID } )
207 self.__packagesDirToID.update( { packageDir : packageDeps.packageID } )
208
209
210 def numPackages(self):
211 return len(self.__packagesList)
212
213
214 def getPackagesNamesList(self, onlyTopLevelPackages=True):
215 packagesNamesList = []
216 for packageDep in self.__packagesList:
217 #print ("packageDep.packageName = "+packageDep.packageName)
218 #print ("packageDep.parentPackage = "+packageDep.parentPackage)
219 if packageDep.parentPackage == "":
220 addPackage = True
221 elif not onlyTopLevelPackages:
222 addPackage = True
223 else:
224 addPackage = False
225 if addPackage:
226 packagesNamesList.append(packageDep.packageName)
227 return packagesNamesList
228
229
230 def packageNameToID(self, packageName):
231 return self.__packagesNameToID.get(packageName, -1)
232
233
234 def getPackageByID(self, packageID):
235 return self.__packagesList[packageID]
236
237
238 def getPackageByName(self, packageName):
239 return self.getPackageByID(self.__packagesNameToID[packageName])
240
241
242 def getPackageByDir(self, packageDir):
243 packageID = self.__packagesDirToID.get(packageDir, -1)
244 #print("\ngetPackageByDir: packageDir="+packageDir+", packageID="+str(packageID))
245 if packageID >= 0:
246 return self.__packagesList[packageID]
247 return None
248
249
250 # Note: Path must contain ending "/"
251 def getPackageNameFromPath(self, fullPath):
252 for packageDep in self.__packagesList:
253 regexFilePath = packageDep.packageDir+"/"
254 #print("\nregexFilePath="+regexFilePath)
255 #print("fullPath="+fullPath)
256 if re.match(regexFilePath, fullPath):
257 #print("MATCH!")
258 return packageDep.packageName
259 return u""
260 # NOTE: The above loop with match subpackages before it matches
261 # packages because subpackages are listed before packages!
262
263
264 # Returns the paraent package name given a test name
265 def getPackageNameFromTestName(self, testName):
266 for packageDep in self.__packagesList:
267 startTestName = packageDep.packageName+"_"
268 #print("\nstartTestName="+startTestName)
269 testNameStartIdx = testName.find(startTestName, 0)
270 if testNameStartIdx == 0:
271 #print("MATCH!")
272 if packageDep.parentPackage:
273 #print("Subpackage match!")
274 return self.getPackageByName(packageDep.parentPackage).packageName
275 # Else, is not a subpackage
276 return packageDep.packageName
277 return u""
278
279
280 def filterPackageNameList(self, inputPackagesList, keepTypesList, verbose=False):
281 if len(inputPackagesList)==1 and inputPackagesList[0]=='':
282 return []
283 i = 0
284 outputPackagesList = []
285 for packageName in inputPackagesList:
286 #print("packageName = " + packageName)
287 if packageName == "ALL_PACKAGES":
288 outputPackagesList.append(packageName)
289 continue
290 packageDep = self.getPackageByName(packageName)
291 packageType = packageDep.packageType
292 #print("packageType = " + packageType)
293 if findInSequence(keepTypesList, packageType) >= 0:
294 outputPackagesList.append(packageName)
295 else:
296 if verbose:
297 print(packageName + " of type " + packageType +
298 " is being excluded because it is not in the valid list of " +
299 "package types [" + ','.join(keepTypesList) + "]")
300 return outputPackagesList
301
302 def __str__(self):
303 strRep = ""
304 for packageDep in self.__packagesList:
305 strRep += str(packageDep)
306 return strRep
307
308
309 def updateDepCell(self, packageRow, packageID, depStats, depCategoryName):
310
311 currentDepName = packageRow[packageID+1]
312
313 newDepName = depCategoryName
314
315 # If we are in a test dependency chain, we must change library
316 # dependencies to test dependencies.
317 if depStats.isTestDepChain:
318 newDepName = 'T'+newDepName[1:]
319
320 if depStats.isDirect:
321 newDepName = newDepName
322 else:
323 newDepName = "I"+newDepName
324
325 if not depStats.isRequired:
326 newDepName = newDepName[0:-1]+"O"
327
328 if currentDepName:
329 #print("\n updateDepCell: depStats.isDirect="+str(depStats.isDirect)+", depStats.isRequired="+str(depStats.isRequired)+", depCategoryName="+depCategoryName)
330 newDepName = updatePackageDep(currentDepName, newDepName)
331
332 packageRow[packageID+1] = newDepName
333
334
335 def updatePackageDepsCategory(self, libsOnly, packageRowID, packageID, depCategory,
336 depCategoryName, depStats, projectDepsTable
337 ):
338
339 packageRow = projectDepsTable[packageRowID+1]
340 #print("\npackageRow =", packageRow)
341
342 depList = getattr(self.__packagesList[packageID], depCategory)
343 #print("\ndepList =", depList)
344
345 for dep in depList:
346
347 depPackage = self.getPackageByName(dep)
348 #print("\n depPackageName =", depPackage.packageName)
349
350 dep_i = depPackage.packageID
351
352 self.updateDepCell(packageRow, dep_i, depStats, depCategoryName)
353
354 if not depStats.isRequired:
355 isRequiredDep = False
356 elif depCategoryName[-1]=="R":
357 isRequiredDep = True
358 else:
359 isRequiredDep = False
360
361 childDepStats = DepStats(False, isRequiredDep, depStats.isTestDepChain)
362
363 self.updatePackageDeps(libsOnly, packageRowID, dep_i, childDepStats,
364 projectDepsTable)
365
366
367 def updatePackageDeps(self, libsOnly, packageRowID, packageID, depStats,
368 projectDepsTable
369 ):
370
371 self.updatePackageDepsCategory(libsOnly, packageRowID, packageID,
372 "libRequiredDepPackages", "LR", depStats, projectDepsTable)
373 self.updatePackageDepsCategory(libsOnly, packageRowID, packageID,
374 "libOptionalDepPackages", "LO", depStats, projectDepsTable)
375
376 # Only process the test dependencies if we are asked to do so
377 # (i.e. libsOnly=True) or if this is the top-level package. The tests for
378 # dependent packages are not any kind of dependency for tests for the
379 # top-level package. However, we need to record that these are test
380 # dependencies so that any package libraries that get recursed are
381 # recorded as 'ITR' or 'ITO' and not as library dependencies.
382 if not libsOnly and depStats.isDirect:
383 libDepStats = DepStats(True, depStats.isRequired, True)
384 self.updatePackageDepsCategory(False, packageRowID, packageID,
385 "testRequiredDepPackages", "TR", libDepStats, projectDepsTable)
386 self.updatePackageDepsCategory(False, packageRowID, packageID,
387 "testOptionalDepPackages", "TO", libDepStats, projectDepsTable)
388
389
390 def createRawTable(self, libsOnly):
391
392 numPackages = self.numPackages()
393 #print("\nnumPackages =", numPackages)
394
395 projectDepsTable = []
396
397 topRow = [ "Packages" ]
398 topRow.extend(["P%02d"%(i+1) for i in range(numPackages)] )
399 projectDepsTable.append(topRow)
400
401 for packageDeps in self.__packagesList:
402 i = packageDeps.packageID
403 row = ["P%02d"%(i+1)+") "+packageDeps.packageName]
404 row.extend(["" for i in range(numPackages)])
405 projectDepsTable.append(row)
406
407 for packageDeps in self.__packagesList:
408 #print("\npackageName =", packageDeps.packageName)
409 i = packageDeps.packageID
410 projectDepsTable[i+1][i+1] = "X"
411 self.updatePackageDeps(libsOnly, i, i, DepStats(True, True, False), projectDepsTable)
412
413 return projectDepsTable
414
415 def createProjectPackagesNumberedList(self):
416 numPackages = self.numPackages()
417 htmlText = "<p><b>Packages:</b> " + \
418 ", ".join( \
419 [ "P%02d"%(i+1)+") "+self.__packagesList[i].packageName \
420 for i in range(self.numPackages())] \
421 ) + \
422 "</p>"
423 return htmlText
424
425 def createHtmlFromTable(self, rawTable):
426
427 numPackages = self.numPackages()
428
429 htmlText = \
430 "<TABLE BORDER=4>\n"+\
431 "\n"
432
433 for i in range(numPackages+2):
434 htmlText += "<COL ALIGN=LEFT>\n"
435
436 topRow = rawTable[0]
437 htmlText += "\n<TR>\n"
438 for j in range(numPackages+1):
439 htmlText += " <TD><b>"+topRow[j]+"</b></TD>\n"
440 htmlText += " <TD><b>Packages</b></TD>\n"
441 htmlText += "</TR>\n"
442
443 for package_i in range(numPackages):
444 row = rawTable[package_i+1]
445 htmlText += "\n<TR>\n"
446 htmlText += " <TD><b>"+row[0]+"</b></TD>\n"
447 for j in range(numPackages):
448 entry = row[j+1]
449 if not entry: entry = "."
450 htmlText += " <TD>"+entry+"</TD>\n"
451 htmlText += " <TD><b>"+row[0]+"</b></TD>\n"
452 htmlText += "</TR>\n"
453
454 htmlText += "\n<TR>\n"
455 htmlText += " <TD><b>Packages</b></TD>\n"
456 for j in range(numPackages):
457 htmlText += " <TD><b>P%02d"%(j+1)+"</b></TD>\n"
458 htmlText += " <TD><b>Packages</b></TD>\n"
459 htmlText += "</TR>\n"
460
461 htmlText += "</TABLE>\n"
462
463 return htmlText
464
465
466 def createHtmlTableLegend(self, libsOnly):
467
468 htmlText =\
469 "\n"+\
470 "<ul>\n"+\
471 "<li> <b>X</b>: Diagonal entry for the package itself\n"+\
472 "<li> <b>LR</b>: Direct library required dependency\n"+\
473 "<li> <b>ILR</b>: Indirect library required dependency\n"+\
474 "<li> <b>LO</b>: Direct library optional dependency\n"+\
475 "<li> <b>ILO</b>: Indirect library optional dependency\n"
476
477 if not libsOnly:
478 htmlText +=\
479 "<li> <b>TR</b>: Direct test/example required dependency\n"+\
480 "<li> <b>ITR</b>: Indirect test/example required dependency\n"+\
481 "<li> <b>TO</b>: Direct test/example optional dependency\n"+\
482 "<li> <b>ITO</b>: Indirect test/example optional dependency\n"
483
484 htmlText +=\
485 "</ul>\n"+\
486 "\n"+\
487 "NOTE: When more than one type of dependency is present for any cell"+\
488 " the final selection is determined in the following order:\n"+\
489 "<ul>\n"+\
490 "<li> A required dependency trumps an optional dependency\n"+\
491 "<li> A direct dependency trumps an indirect dependency\n"+\
492 "<li> A library dependency trumps a test/example dependency\n"+\
493 "</ul>\n"
494
495 return htmlText
496
497
498 def createFullHtmlForTables(self):
499
500 packagesListHtml = self.createProjectPackagesNumberedList()
501
502 htmlText = \
503 "<p><huge><b>"+self.getProjectName()+" Test/Example and Library Package Dependencies</b></huge></p>\n"+\
504 "\n"+\
505 self.createHtmlFromTable(self.createRawTable(False))+\
506 "\n"+\
507 packagesListHtml+"\n"+\
508 "\n"+\
509 "<p><b>Legend</b></p>\n"+\
510 "\n"+\
511 self.createHtmlTableLegend(False)+\
512 "\n"+\
513 "<p><b><huge>"+self.getProjectName()+" Libary-Only Package Dependencies</huge></b></p>\n"+\
514 "\n"+\
515 self.createHtmlFromTable(self.createRawTable(True))+\
516 "\n"+\
517 packagesListHtml+"\n"+\
518 "\n"+\
519 "<p><b>Legend</b></p>\n"+\
520 "\n"+\
521 self.createHtmlTableLegend(True)
522
523 return htmlText
524
525 def createFullHtmlPage(self):
526
527 htmlText = \
528 "<html>\n"+\
529 "<head>\n"+\
530 "<title>"+self.getProjectName()+" Package Dependencies</title>\n"+\
531 "</head>\n"+\
532 "\n"+\
533 "<body>\n"+\
534 "\n"+\
535 self.createFullHtmlForTables()+\
536 "\n"+\
537 "</body>\n"+\
538 "\n"+\
539 "</html>\n"
540
541 return htmlText
542
543
544 def writeFullHtmlPage(self, htmlFileName):
545 htmlString = self.createFullHtmlPage()
546 htmlFile = open(htmlFileName, 'w')
547 htmlFile.write(htmlString)
548 htmlFile.close()
549
550
551 #
552 # CDash stuff
553 #
554
555
556 def createCDashXML(self):
557
558 xmlText = ""
559
560 xmlText += "<Project name=\""+self.getProjectName()+"\">\n"
561
562 projectBaseDirName = self.getProjectBaseDirName()
563
564 numPackages = self.numPackages()
565
566 for package_i in range(numPackages):
567
568 packageDeps = self.__packagesList[package_i]
569
570 packageName = packageDeps.packageName
571 packagePath = packageDeps.packageDir
572
573 if packageDeps.parentPackage == "":
574
575 xmlText += (" <SubProject name=\""+packageName+"\">\n")
576
577 xmlText += (" <Path>"+packagePath+"</Path>\n")
578
579 xmlText += \
580 " <EmailAddresses>\n"+\
581 " <Email address=\""+packageDeps.emailAddresses.regression+"\"/>\n"+\
582 " </EmailAddresses>\n"
583
584 xmlText += (" </SubProject>\n")
585
586 else:
587
588 # We don't want to bother CDash with subpackages. We want CDash testing
589 # to operate on the package level!
590 None
591
592 # end if
593
594 # end for
595
596 xmlText += "</Project>\n"
597
598 return xmlText
599
600
601 def writeCDashXmlDepsFile(self, xmlDepsFile):
602 xmlString = self.createCDashXML()
603 xmlFile = open(xmlDepsFile, 'w')
604 xmlFile.write(xmlString)
605 xmlFile.close()
606
607
608 #
609 # Read in the dependencies from XML
610 #
611
612
613 def getDependenciesByType(packageEle, typeName):
614 packageDepsStr = packageEle.getElementsByTagName(typeName)[0].getAttribute('value');
615 if len(packageDepsStr) == 0:
616 return []
617 return packageDepsStr.split(',')
618
619
620 def getSingleEmailAddress(emailEle, emailType):
621 singleEmailEle = emailEle.getElementsByTagName(emailType)[0]
622 singleEmailAddress = singleEmailEle.getAttribute('address');
623 return singleEmailAddress
624
625
626 def getPackageEmailAddresses(packageEle):
627 emailEle = packageEle.getElementsByTagName("EmailAddresses")[0]
628 regressionEmail = getSingleEmailAddress(emailEle, "Regression")
629 return PackageEmailAddresses(regressionEmail)
630
631
632 def getParentPackage(packageEle):
633 parentPackageEle = packageEle.getElementsByTagName("ParentPackage")[0]
634 parentPackage = parentPackageEle.getAttribute('value');
635 return parentPackage
636
637
638 def getProjectDependenciesFromXmlFile(xmlFile):
639 packageDepXmlDom = xml.dom.minidom.parse(xmlFile)
640 #print("\npackageDepXmlDom =", dir(packageDepXmlDom))
641 #print("\npackageDepXmlDom.documentElement =", dir(packageDepXmlDom.documentElement))
642 projectDependencies = TribitsDependencies()
643 projectDependencies.setProjectName(
644 packageDepXmlDom.documentElement.getAttribute('project'))
645 projectDependencies.setProjectBaseDirName(
646 packageDepXmlDom.documentElement.getAttribute('baseDirName'))
647 for ele in packageDepXmlDom.childNodes[0].childNodes:
648 if ele.nodeType == ele.ELEMENT_NODE:
649 packageName = ele.getAttribute('name')
650 packageDir = ele.getAttribute('dir')
651 packageType = ele.getAttribute('type')
652 #print("\npackageName =", packageName)
653 packageDeps = PackageDependencies(packageName, packageDir, packageType,
654 getDependenciesByType(ele, "LIB_REQUIRED_DEP_PACKAGES"),
655 getDependenciesByType(ele, "LIB_OPTIONAL_DEP_PACKAGES"),
656 getDependenciesByType(ele, "TEST_REQUIRED_DEP_PACKAGES"),
657 getDependenciesByType(ele, "TEST_OPTIONAL_DEP_PACKAGES"),
658 getPackageEmailAddresses(ele),
659 getParentPackage(ele)
660 )
661 #print("\npackageDeps =", str(packageDeps))
662 projectDependencies.addPackageDependencies(packageDeps)
663 return projectDependencies