Debian Codegraph @

cmake/tribits/ci_support/clone_extra_repos.py

raw
1 #!/usr/bin/env python3
2
3 # @HEADER
4 # *****************************************************************************
5 # TriBITS: Tribal Build, Integrate, and Test System
6 #
7 # Copyright 2013-2016 NTESS and the TriBITS contributors.
8 # SPDX-License-Identifier: BSD-3-Clause
9 # *****************************************************************************
10 # @HEADER
11
12 from CheckinTestConstants import *
13 from FindGeneralScriptSupport import *
14 from GeneralScriptSupport import *
15 from gitdist import addOptionParserChoiceOption
16 import gitdist
17
18
19 #
20 # Constants
21 #
22
23 g_extraReposTypes = g_knownTribitsTestRepoTypes
24 g_extraReposTypeDefault = "Nightly"
25 g_extraReposTypesDefaulIdx = findInSequence(g_extraReposTypes, g_extraReposTypeDefault)
26
27 g_extraRerposFileDefault = "cmake/ExtraRepositoriesList.cmake"
28
29 g_verbosityLevels = [ "none", "minimal", "more", "most" ]
30 g_verbosityLevelDefault = "more"
31 g_verbosityLevelDefaultIdx = findInSequence(g_verbosityLevels, g_verbosityLevelDefault)
32
33
34 #
35 # Help message
36 #
37
38 # This part can be reused in other scripts that are project-specific
39 genericUsageHelp = \
40 r"""
41 By default, this will clone all the 'Nightly' extra repos that are listed in
42 the file:
43
44 <projectDir>/"""+g_extraRerposFileDefault+r"""
45
46 (other repo types can be selected using --extra-repos-type).
47
48 The list of which repos to clone can be "white-list" selected with the option
49 --extra-repos (see options below for details). Extra repos can in addition be
50 "back-listed" using the option --not-extra-repos.
51
52 To see the full list of repos that can be cloned, pass in just:
53
54 --skip-clone --verbosity=more
55
56 That will print out a table like:
57
58 ------------------------------------------------------------------------------
59 | ID | Repo Name | Repo Dir | VC | Repo URL | Category |
60 |----|------------|------------|-----|--------------------------|------------|
61 | 1 | ExtraRepo1 | ExtraRepo1 | GIT | someurl.com.ExtraRepo1 | Continuous |
62 | 2 | ExtraRepo3 | ExtraRepo3 | GIT | someurl3.com:/ExtraRepo3 | Continuous |
63 ------------------------------------------------------------------------------
64
65 If the git repo server is using gitolite, one can set
66 --gitolite-root=<gitolite-root> and that will result in git repos being
67 selected only if the selected repos are listed in 'ssh <gitolite-root> info'.
68 This allows one to automatically exclude repos from being cloned that the user
69 has no permissions to clone. NOTE: See warning about the --gitolite-root option below!
70
71 TIP: After cloning the set of repos, a nice way to interact with the repos is
72 to use the tool 'gitdist'. If your project does not have a version controlled
73 .gitdist.default file, you can generate one using the
74 --create-gitdist-file=<gitdist-file> argument, for example with:
75
76 --create-gitdist-file=.gitdist
77
78 This will restrict the list of repos processed by gitdist to just the repos cloned.
79 """
80
81
82 usageHelp = r"""clone_extra_repos.py [options]
83
84 This script clones one more extra repos listed in a TriBITS
85 ExtraRepositoriesList.cmake file. The standard usage is:
86
87 $ cd base <projectDir>
88 $ ./cmake/tribits/ci_support/clone-extra-repos.py
89
90 where <projectDir> is the base TriBITS project dir and base git repo.
91 """ + \
92 genericUsageHelp
93
94
95 #
96 # Helper functions
97 #
98
99
100 def injectCmndLineOptionsInParser(clp, gitoliteRootDefault=""):
101
102 clp.add_option(
103 "--extra-repos", dest="extraRepos", type="string", default="",
104 help="List of names of extra repos to be cloned <extra-repos>" \
105 " (i.e. \"repo0,repo1,,...\"). When set to empty '' (the default value)" \
106 " then all repos that match <extra-repos-type> listed in <extra-repos-file>" \
107 " will be selected. But the repos listed in <extra-repos> must always" \
108 " be a subset of the repos of type <extra-repos-type> selected from" \
109 " <extra-repos-file>. (Default '')" )
110
111 clp.add_option(
112 "--not-extra-repos", dest="notExtraRepos", type="string", default="",
113 help="List of names of extra repos *NOT* to clone (i.e. \"repo0,repo1,...\")." \
114 " (Default '')" )
115
116 clp.add_option(
117 "--extra-repos-file", dest="extraReposFile", type="string",
118 default=g_extraRerposFileDefault,
119 help="The file path <extra-repos-file> for the ExtraRepositoriesList.cmake file." \
120 " This can be an absolute or relative path." \
121 " (Default = '"+g_extraRerposFileDefault+"')")
122
123 addOptionParserChoiceOption(
124 "--extra-repos-type", "extraReposType",
125 g_extraReposTypes , g_extraReposTypesDefaulIdx,
126 "Type of extra repositories <extra-repos-type> to select from " \
127 "<extra-repos-file>. When --extra-repos is set, then this argument" \
128 " is ignored.",
129 clp )
130
131 clp.add_option(
132 "--gitolite-root", dest="gitoliteRoot", type="string", default=gitoliteRootDefault,
133 help="Gives the root for a gitolite repos <gitolite-root> (e.g. git@<some-url>)." \
134 " If specified, then any git repos with the <gitolite-root> listed as their" \
135 " root will only be selected if they are listed with 'R' permissions returned" \
136 " from 'ssh <gitolite-root> info'. WARNING: Make sure that you have your" \
137 " gitoliote SSH registered correctly before using this option by typing" \
138 " the command 'ssh <gitlite-root> info' and make sure that it does *not*"
139 " ask for a password! (Default = '"+gitoliteRootDefault+"')" )
140
141 clp.add_option(
142 "--with-cmake", dest="withCmake", type="string", default="cmake",
143 help="CMake executable to use with cmake -P scripts internally (only set" \
144 +" by unit testing code). (Default = 'cmake')")
145
146 addOptionParserChoiceOption(
147 "--verbosity", "verbLevel", g_verbosityLevels, g_verbosityLevelDefaultIdx,
148 "Verbosity of the script (levels are cumulative):" \
149 " none = no output at all (except for commands with --no-op). " \
150 " minimal = print script args echo and clone commands." \
151 " more = print basic repo include/exclude logic and print repo table." \
152 " most = print output from cmake script called, the output from gitolite," \
153 " and other detailed info." \
154 ,
155 clp )
156
157 clp.add_option(
158 "--do-clone", dest="doClone", action="store_true",
159 help="Do the clone of the selected repos. [default]")
160 clp.add_option(
161 "--skip-clone", dest="doClone", action="store_false",
162 help="Skip the clone of the repos and just show what would be done.",
163 default=True )
164
165 clp.add_option(
166 "--do-op", dest="doOp", action="store_true",
167 help="Do the clone of the selected repos. [default]" )
168 clp.add_option(
169 "--no-op", dest="doOp", action="store_false",
170 help="Skip cloning the repos and just show the clone commands.",
171 default=True )
172
173 clp.add_option(
174 "--create-gitdist-file", dest="createGitdistFile", type="string", default="",
175 help="If specified, the file <gitdist-file> will get generated with the list" \
176 " of git repos (the same list that is cloned with --do-clone)." \
177 " (Default = '')")
178
179 clp.add_option(
180 "--show-defaults", dest="showDefaults", action="store_true",
181 help="Show the default option values and do nothing at all.",
182 default=False )
183
184
185 def getCmndLineOptions():
186 from optparse import OptionParser
187 clp = OptionParser(usage=usageHelp)
188 injectCmndLineOptionsInParser(clp)
189 (options, args) = clp.parse_args()
190 return options
191
192
193 def isVerbosityLevel(inOptions, testVerbLevel):
194 requestedVerbLevelInt = findInSequence(g_verbosityLevels, inOptions.verbLevel)
195 testVerbLevelInt = findInSequence(g_verbosityLevels, testVerbLevel)
196 if testVerbLevelInt <= requestedVerbLevelInt:
197 return True
198 return False
199
200
201 def fwdCmndLineOptions(inOptions, terminator=""):
202 cmndLineOpts = \
203 " --extra-repos='"+inOptions.extraRepos+"'"+terminator + \
204 " --not-extra-repos='"+inOptions.notExtraRepos+"'"+terminator + \
205 " --extra-repos-file='"+inOptions.extraReposFile+"'"+terminator + \
206 " --extra-repos-type='"+inOptions.extraReposType+"'"+terminator + \
207 " --gitolite-root='"+inOptions.gitoliteRoot+"'"+terminator + \
208 " --with-cmake='"+inOptions.withCmake+"'"+terminator + \
209 " --verbosity='"+inOptions.verbLevel+"'"+terminator
210 if inOptions.doClone:
211 cmndLineOpts += " --do-clone" + terminator
212 else:
213 cmndLineOpts +=" --skip-clone" + terminator
214 if inOptions.doOp:
215 cmndLineOpts += " --do-op" + terminator
216 else:
217 cmndLineOpts += " --no-op" + terminator
218 cmndLineOpts += \
219 " --create-gitdist-file='"+inOptions.createGitdistFile+"'"+terminator
220 return cmndLineOpts
221
222
223 def echoCmndLineOptions(inOptions):
224 print(fwdCmndLineOptions(inOptions, " \\\n"))
225
226
227 def echoCmndLine(inOptions):
228
229 print("")
230 print("**************************************************************************")
231 print("Script: clone_extra_repos.py \\")
232
233 echoCmndLineOptions(inOptions)
234
235
236 def getHeaderOutputAndExtraReposDictList(rawOutputFromCmakefile):
237 rawOutputFromCmakefile = s(rawOutputFromCmakefile)
238 headerOuput = ""
239 pythonDictListStr = ""
240 processingPythonDict = False
241 for line in rawOutputFromCmakefile.splitlines():
242 if line == "*** Extra Repositories Python Dictionary":
243 processingPythonDict=True
244 continue
245 if processingPythonDict:
246 pythonDictListStr += (line + "\n")
247 else:
248 headerOuput += (line + "\n")
249 #print("\nheaderOuput:\n\n", headerOuput)
250 #print("\npythonDictListStr = '"+pythonDictListStr+"'")
251 pythonDictList = eval(pythonDictListStr)
252 return (headerOuput, pythonDictList)
253
254
255 def getExtraReposDictListFromCmakefile(projectDir, extraReposFile, withCmake,
256 extraReposType=g_extraReposTypeDefault, extraRepos="",
257 tribitsDir=tribitsDir, verbose=True,
258 ):
259 cmnd = "\""+withCmake+"\""+ \
260 " -DPROJECT_SOURCE_DIR="+projectDir+ \
261 " -DTRIBITS_BASE_DIR="+tribitsDir+ \
262 " -DEXTRA_REPOS_FILE="+extraReposFile+ \
263 " -DENABLE_KNOWN_EXTERNAL_REPOS_TYPE="+extraReposType+ \
264 " -DEXTRA_REPOS="+extraRepos+\
265 " -DCHECK_EXTRAREPOS_EXIST=FALSE"
266 # " -DTRIBITS_PROCESS_EXTRAREPOS_LISTS_DEBUG=TRUE"
267 cmnd += \
268 " -P "+ciSupportDir+"/TribitsGetExtraReposForCheckinTest.cmake"
269 rawOutput = getCmndOutput(cmnd, throwOnError=True, getStdErr=True)
270 (headerOutput, extraReposPytonDictList) = getHeaderOutputAndExtraReposDictList(rawOutput)
271 if verbose:
272 print("\n" + headerOutput)
273 return extraReposPytonDictList
274
275
276 # Parse raw 'ssh <gitolite-root> info' output and return list of gitolite
277 # repos.
278 def parseRawSshGitoliteRootInfoOutput(rawSshGitoliteRootInfoOutput, verbose=False):
279
280 rawSshGitoliteRootInfoOutputList = rawSshGitoliteRootInfoOutput.splitlines()
281
282 gitoliteSshHeader = rawSshGitoliteRootInfoOutputList[0]
283 if verbose:
284 print(gitoliteSshHeader)
285
286 gitolioteReposList = []
287 parsingRepos = False
288 for line in rawSshGitoliteRootInfoOutputList:
289 #print("line = '"+line+"'")
290 # Look for the first blank line and then the next line will be the first
291 # listed repo.
292 if line == "":
293 parsingRepos = True
294 continue
295 # Parse the repos
296 if parsingRepos:
297 # The permissions and the repo name are split by a tab such as:
298 # R W ExtraRepo1
299 repoSplit = line.split("\t")
300 #print("repoSplit =", repoSplit)
301 if len(repoSplit) == 2:
302 gitolioteReposList.append(repoSplit[1])
303 return gitolioteReposList
304
305
306 # Get the list of gitolite repos from 'ssh <gitolite-root> info'.
307 def getGitoliteReposList(inOptions, trace=False, dumpGitoliteOutput=False):
308 cmnd = "ssh "+inOptions.gitoliteRoot+" info"
309 if trace:
310 "Running: "+cmnd
311 rawSshGitoliteRootInfoOutput = getCmndOutput(cmnd)
312 if dumpGitoliteOutput:
313 print(rawSshGitoliteRootInfoOutput)
314 if (trace and not dumpGitoliteOutput): verbose=True
315 else: verbose=False
316 return parseRawSshGitoliteRootInfoOutput(rawSshGitoliteRootInfoOutput, verbose=verbose)
317
318
319 def filterOutNotExtraRepos(extraRepoDictList_in, notExtraRepos, verbose=False):
320 notExtraReposSet = set(notExtraRepos)
321 extraRepoDictList = []
322 for extraReposDict in extraRepoDictList_in:
323 extraRepoName = extraReposDict["NAME"]
324 if extraRepoName in notExtraReposSet:
325 if verbose:
326 print("Excluding extra repo '"+extraRepoName+"'!")
327 else:
328 extraRepoDictList.append(extraReposDict)
329 return extraRepoDictList
330
331
332 def filterOutMissingGitoliteRepos(extraRepoDictList_in,
333 gitoliteRepos, verbose=False):
334 gitoliteReposSet = set(gitoliteRepos)
335 extraRepoDictList = []
336 for extraReposDict in extraRepoDictList_in:
337 extraRepoName = extraReposDict["NAME"]
338 if not extraRepoName in gitoliteReposSet:
339 if verbose:
340 print("Excluding extra repo '" + extraRepoName +
341 "' not listed by gitolite!")
342 else:
343 extraRepoDictList.append(extraReposDict)
344 return extraRepoDictList
345
346
347 def getExtraReposTable(extraRepoDictList):
348
349 # Get the lists for each column in the table
350 repoIdList = []
351 repoNameList = []
352 repoDirList = []
353 repoVcTypeList = []
354 repoUrlList = []
355 repoCategoryList = []
356 repoId = 1 # Use one-base indexing to match gitdist IDs!
357 for extraRepoDict in extraRepoDictList:
358 repoIdList.append(str(repoId))
359 repoNameList.append(extraRepoDict["NAME"])
360 repoDirList.append(extraRepoDict["DIR"])
361 repoVcTypeList.append(extraRepoDict["REPOTYPE"])
362 repoUrlList.append(extraRepoDict["REPOURL"])
363 repoCategoryList.append(extraRepoDict["CATEGORY"])
364 repoId += 1
365
366 # Create the table
367 extraReposTableDictList = [
368 { "label":"ID", "align":"R", "fields":repoIdList },
369 { "label":"Repo Name", "align":"L", "fields":repoNameList },
370 { "label":"Repo Dir", "align":"L", "fields":repoDirList },
371 { "label":"VC", "align":"L", "fields":repoVcTypeList },
372 { "label":"Repo URL", "align":"L", "fields":repoUrlList },
373 { "label":"Category", "align":"L", "fields":repoCategoryList },
374 ]
375 #print("extraReposTableDictList =", extraReposTableDictList)
376
377 extraReposTable = gitdist.createTable(extraReposTableDictList)
378
379 # Return the table
380 return extraReposTable
381
382
383 def createGitDistFile(extraRepoDictList, gitdistFile, verbose=False):
384 gitdistFileStr = ""
385 for extraRepoDict in extraRepoDictList:
386 gitdistFileStr += extraRepoDict["DIR"]+"\n"
387 if verbose:
388 print("Writing the file '" + gitdistFile + "' ...")
389 open(gitdistFile, 'w').write(gitdistFileStr)
390
391
392 def cloneExtraRepo(inOptions, extraRepoDict):
393 repoName = extraRepoDict["NAME"]
394 repoDir = extraRepoDict["DIR"]
395 repoUrl = extraRepoDict["REPOURL"]
396 repoVcType = extraRepoDict["REPOTYPE"]
397 verbLevelIsMinimum = isVerbosityLevel(inOptions, "minimal")
398 if verbLevelIsMinimum:
399 print("\nCloning repo " + repoName + " ...")
400 if os.path.exists(repoDir):
401 if verbLevelIsMinimum:
402 print("\n ==> Repo dir = '" + repoDir +
403 "' already exists. Skipping clone!")
404 return
405 if repoVcType != "GIT":
406 print("\n ==> ERROR: Repo type '"+repoVcType+"' not supported!")
407 sys.exit(1)
408 cmnd = "git clone "+repoUrl+" "+repoDir
409 if inOptions.doOp:
410 echoRunSysCmnd(cmnd, timeCmnd=True, verbose=verbLevelIsMinimum)
411 elif verbLevelIsMinimum:
412 print("\nRunning: " + cmnd)
413
414
415 def cloneExtraRepos(inOptions):
416
417 verbLevelIsMinimal = isVerbosityLevel(inOptions, "minimal")
418
419 #
420 # A) Get the list of extra repos
421 #
422
423 extraRepoDictList = getExtraReposDictListFromCmakefile(
424 projectDir=os.getcwd(),
425 extraReposFile=inOptions.extraReposFile,
426 extraReposType=inOptions.extraReposType,
427 extraRepos=inOptions.extraRepos,
428 withCmake=inOptions.withCmake,
429 verbose=isVerbosityLevel(inOptions, "most")
430 )
431 #print("extraRepoDictList =" + str(extraRepoDictList))
432
433 #
434 # B) Get the list of gitolite repos
435 #
436
437 if inOptions.gitoliteRoot:
438 if verbLevelIsMinimal:
439 print("\n***")
440 print("*** Get the list of repos that can be cloned from gitolite server:")
441 print("***\n")
442 gitoliteRepos = getGitoliteReposList(inOptions,
443 trace=verbLevelIsMinimal,
444 dumpGitoliteOutput=isVerbosityLevel(inOptions, "most"))
445 else:
446 gitoliteRepos = []
447 #print("gitoliteRepos =", gitoliteRepos)
448
449 #
450 # C) Filter the list of extra repos
451 #
452
453 if gitoliteRepos:
454 if verbLevelIsMinimal:
455 print("\n***")
456 print("*** Filtering the set of extra repos based on gitolite repos:")
457 print("***\n")
458 extraRepoDictList = filterOutMissingGitoliteRepos(
459 extraRepoDictList, gitoliteRepos, verbose=verbLevelIsMinimal)
460
461 if inOptions.notExtraRepos:
462 if verbLevelIsMinimal:
463 print("\n***")
464 print("*** Filtering the set of extra repos based on --not-extra-repos:")
465 print("***\n")
466 extraRepoDictList = filterOutNotExtraRepos(
467 extraRepoDictList,
468 inOptions.notExtraRepos.split(","),
469 verbose=verbLevelIsMinimal)
470
471 #
472 # D) print out table of repos
473 #
474
475 if isVerbosityLevel(inOptions, "more"):
476
477 print("\n***")
478 print("*** List of selected extra repos to clone:")
479 print("***\n")
480
481 extraReposTable = getExtraReposTable(extraRepoDictList)
482 print(extraReposTable)
483
484 #
485 # E) Clone the repos
486 #
487
488 if inOptions.doClone:
489
490 if verbLevelIsMinimal:
491 print("\n***")
492 print("*** Clone the selected extra repos:")
493 print("***\n")
494
495 for extraRepoDict in extraRepoDictList:
496 cloneExtraRepo(inOptions, extraRepoDict)
497
498 #
499 # F) Create the gitdist file
500 #
501
502 if inOptions.createGitdistFile:
503
504 if verbLevelIsMinimal:
505 print("\n***")
506 print("*** Create the gitdist file:")
507 print("***\n")
508
509 createGitDistFile(extraRepoDictList, inOptions.createGitdistFile,
510 verbose=verbLevelIsMinimal)
511
512
513 #
514 # Run the script
515 #
516
517 if __name__ == '__main__':
518
519 inOptions = getCmndLineOptions()
520 if isVerbosityLevel(inOptions, "minimal"):
521 echoCmndLine(inOptions)
522 if inOptions.showDefaults:
523 sys.exit(0)
524
525 cloneExtraRepos(inOptions)