Debian Codegraph @

cmake/tribits/python_utils/extract_rst_cmake_doc.py

raw
1 #!/usr/bin/env python3
2
3 # @HEADER
4 # *****************************************************************************
5 # TriBITS: Tribal Build, Integrate, and Test System
6 #
7 # Copyright 2013-2016 NTESS and the TriBITS contributors.
8 # SPDX-License-Identifier: BSD-3-Clause
9 # *****************************************************************************
10 # @HEADER
11
12 import sys
13 import os
14 import subprocess
15 import re
16 import pprint
17 import glob
18
19
20 pp = pprint.PrettyPrinter(indent=4)
21
22
23 #
24 # Script documentation
25 #
26
27
28 usageHelp = r"""extract_rst_cmake_doc.py [OPTIONS]
29
30 This script implements a system to extract RST-formatted (reStructuredText)
31 comment blocks out of *.cmake files and populate them in a set of template
32 *.rst files.
33
34 An example of usage would be::
35
36 $ cd tribits/doc/developers_guide
37 $ extract_rst_cmake_doc.py \
38 --extract-from=../../package_arch/,../..//utils/ \
39 --rst-file-pairs=TribitsDetailedMacroFunctionDocsTemplate.rst:TribitsDetailedMacroFunctionDocs.rst
40
41 The --extract-from argument will read every *.cmake file in the
42 directory given if the directory is specified with '/' at the end. We use
43 --file-extensions (default is '.cmake') to provide the set of file patterns.
44
45 The --rst-file-pairs argument gives pairs of files
46 <name>Template.rst:<name>.rst that will subsutute the placeholders for macros
47 and function definitions from the file <name>Template.rst and will produce the
48 file <name>.rst.
49
50 The way it works is that the *.cmake files specified by
51 --extract-from must contain comment blocks of the form:
52
53 {{{
54 #
55 # @MACRO: SOME_MACRO_NAME()
56 #
57 # [rst formatted macro doc]
58 #
59 MACRO(SOME_MACRO_NAME ...
60 ...
61
62 #
63 # @FUNCTION: SOME_FUNC_NAME()
64 #
65 # [rst formatted func doc]
66 #
67 FUNCTION(SOME_FUNC_NAME ...
68 ...
69 }}}
70
71 The rules for these comment blocks are:
72
73 * The comment blocks '#' must begin on the first column.
74
75 * The special comment block identifier '@MACRO: <name>()' must start on the
76 3rd column, i.e. '# @MACRO: <name>()'. Otherwise, the comment block will be
77 ignored.
78
79 * Text in the comment blocks must begin after the 2nd column. That is,
80 '#somthing' is illegal and will result in an error.
81
82 * The comment blocks must terminate at the MACRO or FUNCTION definition and
83 the <name> of the macro or function listed in the marker '@MACRO: <name>()'
84 or '@MACRO: <name>()' must match what is in the MACRO(<name> ...) or
85 FUNCTION(<name> ...) definition.
86
87 * Whitespce is allowed in the block marker between '@MACRO' and ':' and after
88 the definition 'MACRO(' and 'FUNTION('. However, the name must be on the
89 same line.
90
91 These RST comment blocks are then inserted into RST template files
92 <name>Template.rst producing output files <name>.rst listed in the
93 --rst-file-pairs argument.
94
95 The format of these RST template files is:
96
97 {{{
98
99 Detailed Macro and Function Documentation
100 =========================================
101
102 @FUNCTION: SOME_FUNC_NAME2() -
103
104 @MACRO: SOME_MACRO_NAME1() -
105
106 }}}
107
108 The line of the form '@FUNCTION: SOME_FUNC_NAME2() -' are flagged as
109 replacement blocks that are substitted with the extracted RST blocks of the
110 same name and type. For example, the above RST template file would be
111 substituted to look like:
112
113 {{{
114
115 Detailed Macro and Function Documentation
116 =========================================
117
118 SOME_FUNC_NAME2()
119 -----------------
120
121 [rst formatted func text]
122
123
124 SOME_MACRO_NAME1()
125 ------------------
126
127 [rst formatted macro text]
128
129
130 }}}
131
132 The format of these substitution lines is:
133
134 @<blockType>: <blockName> <sepChar>
135
136 where:
137
138 * <blockType> is either 'MACRO' or 'FUNCTION'.
139 * <blockName> is the function or macro name (including the parath).
140 * <sepChar> is a single char used to build the RST section underline.
141
142 The rules for this format are:
143
144 * The '@' in '@<blockType> must being on the first column. Otherwise the
145 substitution block is ignored.
146
147 * <blockName> must match the name of a previously read in RST documentation
148 block. If it is not, then processing ends with an error.
149
150 * The <sepChar> must be present and must be a single char, typically '-',
151 '+', "=", etc. This should be picked to be consistent with the RST document
152 this is being put into.
153
154 * At least one space must separate the fields '<blockName> <sepChar>' or an
155 error will occur.
156
157 See the unit tests in extract_rst_cmake_doc_UnitTest.py for examples of
158 behavior w.r.t. different errors and formatting mistakes.
159 """
160
161 #
162 # Helper functions
163 #
164
165
166 # Strip off the "()" for the entity name
167 def getBaseEntityName(entityName):
168 parenthIdx = entityName.find("(")
169 if parenthIdx > 0:
170 return entityName[0:parenthIdx]
171 return entityName
172
173
174 # Get the line type and entity name for CMake code
175 def getLineEntityTypeAndName(line, fileNameAndLinePrefix, currentRstDocBlockType,
176 inRstDocBlockBaseName \
177 ):
178 lineSplit = line.split("(")
179 if len(lineSplit) == 1:
180 raise Exception(fileNameAndLinePrefix+ \
181 "error: expecting "+currentRstDocBlockType+"("+inRstDocBlockBaseName+" ...)"+\
182 " on this line. RST comment block must terminate in the stated entity!")
183 lineEntityType = lineSplit[0].strip()
184 lineEntityName = lineSplit[1].strip().split(" ")[0].strip()
185 if lineEntityName[-1] == ")":
186 lineEntityName = lineEntityName[0:-1]
187 return (lineEntityType.upper(), lineEntityName)
188
189
190 # Format file name and line number
191 def getFileNameLineNum(fileName, lineNum):
192 return fileName+":"+str(lineNum)
193
194
195 # Format file name and line number as prefix
196 def getFileNameLineNumPrefix(fileName, lineNum):
197 return getFileNameLineNum(fileName, lineNum)+": "
198
199
200 # Extract a set of RST doc blocks from the given text block (read from a file).
201 #
202 def extractRstDocBlocksFromText(rawText, rstBlockTypes, fileName, traceExtraction=False \
203 ):
204
205 rstDocBlocks = {}
206
207 inCommentBlock = False
208
209 inRstDocBlock = ""
210 currentRstDocBlockType = ""
211 currentRstBlockBody = ""
212
213 line_i = 0
214 for line in rawText.splitlines():
215 #print("\nline = '" + line + "'")
216
217 line_i += 1
218
219 fileNameAndLinePrefix = fileName+":"+str(line_i)+": "
220
221 # Determine if this is a comment line
222 if len(line) == 0:
223 inCommentBlock = False
224 elif line[0] == '#':
225 inCommentBlock = True
226 else:
227 inCommentBlock = False
228
229 justFoundBlockBeginning = False
230 if inCommentBlock and not inRstDocBlock:
231 # Look for beginning of expected RST doc block type
232 for blockType in rstBlockTypes:
233 blockBeginning = "# @"+blockType
234 lineBlockBeginning = line[0:len(blockBeginning)]
235 #print("blockBeginning = '" + blockBeginning + "'")
236 #print("lineBlockBeginning = '" + lineBlockBeginning + "'")
237 if lineBlockBeginning == blockBeginning:
238 #print("Matches block beginning " + blockType + "!")
239 currentRstDocBlockType = blockType
240 splitOnColon = line.split(":")
241 if len(splitOnColon) != 2:
242 raise Exception(fileNameAndLinePrefix+ \
243 "error: '"+line+"' is missing the colon ':' separator!")
244 inRstDocBlock = splitOnColon[1].strip()
245 #print("inRstDocBlock = '" + inRstDocBlock + "'")
246 #print("currentRstDocBlockType = '" + currentRstDocBlockType + "'")
247 inRstDocBlockFileNameLineNum = getFileNameLineNum(fileName, line_i)
248 if traceExtraction:
249 print("Extracting '" + blockType + "' block '" + inRstDocBlock + "'"
250 + " from " + getFileNameLineNumPrefix(fileName, line_i))
251 currentRstBlockBody = ""
252 justFoundBlockBeginning = True
253 break
254
255 if justFoundBlockBeginning:
256 # Just found the beginning of the RST block so move to next line!
257 continue
258
259 if inCommentBlock and inRstDocBlock:
260 # Strip off the beginning "# " and add to body
261 if len(line) > 1 and line[1] != " ":
262 raise Exception( \
263 getFileNameLineNumPrefix(fileName, line_i)+ \
264 "error: Comment blocks must have at least one space after '#'"+\
265 " in line = '"+line+"'")
266 rstBlockBodyLine = line[2:]
267 #print("rstBlockBodyLine = '" + rstBlockBodyLine + "'")
268 currentRstBlockBody += (rstBlockBodyLine + "\n")
269 #print("currentRstBlockBody = {\n" + currentRstBlockBody + "}")
270
271 if not inCommentBlock and inRstDocBlock:
272 # This is the end of the RST block
273 fileNameAndLinePrefix = getFileNameLineNumPrefix(fileName, line_i)
274 inRstDocBlockBaseName = getBaseEntityName(inRstDocBlock)
275 # First, verify that the comment block matches the entity
276 (lineEntityType, lineEntityName) = getLineEntityTypeAndName(
277 line, fileNameAndLinePrefix, currentRstDocBlockType, inRstDocBlockBaseName)
278 if lineEntityType != currentRstDocBlockType:
279 raise Exception(fileNameAndLinePrefix+ \
280 "error: expecting '"+currentRstDocBlockType+\
281 "' but got type '"+lineEntityType+"''")
282 if lineEntityName != inRstDocBlockBaseName:
283 raise Exception(fileNameAndLinePrefix+ \
284 "error: expecting "+lineEntityType+" "+inRstDocBlock+\
285 " but got wrong "+lineEntityType+" name '"+lineEntityName+"'")
286 # ToDo: Assert that blockName is the same!
287 # Next, story the RST block
288 rstDocBlocks.update( {
289 inRstDocBlock : {
290 "type" : currentRstDocBlockType,
291 "body" : currentRstBlockBody,
292 "fileNameLineNum" : inRstDocBlockFileNameLineNum,
293 }
294 } )
295 # Finally, rest to look for the next RST block
296 inRstDocBlock = ""
297 currentRstDocBlockType = ""
298 #print("rstDocBlocks:\n" + str(rstDocBlocks))
299
300 # ToDo: Check that this line starts with currentRstDocBlockType or
301 # throw exception!
302
303 # end for line
304
305 return rstDocBlocks
306
307
308 #
309 # Get an RST block from dict or throw except if it does not exist
310 #
311 def getRstDocBlock(rstDocBlocks, rstBlockName, fileNameAndLinePrefix, line):
312 rstDocBlock = rstDocBlocks.get(rstBlockName, "")
313 if rstDocBlock == "":
314 raise Exception(fileNameAndLinePrefix+"Error, block name"+\
315 " '"+rstBlockName+"' does not exist in rstDocBlocks for line = '"+line+"'")
316 return rstDocBlock
317
318
319 #
320 # Get RST section separator of length lineLen
321 #
322 def getRstSectStr(sepChar, lineLen):
323 return sepChar * lineLen
324
325
326 def removeEmtpyElements(arrayIn):
327 arrayOut = []
328 for ele in arrayIn:
329 if ele != "":
330 arrayOut.append(ele)
331 return arrayOut
332
333
334 # Replace references to RST blocks with sections
335 #
336 def replaceWithRstDocBlocksInText(textToReplace, rstBlockTypes, rstDocBlocks, fileName, \
337 traceReplacements=False, fileNamePathBaseDir="", includeFileNameLineNum=False \
338 ):
339
340 replacedText = ""
341
342 textToReplaceLines = textToReplace.splitlines()
343 numTextToReplaceLines = len(textToReplaceLines)
344
345 if not fileNamePathBaseDir == "":
346 fileNamePathBaseDirAndSepLen = len(fileNamePathBaseDir)
347 if not fileNamePathBaseDir.endswith("/"): fileNamePathBaseDirAndSepLen += 1
348 else:
349 fileNamePathBaseDirAndSepLen = 0
350
351 line_i = 0
352
353 for line in textToReplaceLines:
354
355 line_i += 1
356
357 #print("\nline_i = " + str(line_i))
358 #print("line = '" + line + "'")
359
360 # Look for block of text to replace
361 replacedRstBlock = False
362 for blockType in rstBlockTypes:
363 blockBeginning = "@"+blockType
364 lineBlockBeginning = line[0:len(blockBeginning)]
365 #print("blockBeginning = '" + blockBeginning + "'")
366 #print("lineBlockBeginning = '" + lineBlockBeginning + "'")
367 if lineBlockBeginning == blockBeginning:
368 #print("Matches block beginning " + blockType + "!")
369 lineSplitArray = line.split(":")
370 fileNameAndLinePrefix = fileName+":"+str(line_i)+": "
371 #print("lineSplitArray = " + str(lineSplitArray))
372 if len(lineSplitArray)==1:
373 # Missing the colon so no replacement!
374 break
375 rstBlockNameAndSecCharArray = removeEmtpyElements(
376 lineSplitArray[1].strip().split(" "))
377 #print("rstBlockNameAndSecCharArray = " + str(rstBlockNameAndSecCharArray))
378 if len(rstBlockNameAndSecCharArray) != 2:
379 raise Exception(fileNameAndLinePrefix+"Error, line does not match format"+\
380 " '@"+blockType+": <blockName> <sepChar>' for line = '"+line+"'")
381 rstBlockName = rstBlockNameAndSecCharArray[0].strip()
382 rstBlockSecChar = rstBlockNameAndSecCharArray[1].strip()
383 #print("rstBlockName = '" + rstBlockName + "'")
384 #print("rstBlockSecChar = '" + rstBlockSecChar + "'")
385 if len(rstBlockSecChar) != 1:
386 raise Exception(fileNameAndLinePrefix+\
387 "Error, separation char must be on char in line = '"+line+"'")
388 if traceReplacements:
389 print("Replacing '" + blockType + "' block '" + rstBlockName + "'" +
390 " in " + getFileNameLineNumPrefix(fileName, line_i))
391 rstDocBlock = getRstDocBlock(rstDocBlocks, rstBlockName,
392 fileNameAndLinePrefix, line)
393 if rstDocBlock.get("type") != blockType:
394 raise Exception(fileNameAndLinePrefix+"Error, looked up block type"+\
395 " '@"+rstDocBlock.get("type")+"' does not match block type for line = '"+line+"'")
396 rstSecStr = getRstSectStr(rstBlockSecChar, len(rstBlockName))
397 # Add the replaced block section
398 replacedText += (rstBlockName+"\n")
399 replacedText += (rstSecStr+"\n")
400 replacedText += rstDocBlock.get("body")
401 if includeFileNameLineNum:
402 fileNameLineNum = rstDocBlock.get("fileNameLineNum")
403 if fileNamePathBaseDirAndSepLen > 0:
404 relFileNameLineNum = fileNameLineNum[fileNamePathBaseDirAndSepLen:]
405 else:
406 relFileNameLineNum = fileNameLineNum
407 replacedText += "\nIn: "+relFileNameLineNum + "\n\n"
408 replacedRstBlock = True
409 break
410
411 if not replacedRstBlock and not (line_i == numTextToReplaceLines and line==""):
412 replacedText += (line+"\n")
413
414 #print("replacedText = {\n" + replacedText + "}")
415
416 # end for line
417
418 return replacedText
419
420
421 # Get a list of files give list of files, dirs, and extensions
422 def getExtractFilesList(inOptions):
423 filesList = []
424 for fileOrDir in inOptions.extractFrom.split(","):
425 if fileOrDir[-1] == "/":
426 # This is a directory so grab all files with the matching extensions
427 for fileExtension in inOptions.fileExtensions.split(","):
428 globStr = fileOrDir+"*"+fileExtension
429 globedFiles = glob.glob(globStr)
430 filesList.extend(globedFiles)
431 else:
432 filesList.append(fileOrDir)
433 return filesList
434
435
436 # Get the list of template and replacement files
437 def getRstFilesList(inOptions):
438 rstFilesList = []
439 if inOptions.rstFilePairs:
440 for templateFileAndFile in inOptions.rstFilePairs.split(","):
441 rstFilesList.append(templateFileAndFile.split(":"))
442 return rstFilesList
443
444
445 # Extract RST blocks out of a set of files
446 def extractRstDocBlocksFromFileList(fileList, rstBlockTypes, traceExtraction=False):
447 rstDocBlocks = {}
448 for fileName in fileList:
449 rawFileStr = open(fileName, 'r').read()
450 rstDocBlocks.update(
451 extractRstDocBlocksFromText(rawFileStr, rstBlockTypes, fileName, traceExtraction)
452 )
453 return rstDocBlocks
454
455
456 # Make substitutions in a template file and produce a target file
457 def replaceWithRstDocBlocksInTemplateFileList(rstFilesList, rstBlockTypes,
458 rstDocBlocks, traceReplacements=False, fileNamePathBaseDir="",
459 includeFileNameLineNum=False \
460 ):
461 for (templateFileName, fileName) in rstFilesList:
462 templateFileStr = open(templateFileName, 'r').read()
463 fileStr = replaceWithRstDocBlocksInText(templateFileStr, rstBlockTypes,
464 rstDocBlocks, templateFileName, traceReplacements=traceReplacements,
465 fileNamePathBaseDir=fileNamePathBaseDir,
466 includeFileNameLineNum=includeFileNameLineNum )
467 open(fileName, 'w').write(fileStr)
468
469
470 # Run a command and synchronize the output
471 def runCmnd(options, cmnd):
472 if options.debug:
473 print("*** Running command: " + cmnd)
474 if options.noOpt:
475 print(cmnd)
476 else:
477 child = subprocess.Popen(cmnd, stdout=subprocess.PIPE).stdout
478 output = child.read()
479 sys.stdout.flush()
480 print(output)
481 sys.stdout.flush()
482
483
484 #
485 # Script body
486 #
487
488
489 if __name__ == '__main__':
490
491 #
492 # A) Get command-line options
493 #
494
495 from optparse import OptionParser
496
497 clp = OptionParser(usage=usageHelp)
498
499 clp.add_option(
500 "--extract-from", dest="extractFrom", type="string",
501 default="",
502 help="List (comma separated) of directories (ending with '/') and files that the RST"+\
503 " comment blocks will be extracted from. Directories ending with '/' will result in"+\
504 " files being found with extensions listed in --file-extensions. Otherwise, the given"+\
505 " files are searched.")
506
507 clp.add_option(
508 "--file-extensions", dest="fileExtensions", type="string",
509 default=".cmake",
510 help="List (comma separated) of file extensions (e.g. '.cmake'). The default"+\
511 "is '.cmake'")
512
513 clp.add_option(
514 "--rst-file-pairs", dest="rstFilePairs", type="string",
515 default="",
516 help="List (comma separated) files of the form '<name>Template.rst:<name>.rst,...'"+\
517 " that gives sets of template files to have comment blocks inserted into.")
518
519 clp.add_option(
520 "--show-file-name-line-num", dest="showFileNameLineNum", action="store_true",
521 help="Include the relative file name path and line number for each documentation block.")
522 clp.add_option(
523 "--no-show-file-name-line-num", dest="showFileNameLineNum", action="store_false", default=False,
524 help="Include the relative file name path and line number for each documentation block. [default]" )
525
526 clp.add_option(
527 "--file-name-path-base-dir", dest="fileNamePathBaseDir", type="string",
528 default="",
529 help="Base path stripped off of file names reported when --show-file-name-line-num is set."+\
530 " NOTE: This path should be relative to the paths in --extract-from and may be relative paths." )
531
532 clp.add_option(
533 "--dump-rst-blocks", dest="dumpRstBlocks", action="store_true",
534 help="Print out the RST documentation blocks that are read in.")
535 clp.add_option(
536 "--no-dump-rst-blocks", dest="dumpRstBlocks", action="store_false", default=False,
537 help="Print out the RST documentation blocks that are read in. [default]" )
538
539 clp.add_option(
540 "--do-trace", dest="doTrace", action="store_true",
541 help="Print out trace of RST blocks read and replaced.")
542 clp.add_option(
543 "--no-do-trace", dest="doTrace", action="store_false", default=False,
544 help="Do not print out trace of RST blocks read and replaced. [default]" )
545
546
547 (options, args) = clp.parse_args()
548
549 # Echo the command-line
550
551 if options.doTrace:
552 print("")
553 print("**************************************************************************")
554 print("Script: extract_rst_cmake_doc.py \\")
555 print(" --extract-from='" + options.extractFrom + "' \\")
556 print(" --file-extensions='" + options.fileExtensions + "' \\")
557 print(" --rst-file-pairs='" + options.rstFilePairs + "' \\")
558 if options.showFileNameLineNum:
559 print(" --show-file-name-line-num \\")
560 else:
561 print(" --no-show-file-name-line-num \\")
562 print(" --file-name-path-base-dir='" + options.fileNamePathBaseDir + "' \\")
563 if options.dumpRstBlocks:
564 print(" --dump-rst-blocks \\")
565 else:
566 print(" --no-dump-rst-blocks \\")
567 if options.doTrace:
568 print(" --do-trace \\")
569 else:
570 print(" --no-do-trace \\")
571
572 # Assert the commandline
573
574 if not options.extractFrom:
575 print("\nError, --extract-from must be specified (see --help)!")
576 sys.exit(1)
577
578 if options.doTrace and not options.rstFilePairs:
579 print("\nWarning: --rst-file-pairs is empty and no RST comment blocks will be set!")
580
581 #
582 # B) Read in all of the RST documentation blocks
583 #
584
585 rstBlockTypes = ["MACRO", "FUNCTION"]
586
587 if options.doTrace:
588 print("\nExtracting RST documentation blocks in --extract-from:")
589 extractFromFilesList = getExtractFilesList(options)
590 rstDocBlocks = extractRstDocBlocksFromFileList(
591 extractFromFilesList, rstBlockTypes, options.doTrace)
592
593 if options.dumpRstBlocks:
594 print("Read in RST blocks:\n")
595 pp.pprint(rstDocBlocks)
596
597 #
598 # C) Make the substitutions in all of the file pairs
599 #
600
601 if options.doTrace:
602 print("\nReplacing RST documentation blocks in RST files in --rst-file-pairs:")
603 rstFilesList = getRstFilesList(options)
604 replaceWithRstDocBlocksInTemplateFileList(rstFilesList, rstBlockTypes,
605 rstDocBlocks, traceReplacements=options.doTrace,
606 fileNamePathBaseDir=options.fileNamePathBaseDir,
607 includeFileNameLineNum=options.showFileNameLineNum )