| 1 |
import argparse |
| 2 |
import os |
| 3 |
import re |
| 4 |
from shutil import copyfile, copytree |
| 5 |
import subprocess |
| 6 |
import sys |
| 7 |
|
| 8 |
try: |
| 9 |
doc_path = f"{os.sep}".join(os.path.abspath(__file__).split(os.sep)[:-2]) |
| 10 |
sys.path.append(doc_path) |
| 11 |
project_path = f"{os.sep}".join(os.path.abspath(__file__).split(os.sep)[:-4]) |
| 12 |
sys.path.append(project_path) |
| 13 |
except Exception as e: |
| 14 |
print(f"Can not add project path to system path! Exiting!\nERROR: {e}") |
| 15 |
exit(1) |
| 16 |
|
| 17 |
|
| 18 |
def is_rst_file(file_path: str) -> bool: |
| 19 |
""" Checks if file_path has .rst extension and if file_path is a file. |
| 20 |
""" |
| 21 |
if os.path.splitext(file_path)[-1] == '.rst' and os.path.isfile(file_path): |
| 22 |
return True |
| 23 |
return False |
| 24 |
|
| 25 |
|
| 26 |
def change_paths_and_get_includes(source_file: str, src_file_path: str, start_path: str, rst_dir: str, |
| 27 |
tribits_base_dir: str, copy_file: bool = True) -> tuple: |
| 28 |
""" Changes paths in source file, to be relative to sphinx_path or parent .rst document. |
| 29 |
Returns a tuple with .rst file content and includes(absolute_path, relative_to). |
| 30 |
""" |
| 31 |
with open(source_file, 'r') as src_file: |
| 32 |
source_file_str = src_file.read() |
| 33 |
source_file_list = list() |
| 34 |
include_file_list = set() |
| 35 |
for line in source_file_str.split('\n'): |
| 36 |
splitted_line = line.split() |
| 37 |
if 'include::' in splitted_line: |
| 38 |
incl_index = splitted_line.index('include::') |
| 39 |
path_index = incl_index + 1 |
| 40 |
if len(splitted_line) > path_index: |
| 41 |
new_line = [] |
| 42 |
spaces_indented = line.find('.') # Below 'join()' statement adds a space! |
| 43 |
if spaces_indented > 0: |
| 44 |
new_line.append(' ' * (spaces_indented - 1)) |
| 45 |
new_line.extend(splitted_line[:path_index]) |
| 46 |
abs_path = os.path.abspath(os.path.join(src_file_path, |
| 47 |
splitted_line[path_index])) |
| 48 |
tbd = tribits_base_dir.split(os.sep)[1:] |
| 49 |
path_elem = abs_path.split(os.sep)[len(tbd) + 1:] |
| 50 |
new_path = os.path.join(rst_dir, *path_elem) |
| 51 |
os.makedirs(os.path.dirname(new_path), exist_ok=True) |
| 52 |
if not os.path.isfile(new_path) and copy_file: |
| 53 |
copyfile(src=abs_path, dst=new_path, follow_symlinks=True) |
| 54 |
if is_rst_file(file_path=new_path): |
| 55 |
include_file_list.add(abs_path) |
| 56 |
rel_path_from_sphinx_dir = os.path.relpath(path=new_path, start=start_path) |
| 57 |
new_line.append(rel_path_from_sphinx_dir) |
| 58 |
new_line = ' '.join(new_line) |
| 59 |
# Make sure version is formatted correctly: |
| 60 |
if ':Version:' in new_line: |
| 61 |
new_line = f":Version:\n {new_line.split(':Version:')[-1]}" |
| 62 |
source_file_list.append(new_line) |
| 63 |
else: |
| 64 |
source_file_list.append(line) |
| 65 |
else: |
| 66 |
source_file_list.append(line) |
| 67 |
abs_path_str = '\n'.join(source_file_list) |
| 68 |
|
| 69 |
return abs_path_str, include_file_list |
| 70 |
|
| 71 |
|
| 72 |
class SphinxRstGenerator: |
| 73 |
""" Changes include paths to relative to Sphinx build dir. Saves three main .rst docs files inside Sphinx dir. |
| 74 |
""" |
| 75 |
|
| 76 |
def __init__(self): |
| 77 |
self.paths = { |
| 78 |
'maintainers_guide': { |
| 79 |
'src': os.path.join(doc_path, 'guides', 'maintainers_guide', 'TribitsMaintainersGuide.rst'), |
| 80 |
'src_path': os.path.join(doc_path, 'guides', 'maintainers_guide'), |
| 81 |
'final_path': os.path.join(doc_path, 'sphinx', 'maintainers_guide', 'index.rst'), |
| 82 |
'sphinx_path': os.path.join(doc_path, 'sphinx', 'maintainers_guide'), |
| 83 |
'title': 'TriBITS Maintainers Guide and Reference'}, |
| 84 |
'users_guide': { |
| 85 |
'src': os.path.join(doc_path, 'guides', 'users_guide', 'TribitsUsersGuide.rst'), |
| 86 |
'src_path': os.path.join(doc_path, 'guides', 'users_guide'), |
| 87 |
'final_path': os.path.join(doc_path, 'sphinx', 'users_guide', 'index.rst'), |
| 88 |
'sphinx_path': os.path.join(doc_path, 'sphinx', 'users_guide'), |
| 89 |
'title': 'TriBITS Users Guide and Reference'}, |
| 90 |
'build_ref': { |
| 91 |
'src': os.path.join(doc_path, 'build_ref', 'TribitsBuildReference.rst'), |
| 92 |
'src_path': os.path.join(doc_path, 'build_ref'), |
| 93 |
'final_path': os.path.join(doc_path, 'sphinx', 'build_ref', 'index.rst'), |
| 94 |
'sphinx_path': os.path.join(doc_path, 'sphinx', 'build_ref'), |
| 95 |
'title': 'Generic TriBITS Project, Build, Test, and Install Reference Guide'}} |
| 96 |
self.rst_dir = os.path.join(doc_path, 'sphinx', 'copied_files') |
| 97 |
self.tribits_base_dir = self._cli() |
| 98 |
self.already_modified_files = set() |
| 99 |
self.create_rst_dir() |
| 100 |
self.build_docs() |
| 101 |
|
| 102 |
@staticmethod |
| 103 |
def _cli() -> str: |
| 104 |
""" Support for common line arguments. """ |
| 105 |
parser = argparse.ArgumentParser() |
| 106 |
parser.add_argument("--copy-base-dir", help="Path to TriBITS base directory") |
| 107 |
args = parser.parse_args() |
| 108 |
abs_path = os.path.abspath(args.copy_base_dir) |
| 109 |
if not abs_path or not os.path.exists(abs_path): |
| 110 |
print(f"\n==> Path: `{abs_path}` is not correct!") |
| 111 |
sys.exit(1) |
| 112 |
print(f"Provided TriBITS base dir: {abs_path}") |
| 113 |
return abs_path |
| 114 |
|
| 115 |
def create_rst_dir(self) -> None: |
| 116 |
""" Creates copied_files directory in Sphinx directory. All include files will be copy |
| 117 |
there. |
| 118 |
""" |
| 119 |
if self.rst_dir is not None: |
| 120 |
if not os.path.exists(self.rst_dir): |
| 121 |
os.makedirs(self.rst_dir) |
| 122 |
|
| 123 |
@staticmethod |
| 124 |
def build_docs() -> None: |
| 125 |
""" Builds TriBITS documentation based on shell scripts. |
| 126 |
""" |
| 127 |
build_script_path = os.path.join(doc_path, 'build_docs.sh') |
| 128 |
subprocess.call([build_script_path, '--skip-final-generation']) |
| 129 |
|
| 130 |
@staticmethod |
| 131 |
def run_sphinx(cwd: str) -> None: |
| 132 |
""" Runs Sphinx for each documentation. |
| 133 |
""" |
| 134 |
sphinx_command = ["make", "html"] |
| 135 |
subprocess.call(sphinx_command, cwd=cwd) |
| 136 |
|
| 137 |
def combine_documentation(self, docs_dir: str, change_url_to_landing_page: bool = True, |
| 138 |
change_title_of_docs_main_page: bool = True, title: str = '') -> None: |
| 139 |
""" Renames and moves directory of generated static pages into combined directory |
| 140 |
""" |
| 141 |
new_name = os.path.split(docs_dir)[-1] |
| 142 |
dir_to_rename = os.path.join(docs_dir, '_build', 'html') |
| 143 |
new_name_path = os.path.join(docs_dir, '_build', new_name) |
| 144 |
os.rename(src=dir_to_rename, dst=new_name_path) |
| 145 |
static_dir = os.path.join(doc_path, 'sphinx', 'combined_docs', new_name) |
| 146 |
copytree(src=new_name_path, dst=static_dir) |
| 147 |
if change_url_to_landing_page: |
| 148 |
self.change_url_to_landing_page(docs_static_dir=static_dir) |
| 149 |
if change_title_of_docs_main_page: |
| 150 |
self.change_title_of_docs_main_page(docs_static_dir=static_dir, new_title=title) |
| 151 |
|
| 152 |
@staticmethod |
| 153 |
def change_url_to_landing_page(docs_static_dir: str) -> None: |
| 154 |
""" Changes home url of documentation page, so it points to landing page. |
| 155 |
""" |
| 156 |
index_html = os.path.join(docs_static_dir, 'index.html') |
| 157 |
with open(index_html, 'r') as index_read: |
| 158 |
index_str = index_read.read() |
| 159 |
repl_url = index_str.replace('<a href="#" class="icon icon-home"> TriBITS', |
| 160 |
'<a href="../index.html" class="icon icon-home"> TriBITS') |
| 161 |
with open(index_html, 'w') as index_write: |
| 162 |
index_write.write(repl_url) |
| 163 |
|
| 164 |
@staticmethod |
| 165 |
def change_title_of_docs_main_page(docs_static_dir: str, new_title: str) -> None: |
| 166 |
""" Changes home url of documentation page, so it points to landing page. |
| 167 |
""" |
| 168 |
index_html = os.path.join(docs_static_dir, 'index.html') |
| 169 |
with open(index_html, 'r') as index_read: |
| 170 |
index_str = index_read.read() |
| 171 |
repl_url = re.sub('(<title>1 Introduction —).+(documentation</title>)', |
| 172 |
f'<title>{new_title}</title>', index_str) |
| 173 |
with open(index_html, 'w') as index_write: |
| 174 |
index_write.write(repl_url) |
| 175 |
|
| 176 |
@staticmethod |
| 177 |
def save_rst(file_path: str, file_content: str) -> None: |
| 178 |
""" Saves .rst file with given pathh and content |
| 179 |
""" |
| 180 |
with open(file_path, 'w') as dest_file: |
| 181 |
dest_file.write(file_content) |
| 182 |
|
| 183 |
def generate_rst(self, source_file: str, final_path: str = None, src_path: str = None, |
| 184 |
start_path: str = None) -> set: |
| 185 |
""" Generate correct links in .rst files, so Sphinx can find them |
| 186 |
""" |
| 187 |
if final_path is None: |
| 188 |
overwrite_source = True |
| 189 |
else: |
| 190 |
overwrite_source = False |
| 191 |
|
| 192 |
file_content, includes = change_paths_and_get_includes(source_file=source_file, src_file_path=src_path, |
| 193 |
start_path=start_path, rst_dir=self.rst_dir, |
| 194 |
tribits_base_dir=self.tribits_base_dir) |
| 195 |
|
| 196 |
if overwrite_source: |
| 197 |
self.save_rst(file_path=source_file, file_content=file_content) |
| 198 |
else: |
| 199 |
self.save_rst(file_path=final_path, file_content=file_content) |
| 200 |
|
| 201 |
return includes |
| 202 |
|
| 203 |
def remove_title_numbering(self) -> None: |
| 204 |
""" Removes numbering from docs. |
| 205 |
""" |
| 206 |
for doc_name, sources in self.paths.items(): |
| 207 |
|
| 208 |
str_to_replace = '.. rubric::' |
| 209 |
with open(sources.get('final_path'), 'r') as src_file: |
| 210 |
org_str = src_file.read() |
| 211 |
org_list = org_str.split('\n') |
| 212 |
if org_list[0].startswith('====='): |
| 213 |
del org_list[0] |
| 214 |
if org_list[1].startswith('====='): |
| 215 |
del org_list[1] |
| 216 |
org_list[0] = f'{str_to_replace} {org_list[0]}' |
| 217 |
mod_str = '\n'.join(org_list) |
| 218 |
|
| 219 |
with open(sources.get('final_path'), 'w') as dst_file: |
| 220 |
dst_file.write(mod_str) |
| 221 |
|
| 222 |
def main(self): |
| 223 |
""" Main routine goes for nested .rst docs |
| 224 |
""" |
| 225 |
child_rst = set() |
| 226 |
for doc_name, sources in self.paths.items(): |
| 227 |
includes = self.generate_rst(source_file=sources.get('src'), src_path=sources.get('src_path'), |
| 228 |
final_path=sources.get('final_path'), start_path=sources.get('sphinx_path')) |
| 229 |
child_rst.update(includes) |
| 230 |
self.already_modified_files.update(child_rst) |
| 231 |
tbd = self.tribits_base_dir.split(os.sep)[1:] |
| 232 |
child_rst_lst = list(child_rst) |
| 233 |
|
| 234 |
sphinx_rel_path = self.paths.get('maintainers_guide').get('sphinx_path') |
| 235 |
grand_child_rst = set() |
| 236 |
for child in child_rst_lst: |
| 237 |
path_elem = child.split(os.sep)[len(tbd) + 1:] |
| 238 |
final_path = os.path.join(self.rst_dir, *path_elem) |
| 239 |
os.makedirs(os.path.dirname(final_path), exist_ok=True) |
| 240 |
src_path = os.path.split(child)[0] |
| 241 |
includes_grand = self.generate_rst(source_file=child, src_path=src_path, |
| 242 |
final_path=final_path, start_path=sphinx_rel_path) |
| 243 |
grand_child_rst.update(includes_grand) |
| 244 |
grand_child_rst_lst = [gc_rst for gc_rst in grand_child_rst if gc_rst not in self.already_modified_files] |
| 245 |
|
| 246 |
grand_grand_child_rst = set() |
| 247 |
for grand_child in grand_child_rst_lst: |
| 248 |
path_elem = grand_child.split(os.sep)[len(tbd) + 1:] |
| 249 |
final_path = os.path.join(self.rst_dir, *path_elem) |
| 250 |
os.makedirs(os.path.dirname(final_path), exist_ok=True) |
| 251 |
src_path = os.path.split(grand_child)[0] |
| 252 |
includes_grand_grand = self.generate_rst(source_file=grand_child, src_path=src_path, |
| 253 |
final_path=final_path, start_path=sphinx_rel_path) |
| 254 |
grand_grand_child_rst.update(includes_grand_grand) |
| 255 |
|
| 256 |
if not grand_grand_child_rst: |
| 257 |
print('DONE! ALL GOOD!\n') |
| 258 |
else: |
| 259 |
print('NOT DONE!\n') |
| 260 |
|
| 261 |
self.remove_title_numbering() |
| 262 |
|
| 263 |
print('===> Generating Sphinx documentation:\n') |
| 264 |
for doc_name, sources in self.paths.items(): |
| 265 |
cwd = sources.get('sphinx_path') |
| 266 |
print(f'===> Generating {doc_name}\n') |
| 267 |
self.run_sphinx(cwd=cwd) |
| 268 |
self.combine_documentation(docs_dir=cwd, title=sources.get('title')) |
| 269 |
|
| 270 |
|
| 271 |
if __name__ == '__main__': |
| 272 |
SphinxRstGenerator().main() |