| 1 |
from __future__ import absolute_import |
| 2 |
|
| 3 |
import logging |
| 4 |
import os |
| 5 |
import shutil |
| 6 |
import zipfile |
| 7 |
|
| 8 |
import markdown |
| 9 |
|
| 10 |
from ..help_parsers.helpparsers import (Stage2Parser, Stage3Parser, |
| 11 |
Stage4Parser, Stage5Parser) |
| 12 |
from ..utilfunctions import change_filename |
| 13 |
|
| 14 |
|
| 15 |
def build(args): |
| 16 |
"""Compiles the files in args.indir to the archive output args.outfile. |
| 17 |
|
| 18 |
Compiled in several stages: |
| 19 |
- stage1: Transform all input files with markdown placing the results |
| 20 |
into args.temp+"/stage1". |
| 21 |
- stage2: Parses and strips the output of stage1 to build the list |
| 22 |
that will be made into the c-array that contains the compiled names for |
| 23 |
the detailed help of each control. |
| 24 |
- stage3: Parses the output of stage2 to grab the images that are |
| 25 |
referred to in the the output of stage1 |
| 26 |
- stage4: Parses the output of stage1 to fix the relative hyperlinks in |
| 27 |
the output so that they will refer correctly to the correct files when |
| 28 |
in output file. |
| 29 |
- stage5: Generate the index and table of contents for the output file |
| 30 |
from the output of stage4. |
| 31 |
- stage6: Zip up the output of stage5 and put it in the output location. |
| 32 |
""" |
| 33 |
notices = logging.getLogger('notices') |
| 34 |
notices.info("Building...") |
| 35 |
logging.debug("Using '%s' as working directory", args.temp) |
| 36 |
logging.debug("Using '%s' as output file", args.outfile) |
| 37 |
logging.debug("Using '%s' as input directory", args.indir) |
| 38 |
|
| 39 |
files = generate_paths(args) |
| 40 |
|
| 41 |
if should_build(args): |
| 42 |
_do_build(args, files, notices) |
| 43 |
else: |
| 44 |
notices.info(" Up to date.") |
| 45 |
|
| 46 |
|
| 47 |
def _do_build(args, files, notices): |
| 48 |
help_array = list() |
| 49 |
extra_files = list() |
| 50 |
notices.info(" Processing input files:") |
| 51 |
for file_name in generate_file_list(args.indir, ".help"): |
| 52 |
notices.info(" %s", file_name) |
| 53 |
logging.info(" Stage 1") |
| 54 |
name1 = process_input_stage1(file_name, args, files) |
| 55 |
|
| 56 |
logging.info(" Stage 2") |
| 57 |
name2 = process_input_stage2(name1, args, files, help_array) |
| 58 |
|
| 59 |
logging.info(" Stage 3") |
| 60 |
name3 = process_input_stage3(name2, args, files, extra_files) |
| 61 |
|
| 62 |
logging.info(" Stage 4") |
| 63 |
name4 = process_input_stage4(name3, args, files) |
| 64 |
logging.info(" Stage 5") |
| 65 |
process_input_stage5(args, files, extra_files) |
| 66 |
logging.info(" Stage 6") |
| 67 |
process_input_stage6(args, files) |
| 68 |
logging.info(" Generating .cpp files") |
| 69 |
generate_cpp_files(args, files, help_array) |
| 70 |
notices.info("....Done.") |
| 71 |
|
| 72 |
|
| 73 |
def generate_paths(options): |
| 74 |
"""Generates the names of the paths that will be needed by the compiler. |
| 75 |
|
| 76 |
Generated names will be returned as a dictionary.""" |
| 77 |
paths = dict() |
| 78 |
|
| 79 |
for i in range(1, 7): |
| 80 |
stage_name = 'stage{0}'.format(i) |
| 81 |
paths[stage_name] = os.path.join(options.temp, stage_name) |
| 82 |
|
| 83 |
for path in paths.values(): |
| 84 |
if not os.path.exists(path): |
| 85 |
logging.debug(" Making %s", path) |
| 86 |
os.makedirs(path) |
| 87 |
|
| 88 |
return paths |
| 89 |
|
| 90 |
|
| 91 |
def should_build(options): |
| 92 |
"""Should we build the output file? |
| 93 |
:param options arguments from command line |
| 94 |
:return True if the output file should be built/rebuilt |
| 95 |
""" |
| 96 |
logger = logging.getLogger('should_build') |
| 97 |
if options.always_build: |
| 98 |
logger.info(" Always building") |
| 99 |
return True |
| 100 |
elif not os.path.exists(options.outfile): |
| 101 |
logger.info(" Outfile does not exist") |
| 102 |
return True |
| 103 |
elif options.carrayfilename and not os.path.exists(options.carrayfilename): |
| 104 |
logger.info(" .cpp file does not exist") |
| 105 |
return True |
| 106 |
elif check_source_newer_than_outfile(options): |
| 107 |
logger.info(" Source files are newer than output") |
| 108 |
return True |
| 109 |
return False |
| 110 |
|
| 111 |
|
| 112 |
def check_source_newer_than_outfile(options): |
| 113 |
try: |
| 114 |
outfile_time = os.path.getmtime(options.outfile) |
| 115 |
|
| 116 |
for d_path, d_names, f_names in os.walk(options.indir): |
| 117 |
for f_name in f_names: |
| 118 |
f_path = os.path.join(d_path, f_name) |
| 119 |
if os.path.getmtime(f_path) > outfile_time: |
| 120 |
logging.info( |
| 121 |
"%s has been changed since outfile. Causing build", |
| 122 |
f_path) |
| 123 |
return True |
| 124 |
elif os.path.getctime(f_path) > outfile_time: |
| 125 |
logging.info( |
| 126 |
"%s has been created since outfile. Causing build", |
| 127 |
f_path) |
| 128 |
return True |
| 129 |
|
| 130 |
except OSError as e: |
| 131 |
logging.exception("OS does not like %d. Forcing build.", e.filename) |
| 132 |
return True |
| 133 |
return False |
| 134 |
|
| 135 |
|
| 136 |
def generate_file_list(directory, extension): |
| 137 |
for path, dirs, files in os.walk(directory): |
| 138 |
for file_name in files: |
| 139 |
if os.path.splitext(file_name)[1] == extension: |
| 140 |
yield os.path.join(path, file_name) |
| 141 |
|
| 142 |
|
| 143 |
def process_input_stage1(file, options, files): |
| 144 |
infile = open(file, mode="r") |
| 145 |
input = infile.read() |
| 146 |
infile.close() |
| 147 |
|
| 148 |
md = markdown.Markdown( |
| 149 |
extensions=['markdown.extensions.toc'] |
| 150 |
) |
| 151 |
output = md.convert(input) |
| 152 |
|
| 153 |
outfile_name = change_filename(file, ".stage1", options.indir, files['stage1']) |
| 154 |
|
| 155 |
outfile = open(outfile_name, mode="w") |
| 156 |
outfile.write(output) |
| 157 |
outfile.close() |
| 158 |
return outfile_name |
| 159 |
|
| 160 |
|
| 161 |
def process_input_stage2(file, options, files, helparray): |
| 162 |
infile = open(file, mode="r") |
| 163 |
input = infile.read() |
| 164 |
infile.close() |
| 165 |
|
| 166 |
outname = change_filename(file, ".stage2", files['stage1'], files['stage2']) |
| 167 |
outfile = open(outname, mode="w") |
| 168 |
|
| 169 |
parser = Stage2Parser(file=outfile) |
| 170 |
parser.feed(input) |
| 171 |
parser.close() |
| 172 |
outfile.close() |
| 173 |
|
| 174 |
if len(parser.control) > 0: |
| 175 |
filename_in_archive = change_filename(outname, ".htm", files['stage2'], ".", makedirs=False) |
| 176 |
for control in parser.control: |
| 177 |
helparray.append((control, filename_in_archive)) |
| 178 |
logging.debug(" Control name %s", control) |
| 179 |
|
| 180 |
return outname |
| 181 |
|
| 182 |
|
| 183 |
def process_input_stage3(file, options, files, extrafiles): |
| 184 |
infile = open(file, mode="r") |
| 185 |
input = infile.read() |
| 186 |
infile.close() |
| 187 |
|
| 188 |
outname = change_filename(file, ".stage3", files['stage2'], files['stage3']) |
| 189 |
outfile = open(outname, mode="w") |
| 190 |
|
| 191 |
# figure out what subdirectory of the onlinehelp I am in |
| 192 |
subdir = os.path.dirname(outname).replace(os.path.normpath(files['stage3']), "") |
| 193 |
if subdir.startswith(os.path.sep): |
| 194 |
subdir = subdir.replace(os.path.sep, "", 1) # I only want to remove the leading sep |
| 195 |
|
| 196 |
parser = Stage3Parser(options, files, file=outfile, extrafiles=extrafiles, subdir=subdir) |
| 197 |
parser.feed(input) |
| 198 |
parser.close() |
| 199 |
outfile.close() |
| 200 |
|
| 201 |
return outname |
| 202 |
|
| 203 |
|
| 204 |
def process_input_stage4(file, options, files): |
| 205 |
"""Fix the a tags so that they point to the htm files that will be in the output archive.""" |
| 206 |
infile = open(file, mode="r") |
| 207 |
input = infile.read() |
| 208 |
infile.close() |
| 209 |
|
| 210 |
outname = change_filename(file, ".stage4", files['stage3'], files['stage4']) |
| 211 |
outfile = open(outname, mode="w") |
| 212 |
|
| 213 |
# figure out what subdirectory of the onlinehelp I am in |
| 214 |
subdir = os.path.dirname(outname).replace(os.path.normpath(files['stage4']), "") |
| 215 |
if subdir.startswith(os.path.sep): |
| 216 |
subdir = subdir.replace(os.path.sep, "", 1) # I only want to remove the leading sep |
| 217 |
|
| 218 |
parser = Stage4Parser(files=files, file=outfile, options=options, |
| 219 |
subdir=subdir) |
| 220 |
parser.feed(input) |
| 221 |
parser.close() |
| 222 |
outfile.close() |
| 223 |
|
| 224 |
return outname |
| 225 |
|
| 226 |
HTB_HEADER_TEMPLATE = "\n".join([ |
| 227 |
"Contents file={contents}", |
| 228 |
"Index file={index}", |
| 229 |
"Title={title}", |
| 230 |
"Default topic={default}", |
| 231 |
"Charset=UTF-8", |
| 232 |
""]) |
| 233 |
|
| 234 |
INDEXES_FILE_TEMPLATE = "\n".join([ |
| 235 |
'%(tab)s<li>', |
| 236 |
'%(tab)s\t<object type="text/sitemap">', |
| 237 |
'%(tab)s\t\t<param name="Name" value="%(name)s">', |
| 238 |
'%(tab)s\t\t<param name="Local" value="%(file)s">', |
| 239 |
'%(tab)s\t</object>', |
| 240 |
'']) |
| 241 |
|
| 242 |
|
| 243 |
def process_input_stage5(options, files, extrafiles): |
| 244 |
"""Generate the index and table of contents""" |
| 245 |
|
| 246 |
# write header file |
| 247 |
header_file_name = os.path.join(files['stage5'], "header.hhp") |
| 248 |
with open(header_file_name, mode="w") as header_file: |
| 249 |
header_file.write(HTB_HEADER_TEMPLATE.format( |
| 250 |
contents="contents.hhc", |
| 251 |
index="index.hhk", |
| 252 |
title="wxLauncher User Guide", |
| 253 |
default="index.htm", |
| 254 |
)) |
| 255 |
|
| 256 |
# generate both index and table of contents |
| 257 |
tocfile_name = os.path.join(files['stage5'], "contents.hhc") |
| 258 |
tocfile = open(tocfile_name, mode="w") |
| 259 |
tocfile.write("<ul>\n") |
| 260 |
toclevel = 1 |
| 261 |
|
| 262 |
indexfile_name = os.path.join(files['stage5'], "index.hhk") |
| 263 |
indexfile = open(indexfile_name, mode="w") |
| 264 |
indexfile.write("<ul>\n") |
| 265 |
|
| 266 |
help_files = generate_file_list(files['stage4'], ".stage4") |
| 267 |
|
| 268 |
last_path_list = [] |
| 269 |
for path, dirs, thefiles in os.walk(files['stage4']): |
| 270 |
"""new directory. If the directory is not the base directory then check if there will be an index.htm for this directory. |
| 271 |
If there is an index.htm then parse it for the title so that the subsection will have the title of the index.htm as the name of the subsection. |
| 272 |
If there is no index.htm then make a default one and use the name of the directory as the subsection name. |
| 273 |
|
| 274 |
Note that this algorithm requires that os.walk generates the names in alphabetical order.""" |
| 275 |
# relativize directory path for being in the archive |
| 276 |
path_list = find_path_in_archive(path, files['stage4']) |
| 277 |
logging.debug("Processing directory '%s'", os.path.sep.join(path_list)) |
| 278 |
|
| 279 |
if len(path_list) == 1 and path_list[0] == '': |
| 280 |
path_list = [] |
| 281 |
level = len(path_list) |
| 282 |
|
| 283 |
# parse the index.help to get the name of the section |
| 284 |
index_file_name = os.path.join(path, "index.stage4") |
| 285 |
# relativize filename for being in the archive |
| 286 |
index_in_archive = change_filename(index_file_name, ".htm", files['stage4'], ".", False) |
| 287 |
index_in_archive = index_in_archive.replace(os.path.sep, |
| 288 |
"/") # make the separators the same so that it doesn't matter what platform the launcher is built on. |
| 289 |
# find the title |
| 290 |
outindex_name = change_filename(index_file_name, ".htm", files['stage4'], files['stage5']) |
| 291 |
outindex = open(outindex_name, mode="w") |
| 292 |
|
| 293 |
inindex = open(index_file_name, mode="r") |
| 294 |
input = inindex.read() |
| 295 |
inindex.close() |
| 296 |
|
| 297 |
parser = Stage5Parser(file=outindex) |
| 298 |
parser.feed(input) |
| 299 |
parser.close() |
| 300 |
outindex.close() |
| 301 |
|
| 302 |
tocfile.write( |
| 303 |
generate_sections(path_list, last_path_list, index_filename=index_in_archive, section_title=parser.title)) |
| 304 |
last_path_list = path_list |
| 305 |
|
| 306 |
if level > 0: |
| 307 |
# remove index.htm from thefiles because they are used by the section names |
| 308 |
try: |
| 309 |
thefiles.remove('index.stage4') |
| 310 |
except ValueError: |
| 311 |
logging.warning("Directory %s does not have an index.help", |
| 312 |
os.path.sep.join(path_list)) |
| 313 |
|
| 314 |
for file in thefiles: |
| 315 |
full_filename = os.path.join(path, file) |
| 316 |
# relativize filename for being in the archive |
| 317 |
filename_in_archive = change_filename(full_filename, ".htm", files['stage4'], ".", False) |
| 318 |
# find the title |
| 319 |
outfile_name = change_filename(full_filename, ".htm", files['stage4'], files['stage5']) |
| 320 |
outfile = open(outfile_name, mode="w") |
| 321 |
|
| 322 |
infile = open(full_filename, mode="r") |
| 323 |
input = infile.read() |
| 324 |
infile.close() |
| 325 |
|
| 326 |
parser = Stage5Parser(file=outfile) |
| 327 |
parser.feed(input) |
| 328 |
parser.close() |
| 329 |
outfile.close() |
| 330 |
|
| 331 |
filename_in_indexes = filename_in_archive.replace('\\', '/') |
| 332 |
|
| 333 |
tocfile.write(INDEXES_FILE_TEMPLATE % { |
| 334 |
"tab": "\t"*level, |
| 335 |
"name": parser.title, |
| 336 |
"file": filename_in_indexes |
| 337 |
}) |
| 338 |
|
| 339 |
indexfile.write(INDEXES_FILE_TEMPLATE % { |
| 340 |
"tab": "\t", |
| 341 |
"name": parser.title, |
| 342 |
"file": filename_in_indexes |
| 343 |
}) |
| 344 |
|
| 345 |
tocfile.write(generate_sections([], last_path_list)) |
| 346 |
tocfile.write("</ul>\n") |
| 347 |
tocfile.close() |
| 348 |
indexfile.close() |
| 349 |
|
| 350 |
# copy the extra files (i.e. images) from stage3 |
| 351 |
for extrafilename in extrafiles: |
| 352 |
logging.debug(" Copying: %s", extrafilename) |
| 353 |
dst = change_filename(extrafilename, None, orginaldir=files['stage3'], destdir=files['stage5']) |
| 354 |
shutil.copy2(extrafilename, dst) |
| 355 |
|
| 356 |
|
| 357 |
def generate_sections(path_list, last_path_list, basetab=0, orginal_path_list=None, index_filename=None, |
| 358 |
section_title=None): |
| 359 |
"""Return the string that will allow me to write in the correct section.""" |
| 360 |
logging.debug(" generate_sections(%s, %s, %d, %s)", str(path_list), str(last_path_list), basetab, |
| 361 |
orginal_path_list) |
| 362 |
|
| 363 |
if orginal_path_list == None: |
| 364 |
orginal_path_list = path_list |
| 365 |
|
| 366 |
if len(path_list) > 0 and len(last_path_list) > 0 and path_list[0] == last_path_list[0]: |
| 367 |
logging.debug(" matches don't need to do anything") |
| 368 |
return generate_sections(path_list[1:], last_path_list[1:], basetab + 1, orginal_path_list, index_filename, |
| 369 |
section_title) |
| 370 |
|
| 371 |
elif len(path_list) > 0 and len(last_path_list) > 0: |
| 372 |
logging.debug(" go down then up") |
| 373 |
s = generate_sections([], last_path_list, basetab, orginal_path_list, index_filename, section_title) |
| 374 |
s += generate_sections(path_list, [], basetab, orginal_path_list, index_filename, section_title) |
| 375 |
return s |
| 376 |
|
| 377 |
elif len(path_list) > 0 and len(last_path_list) == 0: |
| 378 |
logging.debug(" go up (deeper)") |
| 379 |
if index_filename == None: |
| 380 |
raise Exception("index_filename is None") |
| 381 |
if section_title == None: |
| 382 |
title = path_list[len(path_list) - 1] |
| 383 |
else: |
| 384 |
title = section_title |
| 385 |
|
| 386 |
s = "" |
| 387 |
for path in path_list: |
| 388 |
s += """%(tab)s<li> <object type="text/sitemap">\n%(tab)s <param name="Name" value="%(name)s">\n%(tab)s <param name="Local" value="%(file)s">\n%(tab)s </object>\n%(tab)s <ul>\n""" % { |
| 389 |
"tab": " " * basetab, |
| 390 |
"name": title, |
| 391 |
"file": index_filename} |
| 392 |
|
| 393 |
return s |
| 394 |
elif len(path_list) == 0 and len(last_path_list) > 0: |
| 395 |
logging.debug(" do down") |
| 396 |
s = "" |
| 397 |
basetab += len(last_path_list) |
| 398 |
for path in last_path_list: |
| 399 |
s += """%(tab)s</ul>\n""" % {'tab': " " * (basetab)} |
| 400 |
basetab -= 1 |
| 401 |
return s |
| 402 |
|
| 403 |
elif len(path_list) == 0 and len(last_path_list) == 0: |
| 404 |
logging.debug(" both lists empty, doing nothing") |
| 405 |
return '' |
| 406 |
|
| 407 |
else: |
| 408 |
raise Exception("Should never get here") |
| 409 |
return "" |
| 410 |
|
| 411 |
|
| 412 |
def find_path_in_archive(pLeft, pRight): |
| 413 |
"""Return the parts of the path that is in 'pLeft' but not in 'pRight'""" |
| 414 |
pLeft = os.path.normpath(pLeft) |
| 415 |
pRight = os.path.normpath(pRight) |
| 416 |
|
| 417 |
lLeft = pLeft.split(os.path.sep) |
| 418 |
lRight = pRight.split(os.path.sep) |
| 419 |
|
| 420 |
return find_path_in_archive_helper(lLeft, lRight) |
| 421 |
|
| 422 |
|
| 423 |
def find_path_in_archive_helper(lLeft, lRight): |
| 424 |
logging.debug("make_path_in_archive_helper(%s, %s)", |
| 425 |
str(lLeft), str(lRight)) |
| 426 |
if len(lLeft) > 0 and len(lRight) > 0 and lLeft[0] == lRight[0]: |
| 427 |
return find_path_in_archive_helper(lLeft[1:], lRight[1:]) |
| 428 |
elif len(lLeft) > 0 and len(lRight) == 0: |
| 429 |
return lLeft |
| 430 |
else: |
| 431 |
return [] |
| 432 |
|
| 433 |
|
| 434 |
def process_input_stage6(options, stage_dirs): |
| 435 |
# make sure that the directories exist before creating file |
| 436 |
outfile_path = os.path.dirname(options.outfile) |
| 437 |
if os.path.exists(outfile_path): |
| 438 |
if os.path.isdir(outfile_path): |
| 439 |
pass |
| 440 |
else: |
| 441 |
log.error(" %s exists but is not a directory", outfile_path) |
| 442 |
else: |
| 443 |
os.makedirs(outfile_path) |
| 444 |
|
| 445 |
outzip = zipfile.ZipFile(options.outfile, mode="w", compression=zipfile.ZIP_DEFLATED) |
| 446 |
|
| 447 |
for path, dirs, files in os.walk(stage_dirs['stage5']): |
| 448 |
for filename in files: |
| 449 |
full_filename = os.path.join(path, filename) |
| 450 |
arcname = change_filename(full_filename, None, stage_dirs['stage5'], ".", False) |
| 451 |
logging.debug(" Added %s as %s", full_filename, arcname) |
| 452 |
outzip.write(full_filename, arcname) |
| 453 |
|
| 454 |
outzip.close() |
| 455 |
|
| 456 |
|
| 457 |
def generate_cpp_files(options, files, helparray): |
| 458 |
if options.carrayfilename == None: |
| 459 |
defaultname = os.path.join(os.path.dirname(options.outfile), "helplinks.cpp") |
| 460 |
logging.info("Filename for c-array has not been specified.") |
| 461 |
logging.info(" using default %s" % (defaultname)) |
| 462 |
options.carrayfilename = defaultname |
| 463 |
|
| 464 |
logging.getLogger('notices').info("Writing to %s", options.carrayfilename) |
| 465 |
outfile = open(options.carrayfilename, mode="w") |
| 466 |
outfile.write("// Generated by scripts/onlinehelpmaker.py\n") |
| 467 |
outfile.write("// GENERATED FILE - DO NOT EDIT\n") |
| 468 |
for id, location in helparray: |
| 469 |
outfile.write("""{%s,_T("%s")},\n""" % |
| 470 |
(id, location.replace(os.path.sep, "/"))) |
| 471 |
|
| 472 |
outfile.close() |