Debian Codegraph @

scripts/ohm/jobs/build.py

raw
1 from __future__ import absolute_import
2
3 import logging
4 import os
5 import shutil
6 import zipfile
7
8 import markdown
9
10 from ..help_parsers.helpparsers import (Stage2Parser, Stage3Parser,
11 Stage4Parser, Stage5Parser)
12 from ..utilfunctions import change_filename
13
14
15 def build(args):
16 """Compiles the files in args.indir to the archive output args.outfile.
17
18 Compiled in several stages:
19 - stage1: Transform all input files with markdown placing the results
20 into args.temp+"/stage1".
21 - stage2: Parses and strips the output of stage1 to build the list
22 that will be made into the c-array that contains the compiled names for
23 the detailed help of each control.
24 - stage3: Parses the output of stage2 to grab the images that are
25 referred to in the the output of stage1
26 - stage4: Parses the output of stage1 to fix the relative hyperlinks in
27 the output so that they will refer correctly to the correct files when
28 in output file.
29 - stage5: Generate the index and table of contents for the output file
30 from the output of stage4.
31 - stage6: Zip up the output of stage5 and put it in the output location.
32 """
33 notices = logging.getLogger('notices')
34 notices.info("Building...")
35 logging.debug("Using '%s' as working directory", args.temp)
36 logging.debug("Using '%s' as output file", args.outfile)
37 logging.debug("Using '%s' as input directory", args.indir)
38
39 files = generate_paths(args)
40
41 if should_build(args):
42 _do_build(args, files, notices)
43 else:
44 notices.info(" Up to date.")
45
46
47 def _do_build(args, files, notices):
48 help_array = list()
49 extra_files = list()
50 notices.info(" Processing input files:")
51 for file_name in generate_file_list(args.indir, ".help"):
52 notices.info(" %s", file_name)
53 logging.info(" Stage 1")
54 name1 = process_input_stage1(file_name, args, files)
55
56 logging.info(" Stage 2")
57 name2 = process_input_stage2(name1, args, files, help_array)
58
59 logging.info(" Stage 3")
60 name3 = process_input_stage3(name2, args, files, extra_files)
61
62 logging.info(" Stage 4")
63 name4 = process_input_stage4(name3, args, files)
64 logging.info(" Stage 5")
65 process_input_stage5(args, files, extra_files)
66 logging.info(" Stage 6")
67 process_input_stage6(args, files)
68 logging.info(" Generating .cpp files")
69 generate_cpp_files(args, files, help_array)
70 notices.info("....Done.")
71
72
73 def generate_paths(options):
74 """Generates the names of the paths that will be needed by the compiler.
75
76 Generated names will be returned as a dictionary."""
77 paths = dict()
78
79 for i in range(1, 7):
80 stage_name = 'stage{0}'.format(i)
81 paths[stage_name] = os.path.join(options.temp, stage_name)
82
83 for path in paths.values():
84 if not os.path.exists(path):
85 logging.debug(" Making %s", path)
86 os.makedirs(path)
87
88 return paths
89
90
91 def should_build(options):
92 """Should we build the output file?
93 :param options arguments from command line
94 :return True if the output file should be built/rebuilt
95 """
96 logger = logging.getLogger('should_build')
97 if options.always_build:
98 logger.info(" Always building")
99 return True
100 elif not os.path.exists(options.outfile):
101 logger.info(" Outfile does not exist")
102 return True
103 elif options.carrayfilename and not os.path.exists(options.carrayfilename):
104 logger.info(" .cpp file does not exist")
105 return True
106 elif check_source_newer_than_outfile(options):
107 logger.info(" Source files are newer than output")
108 return True
109 return False
110
111
112 def check_source_newer_than_outfile(options):
113 try:
114 outfile_time = os.path.getmtime(options.outfile)
115
116 for d_path, d_names, f_names in os.walk(options.indir):
117 for f_name in f_names:
118 f_path = os.path.join(d_path, f_name)
119 if os.path.getmtime(f_path) > outfile_time:
120 logging.info(
121 "%s has been changed since outfile. Causing build",
122 f_path)
123 return True
124 elif os.path.getctime(f_path) > outfile_time:
125 logging.info(
126 "%s has been created since outfile. Causing build",
127 f_path)
128 return True
129
130 except OSError as e:
131 logging.exception("OS does not like %d. Forcing build.", e.filename)
132 return True
133 return False
134
135
136 def generate_file_list(directory, extension):
137 for path, dirs, files in os.walk(directory):
138 for file_name in files:
139 if os.path.splitext(file_name)[1] == extension:
140 yield os.path.join(path, file_name)
141
142
143 def process_input_stage1(file, options, files):
144 infile = open(file, mode="r")
145 input = infile.read()
146 infile.close()
147
148 md = markdown.Markdown(
149 extensions=['markdown.extensions.toc']
150 )
151 output = md.convert(input)
152
153 outfile_name = change_filename(file, ".stage1", options.indir, files['stage1'])
154
155 outfile = open(outfile_name, mode="w")
156 outfile.write(output)
157 outfile.close()
158 return outfile_name
159
160
161 def process_input_stage2(file, options, files, helparray):
162 infile = open(file, mode="r")
163 input = infile.read()
164 infile.close()
165
166 outname = change_filename(file, ".stage2", files['stage1'], files['stage2'])
167 outfile = open(outname, mode="w")
168
169 parser = Stage2Parser(file=outfile)
170 parser.feed(input)
171 parser.close()
172 outfile.close()
173
174 if len(parser.control) > 0:
175 filename_in_archive = change_filename(outname, ".htm", files['stage2'], ".", makedirs=False)
176 for control in parser.control:
177 helparray.append((control, filename_in_archive))
178 logging.debug(" Control name %s", control)
179
180 return outname
181
182
183 def process_input_stage3(file, options, files, extrafiles):
184 infile = open(file, mode="r")
185 input = infile.read()
186 infile.close()
187
188 outname = change_filename(file, ".stage3", files['stage2'], files['stage3'])
189 outfile = open(outname, mode="w")
190
191 # figure out what subdirectory of the onlinehelp I am in
192 subdir = os.path.dirname(outname).replace(os.path.normpath(files['stage3']), "")
193 if subdir.startswith(os.path.sep):
194 subdir = subdir.replace(os.path.sep, "", 1) # I only want to remove the leading sep
195
196 parser = Stage3Parser(options, files, file=outfile, extrafiles=extrafiles, subdir=subdir)
197 parser.feed(input)
198 parser.close()
199 outfile.close()
200
201 return outname
202
203
204 def process_input_stage4(file, options, files):
205 """Fix the a tags so that they point to the htm files that will be in the output archive."""
206 infile = open(file, mode="r")
207 input = infile.read()
208 infile.close()
209
210 outname = change_filename(file, ".stage4", files['stage3'], files['stage4'])
211 outfile = open(outname, mode="w")
212
213 # figure out what subdirectory of the onlinehelp I am in
214 subdir = os.path.dirname(outname).replace(os.path.normpath(files['stage4']), "")
215 if subdir.startswith(os.path.sep):
216 subdir = subdir.replace(os.path.sep, "", 1) # I only want to remove the leading sep
217
218 parser = Stage4Parser(files=files, file=outfile, options=options,
219 subdir=subdir)
220 parser.feed(input)
221 parser.close()
222 outfile.close()
223
224 return outname
225
226 HTB_HEADER_TEMPLATE = "\n".join([
227 "Contents file={contents}",
228 "Index file={index}",
229 "Title={title}",
230 "Default topic={default}",
231 "Charset=UTF-8",
232 ""])
233
234 INDEXES_FILE_TEMPLATE = "\n".join([
235 '%(tab)s<li>',
236 '%(tab)s\t<object type="text/sitemap">',
237 '%(tab)s\t\t<param name="Name" value="%(name)s">',
238 '%(tab)s\t\t<param name="Local" value="%(file)s">',
239 '%(tab)s\t</object>',
240 ''])
241
242
243 def process_input_stage5(options, files, extrafiles):
244 """Generate the index and table of contents"""
245
246 # write header file
247 header_file_name = os.path.join(files['stage5'], "header.hhp")
248 with open(header_file_name, mode="w") as header_file:
249 header_file.write(HTB_HEADER_TEMPLATE.format(
250 contents="contents.hhc",
251 index="index.hhk",
252 title="wxLauncher User Guide",
253 default="index.htm",
254 ))
255
256 # generate both index and table of contents
257 tocfile_name = os.path.join(files['stage5'], "contents.hhc")
258 tocfile = open(tocfile_name, mode="w")
259 tocfile.write("<ul>\n")
260 toclevel = 1
261
262 indexfile_name = os.path.join(files['stage5'], "index.hhk")
263 indexfile = open(indexfile_name, mode="w")
264 indexfile.write("<ul>\n")
265
266 help_files = generate_file_list(files['stage4'], ".stage4")
267
268 last_path_list = []
269 for path, dirs, thefiles in os.walk(files['stage4']):
270 """new directory. If the directory is not the base directory then check if there will be an index.htm for this directory.
271 If there is an index.htm then parse it for the title so that the subsection will have the title of the index.htm as the name of the subsection.
272 If there is no index.htm then make a default one and use the name of the directory as the subsection name.
273
274 Note that this algorithm requires that os.walk generates the names in alphabetical order."""
275 # relativize directory path for being in the archive
276 path_list = find_path_in_archive(path, files['stage4'])
277 logging.debug("Processing directory '%s'", os.path.sep.join(path_list))
278
279 if len(path_list) == 1 and path_list[0] == '':
280 path_list = []
281 level = len(path_list)
282
283 # parse the index.help to get the name of the section
284 index_file_name = os.path.join(path, "index.stage4")
285 # relativize filename for being in the archive
286 index_in_archive = change_filename(index_file_name, ".htm", files['stage4'], ".", False)
287 index_in_archive = index_in_archive.replace(os.path.sep,
288 "/") # make the separators the same so that it doesn't matter what platform the launcher is built on.
289 # find the title
290 outindex_name = change_filename(index_file_name, ".htm", files['stage4'], files['stage5'])
291 outindex = open(outindex_name, mode="w")
292
293 inindex = open(index_file_name, mode="r")
294 input = inindex.read()
295 inindex.close()
296
297 parser = Stage5Parser(file=outindex)
298 parser.feed(input)
299 parser.close()
300 outindex.close()
301
302 tocfile.write(
303 generate_sections(path_list, last_path_list, index_filename=index_in_archive, section_title=parser.title))
304 last_path_list = path_list
305
306 if level > 0:
307 # remove index.htm from thefiles because they are used by the section names
308 try:
309 thefiles.remove('index.stage4')
310 except ValueError:
311 logging.warning("Directory %s does not have an index.help",
312 os.path.sep.join(path_list))
313
314 for file in thefiles:
315 full_filename = os.path.join(path, file)
316 # relativize filename for being in the archive
317 filename_in_archive = change_filename(full_filename, ".htm", files['stage4'], ".", False)
318 # find the title
319 outfile_name = change_filename(full_filename, ".htm", files['stage4'], files['stage5'])
320 outfile = open(outfile_name, mode="w")
321
322 infile = open(full_filename, mode="r")
323 input = infile.read()
324 infile.close()
325
326 parser = Stage5Parser(file=outfile)
327 parser.feed(input)
328 parser.close()
329 outfile.close()
330
331 filename_in_indexes = filename_in_archive.replace('\\', '/')
332
333 tocfile.write(INDEXES_FILE_TEMPLATE % {
334 "tab": "\t"*level,
335 "name": parser.title,
336 "file": filename_in_indexes
337 })
338
339 indexfile.write(INDEXES_FILE_TEMPLATE % {
340 "tab": "\t",
341 "name": parser.title,
342 "file": filename_in_indexes
343 })
344
345 tocfile.write(generate_sections([], last_path_list))
346 tocfile.write("</ul>\n")
347 tocfile.close()
348 indexfile.close()
349
350 # copy the extra files (i.e. images) from stage3
351 for extrafilename in extrafiles:
352 logging.debug(" Copying: %s", extrafilename)
353 dst = change_filename(extrafilename, None, orginaldir=files['stage3'], destdir=files['stage5'])
354 shutil.copy2(extrafilename, dst)
355
356
357 def generate_sections(path_list, last_path_list, basetab=0, orginal_path_list=None, index_filename=None,
358 section_title=None):
359 """Return the string that will allow me to write in the correct section."""
360 logging.debug(" generate_sections(%s, %s, %d, %s)", str(path_list), str(last_path_list), basetab,
361 orginal_path_list)
362
363 if orginal_path_list == None:
364 orginal_path_list = path_list
365
366 if len(path_list) > 0 and len(last_path_list) > 0 and path_list[0] == last_path_list[0]:
367 logging.debug(" matches don't need to do anything")
368 return generate_sections(path_list[1:], last_path_list[1:], basetab + 1, orginal_path_list, index_filename,
369 section_title)
370
371 elif len(path_list) > 0 and len(last_path_list) > 0:
372 logging.debug(" go down then up")
373 s = generate_sections([], last_path_list, basetab, orginal_path_list, index_filename, section_title)
374 s += generate_sections(path_list, [], basetab, orginal_path_list, index_filename, section_title)
375 return s
376
377 elif len(path_list) > 0 and len(last_path_list) == 0:
378 logging.debug(" go up (deeper)")
379 if index_filename == None:
380 raise Exception("index_filename is None")
381 if section_title == None:
382 title = path_list[len(path_list) - 1]
383 else:
384 title = section_title
385
386 s = ""
387 for path in path_list:
388 s += """%(tab)s<li> <object type="text/sitemap">\n%(tab)s <param name="Name" value="%(name)s">\n%(tab)s <param name="Local" value="%(file)s">\n%(tab)s </object>\n%(tab)s <ul>\n""" % {
389 "tab": " " * basetab,
390 "name": title,
391 "file": index_filename}
392
393 return s
394 elif len(path_list) == 0 and len(last_path_list) > 0:
395 logging.debug(" do down")
396 s = ""
397 basetab += len(last_path_list)
398 for path in last_path_list:
399 s += """%(tab)s</ul>\n""" % {'tab': " " * (basetab)}
400 basetab -= 1
401 return s
402
403 elif len(path_list) == 0 and len(last_path_list) == 0:
404 logging.debug(" both lists empty, doing nothing")
405 return ''
406
407 else:
408 raise Exception("Should never get here")
409 return ""
410
411
412 def find_path_in_archive(pLeft, pRight):
413 """Return the parts of the path that is in 'pLeft' but not in 'pRight'"""
414 pLeft = os.path.normpath(pLeft)
415 pRight = os.path.normpath(pRight)
416
417 lLeft = pLeft.split(os.path.sep)
418 lRight = pRight.split(os.path.sep)
419
420 return find_path_in_archive_helper(lLeft, lRight)
421
422
423 def find_path_in_archive_helper(lLeft, lRight):
424 logging.debug("make_path_in_archive_helper(%s, %s)",
425 str(lLeft), str(lRight))
426 if len(lLeft) > 0 and len(lRight) > 0 and lLeft[0] == lRight[0]:
427 return find_path_in_archive_helper(lLeft[1:], lRight[1:])
428 elif len(lLeft) > 0 and len(lRight) == 0:
429 return lLeft
430 else:
431 return []
432
433
434 def process_input_stage6(options, stage_dirs):
435 # make sure that the directories exist before creating file
436 outfile_path = os.path.dirname(options.outfile)
437 if os.path.exists(outfile_path):
438 if os.path.isdir(outfile_path):
439 pass
440 else:
441 log.error(" %s exists but is not a directory", outfile_path)
442 else:
443 os.makedirs(outfile_path)
444
445 outzip = zipfile.ZipFile(options.outfile, mode="w", compression=zipfile.ZIP_DEFLATED)
446
447 for path, dirs, files in os.walk(stage_dirs['stage5']):
448 for filename in files:
449 full_filename = os.path.join(path, filename)
450 arcname = change_filename(full_filename, None, stage_dirs['stage5'], ".", False)
451 logging.debug(" Added %s as %s", full_filename, arcname)
452 outzip.write(full_filename, arcname)
453
454 outzip.close()
455
456
457 def generate_cpp_files(options, files, helparray):
458 if options.carrayfilename == None:
459 defaultname = os.path.join(os.path.dirname(options.outfile), "helplinks.cpp")
460 logging.info("Filename for c-array has not been specified.")
461 logging.info(" using default %s" % (defaultname))
462 options.carrayfilename = defaultname
463
464 logging.getLogger('notices').info("Writing to %s", options.carrayfilename)
465 outfile = open(options.carrayfilename, mode="w")
466 outfile.write("// Generated by scripts/onlinehelpmaker.py\n")
467 outfile.write("// GENERATED FILE - DO NOT EDIT\n")
468 for id, location in helparray:
469 outfile.write("""{%s,_T("%s")},\n""" %
470 (id, location.replace(os.path.sep, "/")))
471
472 outfile.close()