Debian Codegraph @

csb/build.py

raw ◀ ▶
2 Debian patches touch this file
1 """
2 CSB build related tools and programs.
3
4 When executed as a program, this module will run the CSB Build Console and
5 build the source tree it belongs to. The source tree is added at the
6 B{beginning} of sys.path to make sure that all subsequent imports from the
7 Test and Doc consoles will import the right thing (think of multiple CSB
8 packages installed on the same server).
9
10 Here is how to build, test and package the whole project::
11
12 $ hg clone https://hg.codeplex.com/csb CSB
13 $ CSB/csb/build.py -o <output directory>
14
15 The Console can also be imported and instantiated as a regular Python class.
16 In this case the Console again builds the source tree it is part of, but
17 sys.path will remain intact. Therefore, the Console will assume that all
18 modules currently in memory, as well as those that can be subsequently imported
19 by the Console itself, belong to the same CSB package.
20
21 @note: The CSB build services no longer support the option to build external
22 source trees.
23 @see: [CSB 0000038]
24 """
25 from __future__ import print_function
26
27 import os
28 import sys
29 import getopt
30 import traceback
31 import compileall
32
33 if os.path.basename(__file__) == '__init__.py':
34 PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
35 else:
36 PARENT = os.path.abspath(os.path.dirname(__file__))
37
38 ROOT = 'csb'
39 SOURCETREE = os.path.abspath(os.path.join(PARENT, ".."))
40
41 if __name__ == '__main__':
42
43 # make sure "import io" imports the built in module, not csb.io
44 # io is required by tarfile
45 for path in sys.path:
46 if path.startswith(SOURCETREE):
47 sys.path.remove(path)
48
49 import io
50 assert hasattr(io, 'BufferedIOBase')
51
52 sys.path = [SOURCETREE] + sys.path
53
54
55 """
56 It is now safe to import any modules
57 """
•58 import importlib
59 import shutil
60 import tarfile
61
62 import csb
63
64 from abc import ABCMeta, abstractmethod
65 from csb.io import Shell
66
67
68 class BuildTypes(object):
69 """
70 Enumeration of build types.
71 """
72
73 SOURCE = 'source'
74 BINARY = 'binary'
75
76 _du = { SOURCE: 'sdist', BINARY: 'bdist' }
77
78 @staticmethod
79 def get(key):
80 try:
81 return BuildTypes._du[key]
82 except KeyError:
83 raise ValueError('Unhandled build type: {0}'.format(key))
84
85
86 class Console(object):
87 """
88 CSB Build Bot. Run with -h for usage.
89
90 @param output: build output directory
91 @type output: str
92 @param verbosity: verbosity level
93 @type verbosity: int
94
95 @note: The build console automatically detects and builds the csb package
96 it belongs to. You cannot build a different source tree with it.
97 See the module documentation for more info.
98 """
99
100 PROGRAM = __file__
101
102 USAGE = r"""
103 CSB Build Console: build, test and package the entire csb project.
104
105 Usage:
106 python {program} -o output [-v verbosity] [-t type] [-h]
107
108 Options:
109 -o output Build output directory
110 -v verbosity Verbosity level, default is 1
111 -t type Build type:
112 source - build source code distribution (default)
113 binary - build executable
114 -h, --help Display this help
115 """
116
117 def __init__(self, output='.', verbosity=1, buildtype=BuildTypes.SOURCE):
118
119 self._input = None
120 self._output = None
121 self._temp = None
122 self._docs = None
123 self._apidocs = None
124 self._root = None
125 self._verbosity = None
126 self._type = buildtype
127 self._dist = BuildTypes.get(buildtype)
128
129 if os.path.join(SOURCETREE, ROOT) != PARENT:
130 raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT))
131 self._input = SOURCETREE
132
133 self._success = True
134
135 self.output = output
136 self.verbosity = verbosity
137
138 @property
139 def input(self):
140 return self._input
141
142 @property
143 def output(self):
144 return self._output
145 @output.setter
146 def output(self, value):
147 #value = os.path.dirname(value)
148 self._output = os.path.abspath(value)
149 self._temp = os.path.join(self._output, 'build')
150 self._docs = os.path.join(self._temp, 'docs')
151 self._apidocs = os.path.join(self._docs, 'api')
152 self._root = os.path.join(self._temp, ROOT)
153
154 @property
155 def verbosity(self):
156 return self._verbosity
157 @verbosity.setter
158 def verbosity(self, value):
159 self._verbosity = int(value)
160
161 def build(self):
162 """
163 Run the console.
164 """
165 self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE))
166
167 self._init()
168 v = self._revision()
169 self._doc(v)
170 self._test()
171
172 self._compile()
173 vn = self._package()
174
175 if self._success:
176 self.log('\n# Done ({0}).\n'.format(vn.full))
177 return True
178
179 self.log('\n# Build failed.\n')
180 return False
181
182 def log(self, message, level=1, ending='\n'):
183
184 if self._verbosity >= level:
185 sys.stdout.write(message)
186 sys.stdout.write(ending)
187 sys.stdout.flush()
188
189 def _init(self):
190 """
191 Collect all required stuff in the output folder.
192 """
193 self.log('# Preparing the file system...')
194
195 if not os.path.exists(self._output):
196 self.log('Creating output directory {0}'.format(self._output), level=2)
197 os.mkdir(self._output)
198
199 if os.path.exists(self._temp):
200 self.log('Deleting existing temp directory {0}'.format(self._temp), level=2)
201 shutil.rmtree(self._temp)
202
203 self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2)
204 shutil.copytree(self._input, self._temp)
205
206 if os.path.exists(self._apidocs):
207 self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2)
208 shutil.rmtree(self._apidocs)
209 if not os.path.isdir(self._docs):
210 self.log('Creating docs directory {0}'.format(self._docs), level=2)
211 os.mkdir(self._docs)
212 self.log('Creating API docs directory {0}'.format(self._apidocs), level=2)
213 os.mkdir(self._apidocs)
214
215 def _revision(self):
216 """
217 Write the actual revision number to L{ROOT}.__version__
218 """
219 self.log('\n# Setting the most recent Revision Number...')
220 root = os.path.join(self._root, '__init__.py')
221
222 self.log('Retrieving revision number from {0}'.format(root), level=2)
223 rh = GitHandler(root)
224 revision = rh.read().revision
225
226 self.log('Writing back revision number {0}'.format(revision), level=2)
227 version = rh.write(revision, root)
228
229 self.log(' This is {0}.__version__ {1}'.format(ROOT, version), level=1)
230 csb.__version__ = version
231
232 return version
233
234 def _test(self):
235 """
236 Run tests. Also make sure the current environment loads all modules from
237 the input folder.
238 """
239 import csb.test
240 assert csb.test.__file__.startswith(self._input), 'csb.test not loaded from the input!' #@UndefinedVariable
241
242 from csb.test import unittest
243
244 newdata = os.path.join(self._temp, ROOT, 'test', 'data')
245 csb.test.Config.setDefaultDataRoot(newdata)
246 csb.test.Config.setDefaultGeneratedDataRoot(newdata)
247
248 self.log('\n# Updating all test pickles in {0} if necessary...'.format(newdata), level=2)
249 csb.test.Config().ensureDataConsistency()
250
251 self.log('\n# Running the Test Console...')
252
253 builder = csb.test.AnyTestBuilder()
254 suite = builder.loadTests(ROOT + '.test.cases.*')
255
256 runner = unittest.TextTestRunner(stream=sys.stderr, verbosity=self.verbosity)
257 result = runner.run(suite)
258 if result.wasSuccessful():
259 self.log('\n Passed all unit tests')
260 else:
261 self.log('\n DID NOT PASS: This build might be broken')
262 self._success = False
263
264 def _doc(self, version):
265 """
266 Build documentation in the output folder.
267 """
268 self.log('\n# Generating API documentation...')
269 try:
270 import epydoc.cli
271 except ImportError:
272 self.log('\n Skipped: epydoc is missing')
273 return
274
275 self.log('\n# Emulating ARGV for the Doc Builder...', level=2)
276 argv = sys.argv
277 sys.argv = ['epydoc', '--html', '-o', self._apidocs,
278 '--name', '{0} v{1}'.format(ROOT.upper(), version),
279 '--no-private', '--introspect-only', '--exclude', 'csb.test.cases',
280 '--css', os.path.join(self._temp, 'epydoc.css'),
281 '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning',
282 self._root]
283
284 if self._verbosity <= 1:
285 sys.argv.append('-q')
286
287 try:
288 epydoc.cli.cli()
289 sys.exit(0)
290 except SystemExit as ex:
•291 if ex.code == 0:
292 self.log('\n Passed all doc tests')
293 else:
294 if ex.code == 2:
295 self.log('\n DID NOT PASS: Generated docs might be broken')
296 self._success = False
297 else:
298 self.log('\n FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex))
299 self._success = False
300
301 self.log('\n# Restoring the previous ARGV...', level=2)
302 sys.argv = argv
303
304 def _compile(self):
305 """
306 Byte-compile all modules and packages.
307 """
308 self.log('\n# Byte-compiling all *.py files...')
309
310 quiet = self.verbosity <= 1
311 valid = compileall.compile_dir(self._root, quiet=quiet, force=True)
312
313 if not valid:
314 self.log('\n FAIL: Compilation error(s)\n')
315 self._success = False
316
317 def _package(self):
318 """
319 Make package.
320 """
321 self.log('\n# Configuring CWD and ARGV for the Setup...', level=2)
322 cwd = os.curdir
323 os.chdir(self._temp)
324
325 if self._verbosity > 1:
326 verbosity = '-v'
327 else:
328 verbosity = '-q'
329 argv = sys.argv
330 sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output]
331
332 self.log('\n# Building {0} distribution...'.format(self._type))
333 version = package = None
334
335 try:
•336 setup = importlib.machinery.SourceFileLoader('setupcsb', 'setup.py').load_module()
337 d = setup.build()
338 version = setup.VERSION
339 package = d.dist_files[0][2]
340
341 if self._type == BuildTypes.BINARY:
342 self._strip_source(package)
343
344 except SystemExit as ex:
•345 if ex.code != 0:
346 self.log('\n FAIL: Setup returned: \n\n{0}\n'.format(ex))
347 self._success = False
348 package = 'FAIL'
349
350 self.log('\n# Restoring the previous CWD and ARGV...', level=2)
351 os.chdir(cwd)
352 sys.argv = argv
353
354 self.log(' Packaged ' + package)
355 return version
356
357 def _strip_source(self, package, source='*.py'):
358 """
359 Delete plain text source code files from the package.
360 """
361 cwd = os.getcwd()
362
363 try:
364 tmp = os.path.join(self.output, 'tmp')
365 os.mkdir(tmp)
366
367 self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2)
368 os.chdir(tmp)
369
370 oldtar = tarfile.open(package, mode='r:gz')
371 oldtar.extractall(tmp)
372 oldtar.close()
373
374 newtar = tarfile.open(package, mode='w:gz')
375
376 try:
377 for i in os.walk('.'):
378 for fn in i[2]:
379 if fn.endswith('.py'):
380 module = os.path.join(i[0], fn);
381 if not os.path.isfile(module.replace('.py', '.pyc')):
382 raise ValueError('Missing bytecode for module {0}'.format(module))
383 else:
384 os.remove(os.path.join(i[0], fn))
385
386 for i in os.listdir('.'):
387 newtar.add(i)
388 finally:
389 newtar.close()
390
391 finally:
392 self.log('\n# Restoring the previous CWD...', level=2)
393 os.chdir(cwd)
394 if os.path.exists(tmp):
395 shutil.rmtree(tmp)
396
397 @staticmethod
398 def exit(message=None, code=0, usage=True):
399
400 if message:
401 print(message)
402 if usage:
403 print(Console.USAGE.format(program=Console.PROGRAM))
404
405 sys.exit(code)
406
407 @staticmethod
408 def run(argv=None):
409
410 if argv is None:
411 argv = sys.argv[1:]
412
413 output = None
414 verb = 1
415 buildtype = BuildTypes.SOURCE
416
417 try:
418 options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help'])
419
420 for option, value in options:
421 if option in('-h', '--help'):
422 Console.exit(message=None, code=0)
423 if option in('-o', '--output'):
424 if not os.path.isdir(value):
425 Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3)
426 output = value
427 if option in('-v', '--verbosity'):
428 try:
429 verb = int(value)
430 except ValueError:
431 Console.exit(message='E: Verbosity must be an integer.', code=4)
432 if option in('-t', '--type'):
433 if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]:
434 Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5)
435 buildtype = value
436 except getopt.GetoptError as oe:
437 Console.exit(message='E: ' + str(oe), code=1)
438
439 if not output:
440 Console.exit(code=1, usage=True)
441 else:
442 try:
443 ok = Console(output, verbosity=verb, buildtype=buildtype).build()
444 Console.exit(code=0 if ok else 66, usage=False)
445 except Exception as ex:
446 msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc())
447 Console.exit(message=msg, code=99, usage=False)
448
449
450 class RevisionError(RuntimeError):
451
452 def __init__(self, msg, code, cmd):
453
454 super(RevisionError, self).__init__(msg)
455 self.code = code
456 self.cmd = cmd
457
458 class RevisionHandler(object):
459 """
460 Determines the current repository revision number of a working copy.
461
462 @param path: a local checkout path to be examined
463 @type path: str
464 @param sc: name of the source control program
465 @type sc: str
466 """
467
468 def __init__(self, path, sc):
469
470 self._path = None
471 self._sc = None
472
473 if os.path.exists(path):
474 self._path = path
475 else:
476 raise IOError('Path not found: {0}'.format(path))
•477 if Shell.run([sc, 'help']).code == 0:
478 self._sc = sc
479 else:
480 raise RevisionError('Source control binary probe failed', None, None)
481
482 @property
483 def path(self):
484 return self._path
485
486 @property
487 def sc(self):
488 return self._sc
489
490 @abstractmethod
491 def read(self):
492 """
493 Return the current revision information.
494 @rtype: L{RevisionInfo}
495 """
496 pass
497
498 def write(self, revision, sourcefile):
499 """
500 Finalize the __version__ = major.minor.micro.{revision} tag.
501 Overwrite C{sourcefile} in place by substituting the {revision} macro.
502
503 @param revision: revision number to write to the source file.
504 @type revision: int
505 @param sourcefile: python source file with a __version__ tag, typically
506 "csb/__init__.py"
507 @type sourcefile: str
508
509 @return: sourcefile.__version__
510 """
511 content = open(sourcefile).readlines()
512
513 with open(sourcefile, 'w') as src:
514 for line in content:
515 if line.startswith('__version__'):
516 src.write(line.format(revision=revision))
517 else:
518 src.write(line)
519
520 self._delcache(sourcefile)
521 return imp.load_source('____source', sourcefile).__version__
522
523 def _run(self, cmd):
524
525 si = Shell.run(cmd)
526 if si.code > 0:
527 raise RevisionError('SC failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd)
528
529 return si.stdout.splitlines()
530
531 def _delcache(self, sourcefile):
532
533 compiled = os.path.splitext(sourcefile)[0] + '.pyc'
534 if os.path.isfile(compiled):
535 os.remove(compiled)
536
537 pycache = os.path.join(os.path.dirname(compiled), '__pycache__')
538 if os.path.isdir(pycache):
539 shutil.rmtree(pycache)
540
541 class SubversionHandler(RevisionHandler):
542
543 def __init__(self, path, sc='svn'):
544 super(SubversionHandler, self).__init__(path, sc)
545
546 def read(self):
547
548 cmd = '{0.sc} info {0.path} -R'.format(self)
549 maxrevision = None
550
551 for line in self._run(cmd):
552 if line.startswith('Revision:'):
553 rev = int(line[9:] .strip())
554 if rev > maxrevision:
555 maxrevision = rev
556
557 if maxrevision is None:
558 raise RevisionError('No revision number found', code=0, cmd=cmd)
559
560 return RevisionInfo(self.path, maxrevision)
561
562 class MercurialHandler(RevisionHandler):
563
564 def __init__(self, path, sc='hg'):
565
566 if os.path.isfile(path):
567 path = os.path.dirname(path)
568
569 super(MercurialHandler, self).__init__(path, sc)
570
571 def read(self):
572
573 wd = os.getcwd()
574 os.chdir(self.path)
575
576 try:
577 cmd = '{0.sc} log -r tip'.format(self)
578
579 revision = None
580 changeset = ''
581
582 for line in self._run(cmd):
583 if line.startswith('changeset:'):
584 items = line[10:].split(':')
585 revision = int(items[0])
586 changeset = items[1].strip()
587 break
588
589 if revision is None:
590 raise RevisionError('No revision number found', code=0, cmd=cmd)
591
592 return RevisionInfo(self.path, revision, changeset)
593
594 finally:
595 os.chdir(wd)
596
597
598 class GitHandler(RevisionHandler):
599
600 def __init__(self, path, sc='git'):
601
602 if os.path.isfile(path):
603 path = os.path.dirname(path)
604
605 super(GitHandler, self).__init__(path, sc)
606
607 def read(self):
608
609 wd = os.getcwd()
610 os.chdir(self.path)
611
612 try:
613 cmd = '{0.sc} rev-parse --short HEAD'.format(self)
614 revision = None
615
616 for line in self._run(cmd):
617 if line.strip():
618 revision = line.strip()
619 break
620
621 if revision is None:
622 raise RevisionError('No revision number found', code=0, cmd=cmd)
623
624 return RevisionInfo(self.path, revision, revision)
625
626 finally:
627 os.chdir(wd)
628
629 class RevisionInfo(object):
630
631 def __init__(self, item, revision, id=None):
632
633 self.item = item
634 self.revision = revision
635 self.id = id
636
637
638 def main():
639 Console.run()
640
641
642 if __name__ == '__main__':
643
644 main()