Debian Codegraph @

cmake/tribits/python_utils/download-cmake.py

raw
1 #!/usr/bin/env python3
2
3 # @HEADER
4 # *****************************************************************************
5 # TriBITS: Tribal Build, Integrate, and Test System
6 #
7 # Copyright 2013-2016 NTESS and the TriBITS contributors.
8 # SPDX-License-Identifier: BSD-3-Clause
9 # *****************************************************************************
10 # @HEADER
11
12 #
13 # General scripting support
14 #
15 # NOTE: Included first to check the version of python!
16 #
17
18 from GeneralScriptSupport import *
19
20 from CMakeBinaries import *
21
22 import glob
23 import re
24 import shutil
25 import string
26 import subprocess
27 import sys
28 import tarfile
29 import urllib2
30 import zipfile
31 import socket
32
33
34 #
35 # Discover domain (detect if in .sandia.gov) if possible.
36 # Use it to set default_http_proxy value appropriately.
37 #
38
39 default_http_proxy = ""
40 default_https_proxy = ""
41 hostname = ""
42
43 hostname = socket.getfqdn()
44
45 if hostname != '':
46 hostname = hostname.strip()
47 hostname = hostname.lower()
48
49 if hostname[-11:] == ".sandia.gov":
50 default_http_proxy = "http://wwwproxy.sandia.gov:80/"
51 default_https_proxy = "https://wwwproxy.sandia.gov:80/"
52
53
54 #
55 # Detailed documentation
56 #
57
58 usageHelp = r"""download-cmake.py [--install-dir=$HOME/cmake ...]
59
60 Script that downloads and installs one of the verified versions of CMake for
61 building Trilinos. The verified versions of CMake and their available
62 installers are indicated by the files CMakeVersions.py and CMakeBinaries.py.
63
64 There are multiple verified versions: you can choose which one you want to use
65 by passing the --installer-type argument to this script.
66
67 Valid values for --installer-type are:
68
69 'min' - minimum required version of CMake that can build Trilinos
70 'release' - default / recommended version, may be same as min
71 'rc' - latest release candidate, may be same as release
72 'dev' - latest development build
73
74 If your computer is in the sandia.gov domain, this script should automatically
75 detect that and use the correct value for --http-proxy to enable downloading
76 files from the web with http. If you need to pass a proxy in manually, pass
77 the fully qualified form, as in: --http-proxy="http://wwwproxy.sandia.gov:80/"
78 If you do not specify an --http-proxy and one is not automatically set due to
79 sandia.gov domain detection, then the http_proxy environment variable is
80 honored.
81
82 By default, if you just type:
83
84 $ SOME_DIR/download-cmake.py
85
86 then the directory download_area will get created in the local working
87 directory. It will contain the extracted install tree of a pre-built
88 binary CMake suitable for this platform. That extracted install tree may
89 be used directly from the download_area directory, or you may move or
90 copy it to any location of your choosing.
91
92 This script will also pull down the list of available installers from the CMake
93 web site and detect the latest available installer for the v2.8 and development
94 builds. After running the detection phase of the script, a new CMakeVersions.py
95 is written in the download_area directory. Periodically as needed, this
96 generated CMakeVersions.py should be committed as the official CMakeVersions.py
97 file in the same directory with this script.
98
99 You can control various parts of the process with the options (see below). Call
100 this script with -h or --help for a list of possible options.
101
102 Example using the --install-dir option:
103
104 $ SOME_DIR/download-cmake.py --install-dir=$HOME/cmake
105
106 This usage would install CMake and the other executables in $HOME/cmake/bin.
107 NOTE: You will have to update your PATH variable to include whatever directory
108 you choose to install CMake in.
109
110 NOTE: If you need to use sudo to install in some place that requires root
111 privileges, or if you need to install to a directory that already exists, do:
112
113 $ ./download-cmake.py --skip-install [other options]
114 $ cd download_area
115 $ sudo cp -r cmake-2.8.0-Linux-i386/ /usr/local
116
117 Alternatively, if you have sudo access you can do the install in one shot as:
118
119 $ sudo ./download-cmake.py --install-dir=/usr/local
120
121 After you have done a successful install, you may remove the downloaded files:
122
123 $ rm -r download_area
124
125 Enjoy CMake!
126
127 """
128
129
130 #
131 # Read in the commandline arguments
132 #
133
134 from optparse import OptionParser
135
136 clp = OptionParser(usage=usageHelp)
137
138 if sys.platform == 'darwin':
139 defInstallDir = "/Applications"
140 defSymlinksDir = "/usr/local/bin"
141 else:
142 defInstallDir = "/usr/local"
143 defSymlinksDir = ""
144
145 clp.add_option(
146 "--all-platforms", dest="allPlatforms", action="store_true", default=False,
147 help="Download and/or extract tarballs for all platforms (default = just this platform)" )
148
149 clp.add_option(
150 "--http-proxy", dest="httpProxy", type="string", default=default_http_proxy,
151 help="Proxy in the form 'http://server:port/' - use if you are behind a firewall with respect to downloading from http://www.cmake.org (default = \""+default_http_proxy+"\")." )
152
153 clp.add_option(
154 "--https-proxy", dest="httpsProxy", type="string", default=default_https_proxy,
155 help="Proxy in the form 'https://server:port/' - use if you are behind a firewall with respect to downloading from https://www.cmake.org (default = \""+default_https_proxy+"\")." )
156
157 clp.add_option(
158 "--install-dir", dest="installDir", type="string", default=defInstallDir,
159 help="The install directory for CMake (default = "+defInstallDir+"." )
160
161 clp.add_option(
162 "--installer-type", dest="installerType", type="string", default="release",
163 help="Which CMake installer: min, release, rc, or dev (default = release)." )
164
165 clp.add_option(
166 "--skip-detect", dest="skipDetect", action="store_true", default=False,
167 help="Skip detecting the latest available builds" )
168
169 clp.add_option(
170 "--skip-download", dest="skipDownload", action="store_true", default=False,
171 help="Skip the download step" )
172
173 clp.add_option(
174 "--skip-extract", dest="skipExtract", action="store_true", default=False,
175 help="Skip the extract step" )
176
177 clp.add_option(
178 "--skip-install", dest="skipInstall", action="store_true", default=False,
179 help="Skip the install step" )
180
181 clp.add_option(
182 "--symlinks", dest="symlinks", action="store_true", default=False,
183 help="Create symlinks to the installed CMake executables in --symlinks-dir" )
184
185 clp.add_option(
186 "--symlinks-dir", dest="symlinksDir", type="string", default=defSymlinksDir,
187 help="The directory in which to create symlinks to CMake executables (default = "+defSymlinksDir+")." )
188
189 (options, args) = clp.parse_args()
190
191
192 # "Latest available CMake build" is defined as the most recent build on the
193 # server in vdir with pre-built binary tarballs available for Mac, Linux and
194 # Windows.
195 #
196 def DetectLatestCMakeBuilds(basedir, baseurl, vdir):
197 url = ''.join([baseurl, "/", vdir, "/?C=M;O=D"])
198 filename = ''.join([basedir, "/CMake_", vdir, ".html"])
199
200 try:
201 createDir(basedir)
202 except:
203 if not os.path.exists(basedir):
204 raise
205
206 print "Querying " + url + "..."
207
208 proxyDict = {}
209 if not options.httpProxy == "":
210 proxyDict["http"] = options.httpProxy
211
212 if not options.httpsProxy == "":
213 proxyDict["https"] = options.httpsProxy
214
215 if len(proxyDict) != 0:
216 proxies = urllib2.ProxyHandler(proxyDict)
217 else:
218 #no proxies set so use proxies from environment
219 proxies = urllib2.ProxyHandler()
220
221 opener = urllib2.build_opener(proxies)
222 url_file = opener.open(url)
223
224 outfile = open(filename, "w")
225 outfile.write(url_file.read())
226 outfile.close()
227
228 print "Detecting ..."
229
230 lines = []
231 regex = re.compile(
232 "href.*cmake-[0-9.]+(-rc[0-9]+)*(-g[0-9a-fA-F]+)*-(Darwin-universal.tar.gz|Linux-i386.tar.gz|win32-x86.zip)"
233 )
234
235 f = open(filename)
236 alllines = f.readlines()
237 for line in alllines:
238 if regex.search(line):
239 lines.append(line)
240
241 count = 0
242 found = 0
243 version_iterator = ""
244
245 versionRegEx = re.compile(r'.*-([0-9.]+-rc[0-9]+|[0-9.]+-g[0-9a-fA-F]+|[0-9.]+)-.*')
246 dateRegEx = re.compile(r'^[0-9].[0-9].([0-9.]+-rc[0-9]+|[0-9.]+-g[0-9a-fA-F]+|[0-9.]+)$')
247 hrefRegEx = re.compile(r'^.*href="([^"]+)".*$')
248
249 for line in lines:
250 version = versionRegEx.match(line).group(1)
251
252 if version == "" or version == line:
253 print "error: line does not match version extraction regex"
254 print " line: [" + line + "]"
255 sys.exit(1)
256
257 date = dateRegEx.match(version).group(1)
258
259 # l, m, w == found an installer for Linux, Mac, Windows respectively
260 # When encountering a new version, reset back to zeroes...
261 #
262 if(version_iterator != version):
263 version_iterator = version
264 l = 0
265 m = 0
266 w = 0
267 lhref = ""
268 mhref = ""
269 whref = ""
270
271 href = hrefRegEx.match(line).group(1)
272
273 if re.search('Linux', line) != None:
274 lhref = href
275 l = 1
276 elif re.search('Darwin', line) != None:
277 mhref = href
278 m = 1
279 elif re.search('win32', line) != None:
280 whref = href
281 w = 1
282 else:
283 print "error: unexpected non-matching line"
284 sys.exit(1)
285
286 count = count + 1
287
288 if l == 1 and m == 1 and w == 1:
289 found = 1
290 print "Detected latest available CMake " + vdir + " build: " + version
291 break
292
293 if not found:
294 print "error: could not find a " + vdir + " version with all 3 platforms available"
295 return ()
296
297 return (('linux2', lhref, version), ('darwin', mhref, version), ('win32', whref, version))
298
299
300 def Download(basedir, url):
301 cmps = url.rsplit("/", 1)
302 href = cmps[1]
303 filename = ''.join([basedir, "/", href])
304
305 print 'Downloading ' + href + '...'
306
307 try:
308 createDir(basedir)
309 except:
310 if not os.path.exists(basedir):
311 raise
312
313 proxyDict = {}
314 if not options.httpProxy == "":
315 proxyDict["http"] = options.httpProxy
316
317 if not options.httpsProxy == "":
318 proxyDict["https"] = options.httpsProxy
319
320 if len(proxyDict) != 0:
321 proxies = urllib2.ProxyHandler(proxyDict)
322 else:
323 #no proxies set so use proxies from environment
324 proxies = urllib2.ProxyHandler()
325
326 opener = urllib2.build_opener(proxies)
327 url_file = opener.open(url)
328
329 outfile = open(filename, "wb")
330 outfile.write(url_file.read())
331 outfile.close()
332
333
334 def Extract(basedir, url):
335 cmps = url.rsplit("/", 1)
336 href = cmps[1]
337 filename = ''.join([basedir, "/", href])
338
339 print 'Extracting ' + href + '...'
340
341 if href[-4:] == ".zip":
342 if sys.version < '2.6':
343 if sys.platform == 'win32':
344 print "error: cannot extract zip files on win32 with older python < 2.6"
345 else:
346 print "warning: avoiding zipfile.extractall on older python < 2.6"
347 print " skipping this extraction..."
348 else:
349 z = zipfile.ZipFile(filename)
350 z.extractall(basedir)
351 z.close()
352 else:
353 if sys.version < '2.6':
354 if sys.platform == 'win32':
355 print "error: cannot extract tar files on win32 with older python < 2.6"
356 else:
357 print "warning: avoiding tarfile.extractall on older python < 2.6"
358 print " trying command line tar instead..."
359 origDir = os.getcwd()
360 echoChDir(basedir)
361 echoRunSysCmnd("tar -xzf " + href)
362 echoChDir(origDir)
363 else:
364 t = tarfile.open(filename)
365 t.extractall(basedir)
366 t.close()
367
368
369 def Install(basedir, url):
370 cmps = url.rsplit("/", 1)
371 href = cmps[1]
372
373 if href[-4:] == ".zip":
374 href = href[:-4]
375 elif href[-7:] == ".tar.gz":
376 href = href[:-7]
377
378 dirname = ''.join([basedir, "/", href])
379
380 print 'Installing ' + href + '...'
381 print ' src dir: [' + dirname + ']'
382 print ' dst dir: [' + options.installDir + ']'
383
384 if sys.platform == 'win32':
385 if os.path.exists(options.installDir):
386 print "error: --install-dir '" + options.installDir + "' already exists - remove it or rename it and try again -- or manually copy the source directory '" + dirname + "' to the final installation location..."
387 sys.exit(1)
388
389 shutil.copytree(dirname, options.installDir)
390 else:
391 # avoid the "copytree doesn't work if dir already exists" problem by using
392 # the sys command "cp"
393 try:
394 createDir(options.installDir)
395 except:
396 if not os.path.exists(options.installDir):
397 raise
398
399 echoRunSysCmnd("cp -r " + dirname + "/* " + options.installDir)
400
401 # After installing, create symlinks if requested.
402 # (This chunk needs work if ever it must work on Windows. For now, it's a
403 # Linux/Mac only chunk of python... Uses 'ln', no '.exe' suffix...)
404 #
405 if options.symlinks and options.symlinksDir != '':
406 cmake_file = ""
407 cmake_install_topdir = ""
408 pre = ""
409
410 cmake_files = glob.glob(dirname + "/*")
411 if len(cmake_files) >= 1:
412 cmake_file = cmake_files[0]
413
414 cmps = cmake_file.rsplit("/", 1)
415 if len(cmps) >= 1:
416 cmake_install_topdir = cmps[1]
417
418 if os.path.exists(options.installDir + "/" + cmake_install_topdir + "/Contents/bin/cmake"):
419 pre = cmake_install_topdir + "/Contents/bin"
420 elif os.path.exists(options.installDir + "/bin/cmake"):
421 pre = "bin"
422
423 if pre == '':
424 print "error: could not determine CMake install tree structure - cannot create symlinks into unexpected directory structure"
425 sys.exit(1)
426
427 if not os.path.exists(options.symlinksDir):
428 echoRunSysCmnd("mkdir -p \"" + options.symlinksDir + "\"")
429
430 for exe in ('ccmake', 'cmake', 'cmake-gui', 'cmakexbuild', 'cpack', 'ctest'):
431 if os.path.exists(options.installDir + "/" + pre + "/" + exe):
432 print "Creating " + exe + " symlink..."
433 echoRunSysCmnd("ln -fs \"" + options.installDir + "/" + pre + "/" + exe + "\" \"" + options.symlinksDir + "/" + exe + "\"")
434
435
436 def DownloadForPlatform(p):
437 if options.allPlatforms:
438 return True
439 if p == sys.platform:
440 return True
441 return False
442
443
444 def PrintDetectedDownloads(detected):
445 print ""
446 print "Detected CMake downloads available:"
447
448 sorted_keys = detected.keys()
449 sorted_keys.sort()
450
451 detected_urls = list()
452
453 for k in sorted_keys:
454 for v in detected[k]:
455 if DownloadForPlatform(v[0]):
456 detected_urls.append(cmake_baseurl + "/" + k + "/" + v[1])
457
458 for u in detected_urls:
459 print "[" + u + "]"
460
461
462 def PrintVerifiedDownloads():
463 print ""
464 print "Verified CMake downloads:"
465
466 verified_urls = list()
467
468 for v in cmake_min_binaries:
469 if DownloadForPlatform(v[0]):
470 verified_urls.append(v[1])
471
472 for v in cmake_release_binaries:
473 if DownloadForPlatform(v[0]):
474 verified_urls.append(v[1])
475
476 for v in cmake_rc_binaries:
477 if DownloadForPlatform(v[0]):
478 verified_urls.append(v[1])
479
480 for v in cmake_dev_binaries:
481 if DownloadForPlatform(v[0]):
482 verified_urls.append(v[1])
483
484 for u in verified_urls:
485 print "[" + u + "]"
486
487
488 # Read file "CMakeVersions.py" from the same directory that this script lives
489 # in. Then write an auto_updated edition of CMakeVersions.py in download_dir
490 # based on the detected latest available build for each auto_update vdir entry.
491 #
492 # Each line that matches the "auto_update" regex will be updated using the
493 # latest data in the "detected" dictionary. Other lines will be written out
494 # verbatim.
495 #
496 def ReadWriteCMakeVersionsFile(download_dir, detected):
497 rfname = os.path.dirname(os.path.abspath(__file__)) + "/CMakeVersions.py"
498
499 fr = open(rfname)
500 lines = fr.readlines()
501
502 wfname = download_dir + "/CMakeVersions.py"
503 fw = open(wfname, "w")
504
505 regex = re.compile(
506 "cmake_version_(.*) = \"(.*)\" # auto_update ([^ \t\n]*)"
507 )
508
509 for line in lines:
510 if regex.search(line):
511 #fw.write("# original: ")
512 #fw.write(line)
513
514 type = regex.match(line).group(1)
515 vdir = regex.match(line).group(3)
516
517 binaries = detected[vdir]
518 binary_0 = binaries[0]
519 version = binary_0[2]
520
521 auto_update_line = "cmake_version_" + type + " = \"" + version + "\" # auto_update " + vdir + "\n"
522
523 fw.write(auto_update_line)
524 else:
525 fw.write(line)
526
527 print ""
528 print "Wrote new '" + wfname + "' -- copy to '" + rfname + "' (if different) to use newly detected installers."
529
530
531 #
532 # The main script
533 #
534
535 print ""
536 print "**************************************************************************"
537 print "Script: download-cmake.py \\"
538
539 if options.allPlatforms:
540 print " --all-platforms \\"
541 print " --http-proxy="+options.httpProxy+" \\"
542 print " --https-proxy="+options.httpsProxy+" \\"
543 print " --install-dir="+options.installDir+" \\"
544 print " --installer-type="+options.installerType+" \\"
545 if options.skipDetect:
546 print " --skip-detect \\"
547 if options.skipDownload:
548 print " --skip-download \\"
549 if options.skipExtract:
550 print " --skip-extract \\"
551 if options.skipInstall:
552 print " --skip-install \\"
553 if options.symlinks:
554 print " --symlinks \\"
555 if options.symlinksDir != '':
556 print " --symlinks-dir="+options.symlinksDir+" \\"
557
558 if not options.httpProxy and not default_http_proxy:
559 print "\nWARNING: Could not detect default http proxy for '"+hostname+"'!"
560
561 if not options.httpsProxy and not default_https_proxy:
562 print "\nWARNING: Could not detect default https proxy for '"+hostname+"'!"
563
564 download_dir = "download_area"
565
566 binaries = None
567 if options.installerType == 'min':
568 binaries = cmake_min_binaries
569 if options.installerType == 'release':
570 binaries = cmake_release_binaries
571 if options.installerType == 'rc':
572 binaries = cmake_rc_binaries
573 if options.installerType == 'dev':
574 binaries = cmake_dev_binaries
575 if binaries == None:
576 print "error: unknown --installer-type: [" + options.installerType + "]"
577 sys.exit(1)
578
579 print ""
580 print ""
581 print "A) Detect the latest available builds of CMake ..."
582 print " (requires network access to www.cmake.org)"
583 print ""
584
585 if options.skipDetect:
586 print "Skipping on request ..."
587 else:
588 detected = dict()
589
590 for vdir in cmake_vdirs:
591 detected[vdir] = DetectLatestCMakeBuilds(download_dir, cmake_baseurl, vdir)
592
593 PrintDetectedDownloads(detected)
594
595 PrintVerifiedDownloads()
596
597 ReadWriteCMakeVersionsFile(download_dir, detected)
598
599
600 print ""
601 print ""
602 print "B) Download CMake for --installer-type '" + options.installerType + "' ..."
603 print " (requires network access to www.cmake.org)"
604 print ""
605
606 if options.skipDownload:
607 print "Skipping on request ..."
608 else:
609 for binary in binaries:
610 if DownloadForPlatform(binary[0]):
611 Download(download_dir, binary[1])
612
613
614 print ""
615 print ""
616 print "C) Extract the CMake install tree ..."
617 print ""
618
619 if options.skipExtract:
620 print "Skipping on request ..."
621 else:
622 for binary in binaries:
623 if DownloadForPlatform(binary[0]):
624 Extract(download_dir, binary[1])
625
626
627 print ""
628 print ""
629 print "D) Install (copy the CMake install tree) ..."
630 print ""
631
632 if options.skipInstall:
633 print "Skipping on request ..."
634 else:
635 for binary in binaries:
636 if binary[0] == sys.platform:
637 Install(download_dir, binary[1])