| 1 |
#!/usr/bin/env python |
| 2 |
# Copyright 2014 The Chromium Authors. All rights reserved. |
| 3 |
# Use of this source code is governed by a BSD-style license that can be |
| 4 |
# found in the LICENSE file. |
| 5 |
|
| 6 |
# Modified from go/bootstrap.py in Chromium infrastructure's repository to patch |
| 7 |
# out everything but the core toolchain. |
| 8 |
# |
| 9 |
# https://chromium.googlesource.com/infra/infra/ |
| 10 |
|
| 11 |
"""Prepares a local hermetic Go installation. |
| 12 |
|
| 13 |
- Downloads and unpacks the Go toolset in ../golang. |
| 14 |
""" |
| 15 |
|
| 16 |
import contextlib |
| 17 |
import logging |
| 18 |
import os |
| 19 |
import platform |
| 20 |
import shutil |
| 21 |
import stat |
| 22 |
import subprocess |
| 23 |
import sys |
| 24 |
import tarfile |
| 25 |
import tempfile |
| 26 |
import urllib |
| 27 |
import zipfile |
| 28 |
|
| 29 |
# TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to |
| 30 |
# git completely. |
| 31 |
|
| 32 |
LOGGER = logging.getLogger(__name__) |
| 33 |
|
| 34 |
|
| 35 |
# /path/to/util/bot |
| 36 |
ROOT = os.path.dirname(os.path.abspath(__file__)) |
| 37 |
|
| 38 |
# Where to install Go toolset to. GOROOT would be <TOOLSET_ROOT>/go. |
| 39 |
TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang') |
| 40 |
|
| 41 |
# Default workspace with infra go code. |
| 42 |
WORKSPACE = os.path.join(ROOT, 'go') |
| 43 |
|
| 44 |
# Platform depended suffix for executable files. |
| 45 |
EXE_SFX = '.exe' if sys.platform == 'win32' else '' |
| 46 |
|
| 47 |
# Pinned version of Go toolset to download. |
| 48 |
TOOLSET_VERSION = 'go1.6.2' |
| 49 |
|
| 50 |
# Platform dependent portion of a download URL. See http://golang.org/dl/. |
| 51 |
TOOLSET_VARIANTS = { |
| 52 |
('darwin', 'x86-64'): 'darwin-amd64.tar.gz', |
| 53 |
('linux2', 'x86-32'): 'linux-386.tar.gz', |
| 54 |
('linux2', 'x86-64'): 'linux-amd64.tar.gz', |
| 55 |
('win32', 'x86-32'): 'windows-386.zip', |
| 56 |
('win32', 'x86-64'): 'windows-amd64.zip', |
| 57 |
} |
| 58 |
|
| 59 |
# Download URL root. |
| 60 |
DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang' |
| 61 |
|
| 62 |
|
| 63 |
class Failure(Exception): |
| 64 |
"""Bootstrap failed.""" |
| 65 |
|
| 66 |
|
| 67 |
def get_toolset_url(): |
| 68 |
"""URL of a platform specific Go toolset archive.""" |
| 69 |
# TODO(vadimsh): Support toolset for cross-compilation. |
| 70 |
arch = { |
| 71 |
'amd64': 'x86-64', |
| 72 |
'x86_64': 'x86-64', |
| 73 |
'i386': 'x86-32', |
| 74 |
'x86': 'x86-32', |
| 75 |
}.get(platform.machine().lower()) |
| 76 |
variant = TOOLSET_VARIANTS.get((sys.platform, arch)) |
| 77 |
if not variant: |
| 78 |
# TODO(vadimsh): Compile go lang from source. |
| 79 |
raise Failure('Unrecognized platform') |
| 80 |
return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant) |
| 81 |
|
| 82 |
|
| 83 |
def read_file(path): |
| 84 |
"""Returns contents of a given file or None if not readable.""" |
| 85 |
assert isinstance(path, (list, tuple)) |
| 86 |
try: |
| 87 |
with open(os.path.join(*path), 'r') as f: |
| 88 |
return f.read() |
| 89 |
except IOError: |
| 90 |
return None |
| 91 |
|
| 92 |
|
| 93 |
def write_file(path, data): |
| 94 |
"""Writes |data| to a file.""" |
| 95 |
assert isinstance(path, (list, tuple)) |
| 96 |
with open(os.path.join(*path), 'w') as f: |
| 97 |
f.write(data) |
| 98 |
|
| 99 |
|
| 100 |
def remove_directory(path): |
| 101 |
"""Recursively removes a directory.""" |
| 102 |
assert isinstance(path, (list, tuple)) |
| 103 |
p = os.path.join(*path) |
| 104 |
if not os.path.exists(p): |
| 105 |
return |
| 106 |
LOGGER.info('Removing %s', p) |
| 107 |
# Crutch to remove read-only file (.git/* in particular) on Windows. |
| 108 |
def onerror(func, path, _exc_info): |
| 109 |
if not os.access(path, os.W_OK): |
| 110 |
os.chmod(path, stat.S_IWUSR) |
| 111 |
func(path) |
| 112 |
else: |
| 113 |
raise |
| 114 |
shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None) |
| 115 |
|
| 116 |
|
| 117 |
def install_toolset(toolset_root, url): |
| 118 |
"""Downloads and installs Go toolset. |
| 119 |
|
| 120 |
GOROOT would be <toolset_root>/go/. |
| 121 |
""" |
| 122 |
if not os.path.exists(toolset_root): |
| 123 |
os.makedirs(toolset_root) |
| 124 |
pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:]) |
| 125 |
|
| 126 |
LOGGER.info('Downloading %s...', url) |
| 127 |
download_file(url, pkg_path) |
| 128 |
|
| 129 |
LOGGER.info('Extracting...') |
| 130 |
if pkg_path.endswith('.zip'): |
| 131 |
with zipfile.ZipFile(pkg_path, 'r') as f: |
| 132 |
f.extractall(toolset_root) |
| 133 |
elif pkg_path.endswith('.tar.gz'): |
| 134 |
with tarfile.open(pkg_path, 'r:gz') as f: |
| 135 |
f.extractall(toolset_root) |
| 136 |
else: |
| 137 |
raise Failure('Unrecognized archive format') |
| 138 |
|
| 139 |
LOGGER.info('Validating...') |
| 140 |
if not check_hello_world(toolset_root): |
| 141 |
raise Failure('Something is not right, test program doesn\'t work') |
| 142 |
|
| 143 |
|
| 144 |
def download_file(url, path): |
| 145 |
"""Fetches |url| to |path|.""" |
| 146 |
last_progress = [0] |
| 147 |
def report(a, b, c): |
| 148 |
progress = int(a * b * 100.0 / c) |
| 149 |
if progress != last_progress[0]: |
| 150 |
print >> sys.stderr, 'Downloading... %d%%' % progress |
| 151 |
last_progress[0] = progress |
| 152 |
# TODO(vadimsh): Use something less crippled, something that validates SSL. |
| 153 |
urllib.urlretrieve(url, path, reporthook=report) |
| 154 |
|
| 155 |
|
| 156 |
@contextlib.contextmanager |
| 157 |
def temp_dir(path): |
| 158 |
"""Creates a temporary directory, then deletes it.""" |
| 159 |
tmp = tempfile.mkdtemp(dir=path) |
| 160 |
try: |
| 161 |
yield tmp |
| 162 |
finally: |
| 163 |
remove_directory([tmp]) |
| 164 |
|
| 165 |
|
| 166 |
def check_hello_world(toolset_root): |
| 167 |
"""Compiles and runs 'hello world' program to verify that toolset works.""" |
| 168 |
with temp_dir(toolset_root) as tmp: |
| 169 |
path = os.path.join(tmp, 'hello.go') |
| 170 |
write_file([path], r""" |
| 171 |
package main |
| 172 |
func main() { println("hello, world\n") } |
| 173 |
""") |
| 174 |
out = subprocess.check_output( |
| 175 |
[get_go_exe(toolset_root), 'run', path], |
| 176 |
env=get_go_environ(toolset_root, tmp), |
| 177 |
stderr=subprocess.STDOUT) |
| 178 |
if out.strip() != 'hello, world': |
| 179 |
LOGGER.error('Failed to run sample program:\n%s', out) |
| 180 |
return False |
| 181 |
return True |
| 182 |
|
| 183 |
|
| 184 |
def ensure_toolset_installed(toolset_root): |
| 185 |
"""Installs or updates Go toolset if necessary. |
| 186 |
|
| 187 |
Returns True if new toolset was installed. |
| 188 |
""" |
| 189 |
installed = read_file([toolset_root, 'INSTALLED_TOOLSET']) |
| 190 |
available = get_toolset_url() |
| 191 |
if installed == available: |
| 192 |
LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION) |
| 193 |
return False |
| 194 |
|
| 195 |
LOGGER.info('Installing Go toolset.') |
| 196 |
LOGGER.info(' Old toolset is %s', installed) |
| 197 |
LOGGER.info(' New toolset is %s', available) |
| 198 |
remove_directory([toolset_root]) |
| 199 |
install_toolset(toolset_root, available) |
| 200 |
LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION) |
| 201 |
write_file([toolset_root, 'INSTALLED_TOOLSET'], available) |
| 202 |
return True |
| 203 |
|
| 204 |
|
| 205 |
def get_go_environ( |
| 206 |
toolset_root, |
| 207 |
workspace=None): |
| 208 |
"""Returns a copy of os.environ with added GO* environment variables. |
| 209 |
|
| 210 |
Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent. |
| 211 |
|
| 212 |
Args: |
| 213 |
toolset_root: GOROOT would be <toolset_root>/go. |
| 214 |
workspace: main workspace directory or None if compiling in GOROOT. |
| 215 |
""" |
| 216 |
env = os.environ.copy() |
| 217 |
env['GOROOT'] = os.path.join(toolset_root, 'go') |
| 218 |
if workspace: |
| 219 |
env['GOBIN'] = os.path.join(workspace, 'bin') |
| 220 |
else: |
| 221 |
env.pop('GOBIN', None) |
| 222 |
|
| 223 |
all_go_paths = [] |
| 224 |
if workspace: |
| 225 |
all_go_paths.append(workspace) |
| 226 |
env['GOPATH'] = os.pathsep.join(all_go_paths) |
| 227 |
|
| 228 |
# New PATH entries. |
| 229 |
paths_to_add = [ |
| 230 |
os.path.join(env['GOROOT'], 'bin'), |
| 231 |
env.get('GOBIN'), |
| 232 |
] |
| 233 |
|
| 234 |
# Make sure not to add duplicates entries to PATH over and over again when |
| 235 |
# get_go_environ is invoked multiple times. |
| 236 |
path = env['PATH'].split(os.pathsep) |
| 237 |
paths_to_add = [p for p in paths_to_add if p and p not in path] |
| 238 |
env['PATH'] = os.pathsep.join(paths_to_add + path) |
| 239 |
|
| 240 |
return env |
| 241 |
|
| 242 |
|
| 243 |
def get_go_exe(toolset_root): |
| 244 |
"""Returns path to go executable.""" |
| 245 |
return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX) |
| 246 |
|
| 247 |
|
| 248 |
def bootstrap(logging_level): |
| 249 |
"""Installs all dependencies in default locations. |
| 250 |
|
| 251 |
Supposed to be called at the beginning of some script (it modifies logger). |
| 252 |
|
| 253 |
Args: |
| 254 |
logging_level: logging level of bootstrap process. |
| 255 |
""" |
| 256 |
logging.basicConfig() |
| 257 |
LOGGER.setLevel(logging_level) |
| 258 |
ensure_toolset_installed(TOOLSET_ROOT) |
| 259 |
|
| 260 |
|
| 261 |
def prepare_go_environ(): |
| 262 |
"""Returns dict with environment variables to set to use Go toolset. |
| 263 |
|
| 264 |
Installs or updates the toolset if necessary. |
| 265 |
""" |
| 266 |
bootstrap(logging.INFO) |
| 267 |
return get_go_environ(TOOLSET_ROOT, WORKSPACE) |
| 268 |
|
| 269 |
|
| 270 |
def find_executable(name, workspaces): |
| 271 |
"""Returns full path to an executable in some bin/ (in GOROOT or GOBIN).""" |
| 272 |
basename = name |
| 273 |
if EXE_SFX and basename.endswith(EXE_SFX): |
| 274 |
basename = basename[:-len(EXE_SFX)] |
| 275 |
roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')] |
| 276 |
for path in workspaces: |
| 277 |
roots.extend([ |
| 278 |
os.path.join(path, 'bin'), |
| 279 |
]) |
| 280 |
for root in roots: |
| 281 |
full_path = os.path.join(root, basename + EXE_SFX) |
| 282 |
if os.path.exists(full_path): |
| 283 |
return full_path |
| 284 |
return name |
| 285 |
|
| 286 |
|
| 287 |
def main(args): |
| 288 |
if args: |
| 289 |
print >> sys.stderr, sys.modules[__name__].__doc__, |
| 290 |
return 2 |
| 291 |
bootstrap(logging.DEBUG) |
| 292 |
return 0 |
| 293 |
|
| 294 |
|
| 295 |
if __name__ == '__main__': |
| 296 |
sys.exit(main(sys.argv[1:])) |