Debian Codegraph @

utils/git-sync-deps

raw
1 #!/usr/bin/env python3
2 # Copyright 2014 Google Inc.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 # * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 # * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Parse a DEPS file and git checkout all of the dependencies.
31
32 Args:
33 An optional list of deps_os values.
34
35 Environment Variables:
36 GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of
37 ['git', 'git.exe', 'git.bat'] in your default path.
38
39 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset,
40 will use the file ../DEPS relative to this script's directory.
41
42 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages.
43
44 Git Config:
45 To disable syncing of a single repository:
46 cd path/to/repository
47 git config sync-deps.disable true
48
49 To re-enable sync:
50 cd path/to/repository
51 git config --unset sync-deps.disable
52 """
53
54
55 import os
56 import re
57 import subprocess
58 import sys
59 import threading
60 from builtins import bytes
61
62
63 def git_executable():
64 """Find the git executable.
65
66 Returns:
67 A string suitable for passing to subprocess functions, or None.
68 """
69 envgit = os.environ.get('GIT_EXECUTABLE')
70 searchlist = ['git', 'git.exe', 'git.bat']
71 if envgit:
72 searchlist.insert(0, envgit)
73 with open(os.devnull, 'w') as devnull:
74 for git in searchlist:
75 try:
76 subprocess.call([git, '--version'], stdout=devnull)
77 except (OSError,):
78 continue
79 return git
80 return None
81
82
83 DEFAULT_DEPS_PATH = os.path.normpath(
84 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS'))
85
86
87 def usage(deps_file_path = None):
88 sys.stderr.write(
89 'Usage: run to grab dependencies, with optional platform support:\n')
90 sys.stderr.write(' %s %s' % (sys.executable, __file__))
91 if deps_file_path:
92 parsed_deps = parse_file_to_dict(deps_file_path)
93 if 'deps_os' in parsed_deps:
94 for deps_os in parsed_deps['deps_os']:
95 sys.stderr.write(' [%s]' % deps_os)
96 sys.stderr.write('\n\n')
97 sys.stderr.write(__doc__)
98
99
100 def git_repository_sync_is_disabled(git, directory):
101 try:
102 disable = subprocess.check_output(
103 [git, 'config', 'sync-deps.disable'], cwd=directory)
104 return disable.lower().strip() in ['true', '1', 'yes', 'on']
105 except subprocess.CalledProcessError:
106 return False
107
108
109 def is_git_toplevel(git, directory):
110 """Return true iff the directory is the top level of a Git repository.
111
112 Args:
113 git (string) the git executable
114
115 directory (string) the path into which the repository
116 is expected to be checked out.
117 """
118 try:
119 toplevel = subprocess.check_output(
120 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip()
121 return os.path.realpath(bytes(directory, 'utf8')) == os.path.realpath(toplevel)
122 except subprocess.CalledProcessError:
123 return False
124
125
126 def status(directory, checkoutable):
127 def truncate(s, length):
128 return s if len(s) <= length else '...' + s[-(length - 3):]
129 dlen = 36
130 directory = truncate(directory, dlen)
131 checkoutable = truncate(checkoutable, 40)
132 sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable))
133
134
135 def git_checkout_to_directory(git, repo, checkoutable, directory, verbose):
136 """Checkout (and clone if needed) a Git repository.
137
138 Args:
139 git (string) the git executable
140
141 repo (string) the location of the repository, suitable
142 for passing to `git clone`.
143
144 checkoutable (string) a tag, branch, or commit, suitable for
145 passing to `git checkout`
146
147 directory (string) the path into which the repository
148 should be checked out.
149
150 verbose (boolean)
151
152 Raises an exception if any calls to git fail.
153 """
154 if not os.path.isdir(directory):
155 subprocess.check_call(
156 [git, 'clone', '--quiet', repo, directory])
157
158 if not is_git_toplevel(git, directory):
159 # if the directory exists, but isn't a git repo, you will modify
160 # the parent repostory, which isn't what you want.
161 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory)
162 return
163
164 # Check to see if this repo is disabled. Quick return.
165 if git_repository_sync_is_disabled(git, directory):
166 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory)
167 return
168
169 with open(os.devnull, 'w') as devnull:
170 # If this fails, we will fetch before trying again. Don't spam user
171 # with error infomation.
172 if 0 == subprocess.call([git, 'checkout', '--quiet', checkoutable],
173 cwd=directory, stderr=devnull):
174 # if this succeeds, skip slow `git fetch`.
175 if verbose:
176 status(directory, checkoutable) # Success.
177 return
178
179 # If the repo has changed, always force use of the correct repo.
180 # If origin already points to repo, this is a quick no-op.
181 subprocess.check_call(
182 [git, 'remote', 'set-url', 'origin', repo], cwd=directory)
183
184 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory)
185
186 subprocess.check_call([git, 'checkout', '--quiet', checkoutable], cwd=directory)
187
188 if verbose:
189 status(directory, checkoutable) # Success.
190
191
192 def parse_file_to_dict(path):
193 dictionary = {}
194 contents = open(path).read()
195 # Need to convert Var() to vars[], so that the DEPS is actually Python. Var()
196 # comes from Autoroller using gclient which has a slightly different DEPS
197 # format.
198 contents = re.sub(r"Var\((.*?)\)", r"vars[\1]", contents)
199 exec(contents, dictionary)
200 return dictionary
201
202
203 def git_sync_deps(deps_file_path, command_line_os_requests, verbose):
204 """Grab dependencies, with optional platform support.
205
206 Args:
207 deps_file_path (string) Path to the DEPS file.
208
209 command_line_os_requests (list of strings) Can be empty list.
210 List of strings that should each be a key in the deps_os
211 dictionary in the DEPS file.
212
213 Raises git Exceptions.
214 """
215 git = git_executable()
216 assert git
217
218 deps_file_directory = os.path.dirname(deps_file_path)
219 deps_file = parse_file_to_dict(deps_file_path)
220 dependencies = deps_file['deps'].copy()
221 os_specific_dependencies = deps_file.get('deps_os', dict())
222 if 'all' in command_line_os_requests:
223 for value in list(os_specific_dependencies.values()):
224 dependencies.update(value)
225 else:
226 for os_name in command_line_os_requests:
227 # Add OS-specific dependencies
228 if os_name in os_specific_dependencies:
229 dependencies.update(os_specific_dependencies[os_name])
230 for directory in dependencies:
231 for other_dir in dependencies:
232 if directory.startswith(other_dir + '/'):
233 raise Exception('%r is parent of %r' % (other_dir, directory))
234 list_of_arg_lists = []
235 for directory in sorted(dependencies):
236 if '@' in dependencies[directory]:
237 repo, checkoutable = dependencies[directory].split('@', 1)
238 else:
239 raise Exception("please specify commit or tag")
240
241 relative_directory = os.path.join(deps_file_directory, directory)
242
243 list_of_arg_lists.append(
244 (git, repo, checkoutable, relative_directory, verbose))
245
246 multithread(git_checkout_to_directory, list_of_arg_lists)
247
248 for directory in deps_file.get('recursedeps', []):
249 recursive_path = os.path.join(deps_file_directory, directory, 'DEPS')
250 git_sync_deps(recursive_path, command_line_os_requests, verbose)
251
252
253 def multithread(function, list_of_arg_lists):
254 # for args in list_of_arg_lists:
255 # function(*args)
256 # return
257 threads = []
258 for args in list_of_arg_lists:
259 thread = threading.Thread(None, function, None, args)
260 thread.start()
261 threads.append(thread)
262 for thread in threads:
263 thread.join()
264
265
266 def main(argv):
267 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH)
268 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False))
269
270 if '--help' in argv or '-h' in argv:
271 usage(deps_file_path)
272 return 1
273
274 git_sync_deps(deps_file_path, argv, verbose)
275 # subprocess.check_call(
276 # [sys.executable,
277 # os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')])
278 return 0
279
280
281 if __name__ == '__main__':
282 exit(main(sys.argv[1:]))