Debian Codegraph @

llvm/utils/git-llvm-push

raw
1 #!/usr/bin/env python3
2 """A script to automate the creation and landing of a stack of Pull Requests."""
3
4 import argparse
5 import json
6 import os
7 import re
8 import subprocess
9 import sys
10 import time
11 import urllib.error
12 import urllib.request
13
14 # TODO: Remove typing workarounds when we use a newer python.
15 from typing import List, Optional, Tuple
16 from http.client import HTTPResponse
17 from dataclasses import dataclass
18
19 # --- Constants --- #
20 BASE_BRANCH = "main"
21 GITHUB_REMOTE_NAME = "origin"
22 UPSTREAM_REMOTE_NAME = "upstream"
23
24 DEFAULT_LABEL = "skip-precommit-approval"
25
26 LLVM_GITHUB_TOKEN_VAR = "LLVM_GITHUB_TOKEN"
27 LLVM_REPO = "llvm/llvm-project"
28 GITHUB_API = "https://api.github.com"
29
30 MERGE_MAX_RETRIES = 10
31 MERGE_RETRY_DELAY = 5 # seconds
32 REQUEST_TIMEOUT = 30 # seconds
33
34
35 class LlvmPrError(Exception):
36 """Custom exception for errors in the PR automator script."""
37
38
39 @dataclass
40 class PRAutomatorConfig:
41 """Configuration Data."""
42
43 user_login: str
44 token: str
45 base_branch: str
46 upstream_remote: str
47 prefix: str
48 draft: bool
49 labels: List[str]
50 no_merge: bool
51 auto_merge: bool
52
53
54 class CommandRunner:
55 """Handles command execution and output.
56 Supports dry runs and verbosity level."""
57
58 def __init__(
59 self, dry_run: bool = False, verbose: bool = False, quiet: bool = False
60 ):
61 self.dry_run = dry_run
62 self.verbose = verbose
63 self.quiet = quiet
64
65 def print(self, message: str, file=sys.stdout) -> None:
66 if self.quiet and file == sys.stdout:
67 return
68 print(message, file=file)
69
70 def verbose_print(self, message: str, file=sys.stdout) -> None:
71 if self.verbose:
72 print(message, file)
73
74 def run_command(
75 self,
76 command: List[str],
77 check: bool = True,
78 capture_output: bool = False,
79 text: bool = False,
80 stdin_input: Optional[str] = None,
81 read_only: bool = False,
82 env: Optional[dict] = None,
83 ) -> subprocess.CompletedProcess:
84 if self.dry_run and not read_only:
85 self.print(f"[Dry Run] Would run: {' '.join(command)}")
86 return subprocess.CompletedProcess(command, 0, "", "")
87
88 self.verbose_print(f"Running: {' '.join(command)}")
89
90 try:
91 return subprocess.run(
92 command,
93 check=check,
94 capture_output=capture_output,
95 text=text,
96 input=stdin_input,
97 env=env,
98 )
99 except FileNotFoundError as e:
100 raise LlvmPrError(
101 f"Command '{command[0]}' not found. Is it installed and in your PATH?"
102 ) from e
103 except subprocess.CalledProcessError as e:
104 self.print(f"Error running command: {' '.join(command)}", file=sys.stderr)
105 if e.stdout:
106 self.print(f"--- stdout ---\n{e.stdout}", file=sys.stderr)
107 if e.stderr:
108 self.print(f"--- stderr ---\n{e.stderr}", file=sys.stderr)
109 raise e
110
111
112 class GitHubAPI:
113 """A wrapper for the GitHub API."""
114
115 def __init__(self, runner: CommandRunner, token: str):
116 self.runner = runner
117 self.headers = {
118 "Authorization": f"token {token}",
119 "Accept": "application/vnd.github.v3+json",
120 "User-Agent": "llvm-push-pr",
121 }
122 self.opener = urllib.request.build_opener(
123 urllib.request.HTTPHandler(), urllib.request.HTTPSHandler()
124 )
125
126 def _request(
127 self, method: str, endpoint: str, json_payload: Optional[dict] = None
128 ) -> HTTPResponse:
129 url = f"{GITHUB_API}{endpoint}"
130 self.runner.verbose_print(f"API Request: {method.upper()} {url}")
131 if json_payload:
132 self.runner.verbose_print(f"Payload: {json_payload}")
133
134 data = None
135 headers = self.headers.copy()
136 if json_payload:
137 data = json.dumps(json_payload).encode("utf-8")
138 headers["Content-Type"] = "application/json"
139
140 req = urllib.request.Request(
141 url, data=data, headers=headers, method=method.upper()
142 )
143
144 try:
145 return self.opener.open(req, timeout=REQUEST_TIMEOUT)
146 except urllib.error.HTTPError as e:
147 self.runner.print(
148 f"Error making API request to {url}: {e}", file=sys.stderr
149 )
150 self.runner.verbose_print(
151 f"Error response body: {e.read().decode()}", file=sys.stderr
152 )
153 raise e
154
155 def _request_and_parse_json(
156 self, method: str, endpoint: str, json_payload: Optional[dict] = None
157 ) -> dict:
158 with self._request(method, endpoint, json_payload) as response:
159 # Expect a 200 'OK' or 201 'Created' status on success and JSON body.
160 self._log_unexpected_status([200, 201], response.status)
161
162 response_text = response.read().decode("utf-8")
163 if response_text:
164 return json.loads(response_text)
165 return {}
166
167 def _request_no_content(
168 self, method: str, endpoint: str, json_payload: Optional[dict] = None
169 ) -> None:
170 with self._request(method, endpoint, json_payload) as response:
171 # Expected a 204 No Content status on success, indicating the
172 # operation was successful but there is no body.
173 self._log_unexpected_status([204], response.status)
174
175 def _log_unexpected_status(
176 self, expected_statuses: List[int], actual_status: int
177 ) -> None:
178 if actual_status not in expected_statuses:
179 self.runner.print(
180 f"Warning: Expected status {', '.join(map(str, expected_statuses))}, but got {actual_status}",
181 file=sys.stderr,
182 )
183
184 def get_user_login(self) -> str:
185 return self._request_and_parse_json("GET", "/user")["login"]
186
187 def create_pr(
188 self,
189 head_branch: str,
190 base_branch: str,
191 title: str,
192 body: str,
193 draft: bool,
194 ) -> int:
195 if self.runner.dry_run:
196 self.runner.print(f"[Dry Run] Would create pull request for '{head_branch}'...")
197 return 0
198
199 self.runner.print(f"Creating pull request for '{head_branch}'...")
200 data = {
201 "title": title,
202 "body": body,
203 "head": head_branch,
204 "base": base_branch,
205 "draft": draft,
206 }
207 response_data = self._request_and_parse_json(
208 "POST", f"/repos/{LLVM_REPO}/pulls", json_payload=data
209 )
210 self.runner.print(f"Pull request created: {response_data.get("html_url")}")
211 return response_data.get("number")
212
213 def get_repo_settings(self) -> dict:
214 return self._request_and_parse_json("GET", f"/repos/{LLVM_REPO}")
215
216 def _get_pr_details(self, pr_number: str) -> dict:
217 """Fetches the JSON details for a given pull request number."""
218 return self._request_and_parse_json(
219 "GET", f"/repos/{LLVM_REPO}/pulls/{pr_number}"
220 )
221
222 def add_labels(
223 self,
224 pr_number: int,
225 labels: List[str],
226 ) -> None:
227 if self.runner.dry_run:
228 self.runner.print(f"[Dry Run] Would set labels for #{pr_number}: {' '.join(labels)}")
229 return None
230
231 self.runner.print(f"Setting labels for #{pr_number}: {' '.join(labels)}")
232
233 self._request_and_parse_json(
234 "POST", f"/repos/{LLVM_REPO}/issues/{pr_number}/labels",
235 json_payload={"labels": labels},
236 )
237
238 def _attempt_squash_merge(self, pr_number: str) -> bool:
239 """Attempts to squash merge a PR, returning True on success."""
240 try:
241 self._request_and_parse_json(
242 "PUT",
243 f"/repos/{LLVM_REPO}/pulls/{pr_number}/merge",
244 json_payload={"merge_method": "squash"},
245 )
246 return True
247 except urllib.error.HTTPError as e:
248 # A 405 status code means the PR is not in a mergeable state.
249 if e.code == 405:
250 return False
251 # Re-raise other HTTP errors.
252 raise e
253
254 def merge_pr(self, pr_number: int) -> Optional[str]:
255 if self.runner.dry_run:
256 self.runner.print(f"[Dry Run] Would merge #{pr_number}")
257 return None
258
259 for i in range(MERGE_MAX_RETRIES):
260 self.runner.print(
261 f"Attempting to merge #{pr_number} (attempt {i + 1}/{MERGE_MAX_RETRIES})..."
262 )
263
264 pr_data = self._get_pr_details(pr_number)
265 head_branch = pr_data["head"]["ref"]
266
267 if pr_data.get("mergeable_state") == "dirty":
268 raise LlvmPrError("Merge conflict.")
269
270 if pr_data.get("mergeable"):
271 if self._attempt_squash_merge(pr_number):
272 self.runner.print("Successfully merged.")
273 time.sleep(2) # Allow GitHub's API to update.
274 return head_branch
275
276 self.runner.print(
277 f"PR not mergeable yet (state: {pr_data.get('mergeable_state', 'unknown')}). Retrying in {MERGE_RETRY_DELAY} seconds..."
278 )
279 time.sleep(MERGE_RETRY_DELAY)
280
281 raise LlvmPrError(f"PR was not mergeable after {MERGE_MAX_RETRIES} attempts.")
282
283 def enable_auto_merge(self, pr_number: int) -> None:
284 if self.runner.dry_run:
285 self.runner.print(f"[Dry Run] Would enable auto-merge for #{pr_number}")
286 return
287
288 self.runner.print(f"Enabling auto-merge for #{pr_number}...")
289 data = {
290 "enabled": True,
291 "merge_method": "squash",
292 }
293 self._request_no_content(
294 "PUT",
295 f"/repos/{LLVM_REPO}/pulls/{pr_number}/auto-merge",
296 json_payload=data,
297 )
298 self.runner.print("Auto-merge enabled.")
299
300 def delete_branch(
301 self, branch_name: str, default_branch: Optional[str] = None
302 ) -> None:
303 if default_branch and branch_name == default_branch:
304 self.runner.print(
305 f"Error: Refusing to delete the default branch '{branch_name}'.",
306 file=sys.stderr,
307 )
308 return
309 try:
310 self._request_no_content(
311 "DELETE", f"/repos/{LLVM_REPO}/git/refs/heads/{branch_name}"
312 )
313 except urllib.error.HTTPError as e:
314 if e.code == 422:
315 self.runner.print(
316 f"Warning: Remote branch '{branch_name}' was already deleted, skipping deletion.",
317 file=sys.stderr,
318 )
319 else:
320 raise e
321
322
323 class LLVMPRAutomator:
324 """Automates the process of creating and landing a stack of GitHub Pull Requests."""
325
326 def __init__(
327 self,
328 runner: CommandRunner,
329 github_api: "GitHubAPI",
330 config: "PRAutomatorConfig",
331 remote: str,
332 ):
333 self.runner = runner
334 self.github_api = github_api
335 self.config = config
336 self.remote = remote
337 self.original_branch: str = ""
338 self.created_branches: List[str] = []
339 self.repo_settings: dict = {}
340
341 def _get_git_env(self) -> dict:
342 git_env = os.environ.copy()
343 git_env[LLVM_GITHUB_TOKEN_VAR] = self.config.token
344 git_env["GIT_TERMINAL_PROMPT"] = "0"
345 git_env["GIT_CONFIG_COUNT"] = "1"
346 git_env["GIT_CONFIG_KEY_0"] = "credential.helper"
347 git_env[
348 "GIT_CONFIG_VALUE_0"
349 ] = f"!{sys.executable} -c \"import os; print('username=x'); print('password=' + os.environ['{LLVM_GITHUB_TOKEN_VAR}']);\""
350 return git_env
351
352 def _get_current_branch(self) -> str:
353 result = self.runner.run_command(
354 ["git", "rev-parse", "--abbrev-ref", "HEAD"],
355 capture_output=True,
356 text=True,
357 read_only=True,
358 )
359 return result.stdout.strip()
360
361 def _check_work_tree(self) -> None:
362 result = self.runner.run_command(
363 ["git", "status", "--porcelain"],
364 capture_output=True,
365 text=True,
366 read_only=True,
367 )
368 if result.stdout.strip():
369 raise LlvmPrError(
370 "Your working tree is dirty. Please stash or commit your changes."
371 )
372
373 def _rebase_current_branch(self) -> None:
374 self._check_work_tree()
375
376 target = f"{self.config.upstream_remote}/{self.config.base_branch}"
377 self.runner.print(
378 f"Fetching from '{self.config.upstream_remote}' and rebasing '{self.original_branch}' on top of '{target}'..."
379 )
380
381 git_env = self._get_git_env()
382
383 refspec = f"refs/heads/{self.config.base_branch}:refs/remotes/{self.config.upstream_remote}/{self.config.base_branch}"
384 self.runner.run_command(
385 ["git", "fetch", self.config.upstream_remote, refspec], env=git_env
386 )
387
388 try:
389 self.runner.run_command(["git", "rebase", target], env=git_env)
390 except subprocess.CalledProcessError as e:
391 self.runner.print(
392 "Error: The rebase operation failed, likely due to a merge conflict.",
393 file=sys.stderr,
394 )
395 if e.stdout:
396 self.runner.print(f"--- stdout ---\n{e.stdout}", file=sys.stderr)
397 if e.stderr:
398 self.runner.print(f"--- stderr ---\n{e.stderr}", file=sys.stderr)
399
400 # Check if rebase is in progress before aborting
401 rebase_status_result = self.runner.run_command(
402 ["git", "status", "--verify-status=REBASE_HEAD"],
403 check=False,
404 capture_output=True,
405 text=True,
406 read_only=True,
407 env=git_env,
408 )
409
410 # REBASE_HEAD exists, so rebase is in progress
411 if rebase_status_result.returncode == 0:
412 self.runner.print("Aborting rebase...", file=sys.stderr)
413 self.runner.run_command(
414 ["git", "rebase", "--abort"], check=False, env=git_env
415 )
416 raise LlvmPrError("rebase operation failed.") from e
417
418 def _get_commit_stack(self) -> List[str]:
419 target = f"{self.config.upstream_remote}/{self.config.base_branch}"
420 result = self.runner.run_command(
421 ["git", "rev-list", "--reverse", f"{target}..HEAD"],
422 capture_output=True,
423 text=True,
424 read_only=True,
425 )
426 return result.stdout.strip().splitlines()
427
428 def _get_commit_details(self, commit_hash: str) -> Tuple[str, str]:
429 # Get the subject and body from git show. Insert "\n\n" between to make
430 # parsing simple to do w/ split.
431 result = self.runner.run_command(
432 ["git", "show", "-s", "--format=%B", commit_hash],
433 capture_output=True,
434 text=True,
435 read_only=True,
436 )
437 parts = [item.strip() for item in result.stdout.split("\n", 1)]
438 title = parts[0]
439 body = parts[1] if len(parts) > 1 else ""
440 return title, body
441
442 def _sanitize_branch_name(self, text: str) -> str:
443 sanitized = re.sub(r"[^\w\s-]", "", text).strip().lower()
444 sanitized = re.sub(r"[-\s]+", "-", sanitized)
445 # Use "auto-pr" as a fallback.
446 return sanitized or "auto-pr"
447
448 def _validate_merge_config(self, num_commits: int) -> None:
449 if num_commits > 1:
450 if self.config.auto_merge:
451 raise LlvmPrError("--auto-merge is only supported for a single commit.")
452
453 if self.config.no_merge:
454 raise LlvmPrError(
455 "--no-merge is only supported for a single commit. "
456 "For stacks, the script must merge sequentially."
457 )
458
459 self.runner.print(f"Found {num_commits} commit(s) to process.")
460
461 def _create_and_push_branch_for_commit(
462 self, commit_hash: str, base_branch_name: str, index: int
463 ) -> str:
464 branch_name = f"{self.config.prefix}{base_branch_name}-{index + 1}"
465 commit_title, _ = self._get_commit_details(commit_hash)
466 self.runner.print(f"Processing commit {commit_hash[:7]}: {commit_title}")
467 self.runner.print(f"Pushing commit to temporary branch '{branch_name}'")
468
469 git_env = self._get_git_env()
470
471 push_command = [
472 "git",
473 "push",
474 self.remote,
475 f"{commit_hash}:refs/heads/{branch_name}",
476 ]
477 self.runner.run_command(push_command, env=git_env)
478 self.created_branches.append(branch_name)
479 return branch_name
480
481 def _process_commit(
482 self, commit_hash: str, base_branch_name: str, index: int
483 ) -> None:
484 commit_title, commit_body = self._get_commit_details(commit_hash)
485
486 temp_branch = self._create_and_push_branch_for_commit(
487 commit_hash, base_branch_name, index
488 )
489 pr_number = self.github_api.create_pr(
490 head_branch=f"{self.config.user_login}:{temp_branch}",
491 base_branch=self.config.base_branch,
492 title=commit_title,
493 body=commit_body,
494 draft=self.config.draft,
495 )
496
497 # TODO: There's a possibility of a race with PR labelers workflow.
498 # To avoid it, we could create the PR as a draft, set the labels
499 # and change the status, but that requires the use of GraphQL API.
500
501 if self.config.labels:
502 self.github_api.add_labels(pr_number, self.config.labels)
503
504 if self.config.no_merge:
505 return
506
507 if self.config.auto_merge:
508 self.github_api.enable_auto_merge(pr_number)
509 else:
510 merged_branch = self.github_api.merge_pr(pr_number)
511 if merged_branch and not self.repo_settings.get("delete_branch_on_merge"):
512 # After a merge, the branch should be deleted.
513 self.github_api.delete_branch(merged_branch)
514
515 if temp_branch in self.created_branches:
516 # If the branch was successfully merged, it should not be deleted
517 # again during cleanup.
518 self.created_branches.remove(temp_branch)
519
520 def run(self) -> None:
521 self.repo_settings = self.github_api.get_repo_settings()
522 self.original_branch = self._get_current_branch()
523 self.runner.print(f"On branch: {self.original_branch}")
524
525 try:
526 commits = self._get_commit_stack()
527 if not commits:
528 self.runner.print("No new commits to process.")
529 return
530
531 self._validate_merge_config(len(commits))
532 branch_base_name = self.original_branch
533 if self.original_branch == "main":
534 first_commit_title, _ = self._get_commit_details(commits[0])
535 branch_base_name = self._sanitize_branch_name(first_commit_title)
536
537 for i in range(len(commits)):
538 if not commits:
539 self.runner.print("Success! All commits have been landed.")
540 break
541 self._process_commit(commits[0], branch_base_name, i)
542 self._rebase_current_branch()
543 # After a rebase, the commit hashes can change, so we need to
544 # get the latest commit stack.
545 commits = self._get_commit_stack()
546
547 finally:
548 self._cleanup()
549
550 def _cleanup(self) -> None:
551 self.runner.print(f"Returning to original branch: {self.original_branch}")
552 self.runner.run_command(
553 ["git", "checkout", self.original_branch], capture_output=True
554 )
555 if self.created_branches:
556 self.runner.print("Cleaning up temporary remote branches...")
557 for branch in self.created_branches:
558 self.github_api.delete_branch(branch)
559
560
561 def check_prerequisites(runner: CommandRunner) -> None:
562 runner.print("Checking prerequisites...")
563 runner.run_command(["git", "--version"], capture_output=True, read_only=True)
564 result = runner.run_command(
565 ["git", "rev-parse", "--is-inside-work-tree"],
566 check=False,
567 capture_output=True,
568 text=True,
569 read_only=True,
570 )
571
572 if result.returncode != 0 or result.stdout.strip() != "true":
573 raise LlvmPrError("This script must be run inside a git repository.")
574 runner.print("Prerequisites met.")
575
576
577 def main() -> None:
578 parser = argparse.ArgumentParser(
579 description="Create and land a stack of Pull Requests."
580 )
581
582 command_runner = CommandRunner()
583 token = os.getenv(LLVM_GITHUB_TOKEN_VAR)
584 if not token:
585 raise LlvmPrError(f"{LLVM_GITHUB_TOKEN_VAR} environment variable not set.")
586
587 parser.add_argument(
588 "--base",
589 default=BASE_BRANCH,
590 help=f"Base branch to target (default: {BASE_BRANCH})",
591 )
592 parser.add_argument(
593 "--remote",
594 default=GITHUB_REMOTE_NAME,
595 help=f"Remote for your fork to push to (default: {GITHUB_REMOTE_NAME})",
596 )
597 parser.add_argument(
598 "--upstream-remote",
599 default=UPSTREAM_REMOTE_NAME,
600 help=f"Remote for the upstream repository (default: {UPSTREAM_REMOTE_NAME})",
601 )
602 parser.add_argument(
603 "--login",
604 default=None,
605 help="The user login to use. If not provided this will be queried from the TOKEN",
606 )
607 parser.add_argument(
608 "--prefix",
609 default=None,
610 help="Prefix for temporary branches (default: users/<username>)",
611 )
612 parser.add_argument(
613 "--draft", action="store_true", help="Create pull requests as drafts."
614 )
615 parser.add_argument(
616 "--labels",
617 nargs='*',
618 default=[DEFAULT_LABEL],
619 help=f"Set the PR labels (default: {DEFAULT_LABEL})."
620 )
621 merging = parser.add_mutually_exclusive_group()
622 merging.add_argument(
623 "--no-merge", action="store_true", help="Create PRs but do not merge them."
624 )
625 merging.add_argument(
626 "--auto-merge",
627 action="store_true",
628 help="Enable auto-merge for each PR instead of attempting to merge immediately.",
629 )
630 parser.add_argument(
631 "--dry-run", action="store_true", help="Print commands without executing them."
632 )
633 verbosity = parser.add_mutually_exclusive_group()
634 verbosity.add_argument(
635 "-v", "--verbose", action="store_true", help="Print all commands being run."
636 )
637 verbosity.add_argument(
638 "-q",
639 "--quiet",
640 action="store_true",
641 help="Print only essential output and errors.",
642 )
643
644 args = parser.parse_args()
645
646 command_runner = CommandRunner(
647 dry_run=args.dry_run, verbose=args.verbose, quiet=args.quiet
648 )
649 check_prerequisites(command_runner)
650
651 github_api = GitHubAPI(command_runner, token)
652 if not args.login:
653 try:
654 args.login = github_api.get_user_login()
655 except urllib.error.HTTPError as e:
656 raise LlvmPrError(f"Could not fetch user login from GitHub: {e}") from e
657
658 if not args.prefix:
659 args.prefix = f"users/{args.login}/"
660
661 if not args.prefix.endswith("/"):
662 args.prefix += "/"
663
664 try:
665 config = PRAutomatorConfig(
666 user_login=args.login,
667 token=token,
668 base_branch=args.base,
669 upstream_remote=args.upstream_remote,
670 prefix=args.prefix,
671 draft=args.draft,
672 labels=args.labels,
673 no_merge=args.no_merge,
674 auto_merge=args.auto_merge,
675 )
676 automator = LLVMPRAutomator(
677 runner=command_runner,
678 github_api=github_api,
679 config=config,
680 remote=args.remote,
681 )
682 automator.run()
683 except LlvmPrError as e:
684 sys.exit(f"Error: {e}")
685
686
687 if __name__ == "__main__":
688 main()