| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
# SPDX-FileCopyrightText: 2026 Oak Ridge National Laboratory and Contributors |
| 4 |
# |
| 5 |
# SPDX-License-Identifier: Apache-2.0 |
| 6 |
|
| 7 |
import argparse |
| 8 |
import atexit |
| 9 |
import base64 |
| 10 |
from datetime import datetime, timedelta, timezone |
| 11 |
import dateutil.parser |
| 12 |
from github import Github |
| 13 |
import json |
| 14 |
import os |
| 15 |
import re |
| 16 |
import subprocess |
| 17 |
import sys |
| 18 |
import tempfile |
| 19 |
import urllib.parse |
| 20 |
import urllib.request |
| 21 |
|
| 22 |
|
| 23 |
class SpackCIBridge(object): |
| 24 |
|
| 25 |
def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_project="", |
| 26 |
disable_status_post=True, status_context="", sync_draft_prs=False, |
| 27 |
main_branch=None, prereq_checks=[]): |
| 28 |
self.gitlab_repo = gitlab_repo |
| 29 |
self.github_project = github_project |
| 30 |
github_token = os.environ.get('GITHUB_TOKEN') |
| 31 |
self.github_repo = "https://{0}@github.com/{1}.git".format(github_token, self.github_project) |
| 32 |
self.py_github = Github(github_token) |
| 33 |
self.py_gh_repo = self.py_github.get_repo(self.github_project, lazy=True) |
| 34 |
self.status_context = status_context |
| 35 |
|
| 36 |
self.merge_msg_regex = re.compile(r"Merge\s+([^\s]+)\s+into\s+([^\s]+)") |
| 37 |
self.unmergeable_shas = [] |
| 38 |
|
| 39 |
self.post_status = not disable_status_post |
| 40 |
self.sync_draft_prs = sync_draft_prs |
| 41 |
self.main_branch = main_branch |
| 42 |
self.currently_running_sha = None |
| 43 |
self.latest_tested_main_commit = None |
| 44 |
|
| 45 |
self.prereq_checks = prereq_checks |
| 46 |
|
| 47 |
dt = datetime.now(timezone.utc) + timedelta(minutes=-60) |
| 48 |
self.time_threshold_brief = urllib.parse.quote_plus(dt.isoformat(timespec="seconds")) |
| 49 |
|
| 50 |
self.pipeline_api_template = gitlab_host |
| 51 |
self.pipeline_api_template += "/api/v4/projects/" |
| 52 |
self.pipeline_api_template += urllib.parse.quote_plus(gitlab_project) |
| 53 |
self.pipeline_api_template += "/pipelines?ref={0}" |
| 54 |
|
| 55 |
self.commit_api_template = gitlab_host |
| 56 |
self.commit_api_template += "/api/v4/projects/" |
| 57 |
self.commit_api_template += urllib.parse.quote_plus(gitlab_project) |
| 58 |
self.commit_api_template += "/repository/commits/{0}" |
| 59 |
|
| 60 |
self.cached_commits = {} |
| 61 |
|
| 62 |
@atexit.register |
| 63 |
def cleanup(): |
| 64 |
"""Shutdown ssh-agent upon program termination.""" |
| 65 |
if "SSH_AGENT_PID" in os.environ: |
| 66 |
print(" Shutting down ssh-agent({0})".format(os.environ["SSH_AGENT_PID"])) |
| 67 |
subprocess.run(["ssh-agent", "-k"], check=True) |
| 68 |
|
| 69 |
def setup_ssh(self, ssh_key_base64): |
| 70 |
"""Start the ssh agent.""" |
| 71 |
print("Starting ssh-agent") |
| 72 |
output = subprocess.run(["ssh-agent", "-s"], check=True, stdout=subprocess.PIPE).stdout |
| 73 |
|
| 74 |
# Search for PID in output. |
| 75 |
pid_regexp = re.compile(r"SSH_AGENT_PID=([0-9]+)") |
| 76 |
match = pid_regexp.search(output.decode("utf-8")) |
| 77 |
if match is None: |
| 78 |
print("WARNING: could not detect ssh-agent PID.", file=sys.stderr) |
| 79 |
print("ssh-agent will not be killed upon program termination", file=sys.stderr) |
| 80 |
else: |
| 81 |
pid = match.group(1) |
| 82 |
os.environ["SSH_AGENT_PID"] = pid |
| 83 |
self.cleanup_ssh_agent = True |
| 84 |
|
| 85 |
# Search for socket in output. |
| 86 |
socket_regexp = re.compile(r"SSH_AUTH_SOCK=([^;]+);") |
| 87 |
match = socket_regexp.search(output.decode("utf-8")) |
| 88 |
if match is None: |
| 89 |
print("WARNING: could not detect ssh-agent socket.", file=sys.stderr) |
| 90 |
print("Key will be added to caller's ssh-agent (if any)", file=sys.stderr) |
| 91 |
else: |
| 92 |
socket = match.group(1) |
| 93 |
os.environ["SSH_AUTH_SOCK"] = socket |
| 94 |
|
| 95 |
# Add the key. |
| 96 |
ssh_key = base64.b64decode(ssh_key_base64) |
| 97 |
ssh_key = ssh_key.replace(b"\r", b"") |
| 98 |
with tempfile.NamedTemporaryFile() as fp: |
| 99 |
fp.write(ssh_key) |
| 100 |
fp.seek(0) |
| 101 |
subprocess.run(["ssh-add", fp.name], check=True) |
| 102 |
|
| 103 |
def get_commit(self, commit): |
| 104 |
""" Check our cache for a commit on GitHub. |
| 105 |
If we don't have it yet, use the GitHub API to retrieve it.""" |
| 106 |
if commit not in self.cached_commits: |
| 107 |
self.cached_commits[commit] = self.py_gh_repo.get_commit(sha=commit) |
| 108 |
return self.cached_commits[commit] |
| 109 |
|
| 110 |
def list_github_prs(self): |
| 111 |
""" Return two dicts of data about open PRs on GitHub: |
| 112 |
one for all open PRs, and one for open PRs that are not up-to-date on GitLab.""" |
| 113 |
pr_dict = {} |
| 114 |
pulls = self.py_gh_repo.get_pulls(state="open") |
| 115 |
print("Rate limit after get_pulls(): {}".format(self.py_github.rate_limiting[0])) |
| 116 |
for pull in pulls: |
| 117 |
backlogged = False |
| 118 |
push = True |
| 119 |
if pull.draft and not self.sync_draft_prs: |
| 120 |
print("Skipping draft PR {0} ({1})".format(pull.number, pull.head.ref)) |
| 121 |
backlogged = "draft" |
| 122 |
push = False |
| 123 |
|
| 124 |
pr_string = "pr{0}_{1}".format(pull.number, pull.head.ref) |
| 125 |
|
| 126 |
if push and pull.updated_at < datetime.now() + timedelta(minutes=-2880): |
| 127 |
# Skip further analysis of this PR if it hasn't been updated in 48 hours. |
| 128 |
# This helps us avoid wasting our rate limit on PRs with merge conflicts. |
| 129 |
print("Skip pushing stale PR {0}".format(pr_string)) |
| 130 |
backlogged = "stale" |
| 131 |
push = False |
| 132 |
|
| 133 |
if push: |
| 134 |
# Determine if this PR still needs to be pushed to GitLab. This happens in one of two cases: |
| 135 |
# 1) we have never pushed it before |
| 136 |
# 2) we have pushed it before, but the HEAD sha has changed since we pushed it last |
| 137 |
log_args = ["git", "log", "--pretty=%s", "gitlab/{0}".format(pr_string)] |
| 138 |
try: |
| 139 |
merge_commit_msg = subprocess.run( |
| 140 |
log_args, check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout |
| 141 |
match = self.merge_msg_regex.match(merge_commit_msg.decode("utf-8")) |
| 142 |
if match and (match.group(1) == pull.head.sha or match.group(2) == pull.head.sha): |
| 143 |
print("Skip pushing {0} because GitLab already has HEAD {1}".format(pr_string, pull.head.sha)) |
| 144 |
push = False |
| 145 |
except subprocess.CalledProcessError: |
| 146 |
# This occurs when it's a new PR that hasn't been pushed to GitLab yet. |
| 147 |
pass |
| 148 |
|
| 149 |
if push: |
| 150 |
# Check the PRs-to-be-pushed to see if any of them should be considered "backlogged". |
| 151 |
# We currently recognize three types of backlogged PRs: |
| 152 |
# 1) Some required "prerequisite checks" have not yet completed successfully. |
| 153 |
# 2) The PR is based on a version of the "main branch" that has not yet been tested |
| 154 |
# 3) Draft PRs. Handled earlier in this function. |
| 155 |
if not backlogged and self.prereq_checks: |
| 156 |
checks_desc = "waiting for {} check to succeed" |
| 157 |
checks_to_verify = self.prereq_checks.copy() |
| 158 |
pr_check_runs = self.get_commit(pull.head.sha).get_check_runs() |
| 159 |
for check in pr_check_runs: |
| 160 |
if check.name in checks_to_verify: |
| 161 |
checks_to_verify.remove(check.name) |
| 162 |
if check.conclusion != "success": |
| 163 |
backlogged = checks_desc.format(check.name) |
| 164 |
push = False |
| 165 |
break |
| 166 |
if not backlogged and checks_to_verify: |
| 167 |
backlogged = checks_desc.format(checks_to_verify[0]) |
| 168 |
push = False |
| 169 |
if backlogged: |
| 170 |
print("Skip pushing {0} because of {1}".format(pr_string, backlogged)) |
| 171 |
|
| 172 |
if not backlogged: |
| 173 |
if self.main_branch and pull.base.ref == self.main_branch: |
| 174 |
# Check if we should defer pushing/testing this PR because it is based on "too new" of a commit |
| 175 |
# of the main branch. |
| 176 |
tmp_pr_branch = f"temporary_{pr_string}" |
| 177 |
subprocess.run(["git", "fetch", "github", |
| 178 |
f"refs/pull/{pull.number}/head:{tmp_pr_branch}"], check=True) |
| 179 |
# Get the merge base between this PR and the main branch. |
| 180 |
try: |
| 181 |
merge_base_sha = subprocess.run( |
| 182 |
["git", "merge-base", tmp_pr_branch, f"github/{self.main_branch}"], |
| 183 |
check=True, stdout=subprocess.PIPE).stdout.strip() |
| 184 |
except subprocess.CalledProcessError: |
| 185 |
print(f"'git merge-base {tmp_pr_branch} github/{self.main_branch}' " |
| 186 |
"returned non-zero. Skipping") |
| 187 |
self.unmergeable_shas.append(pull.head.sha) |
| 188 |
continue |
| 189 |
|
| 190 |
repo_head_sha = subprocess.run( |
| 191 |
["git", "rev-parse", tmp_pr_branch], |
| 192 |
check=True, stdout=subprocess.PIPE).stdout.decode("utf-8").strip() |
| 193 |
|
| 194 |
if pull.head.sha != repo_head_sha: |
| 195 |
# If gh repo and api don't agree on what the head sha is, don't |
| 196 |
# push. Instead log an error message and backlog the PR. |
| 197 |
a_sha, r_sha = pull.head.sha[:7], repo_head_sha[:7] |
| 198 |
print(f"Skip pushing {pr_string} because api says HEAD is {a_sha}, " |
| 199 |
f"while repo says HEAD is {r_sha}") |
| 200 |
backlogged = f"GitHub HEAD shas out of sync (repo={r_sha}, API={a_sha})" |
| 201 |
push = False |
| 202 |
# Check if our PR's merge base is an ancestor of the latest tested main branch commit. |
| 203 |
elif subprocess.run( |
| 204 |
["git", "merge-base", "--is-ancestor", merge_base_sha, self.latest_tested_main_commit] |
| 205 |
).returncode == 0: |
| 206 |
print(f"{tmp_pr_branch}'s merge base IS an ancestor of latest_tested_main " |
| 207 |
f"{merge_base_sha} vs. {self.latest_tested_main_commit}") |
| 208 |
try: |
| 209 |
subprocess.run(["git", "checkout", self.latest_tested_main_commit], check=True) |
| 210 |
subprocess.run(["git", "checkout", "-b", pr_string], check=True) |
| 211 |
commit_msg = f"Merge {pull.head.sha} into {self.latest_tested_main_commit}" |
| 212 |
subprocess.run( |
| 213 |
["git", "merge", "--no-ff", "-m", commit_msg, tmp_pr_branch], |
| 214 |
check=True) |
| 215 |
print(f"Merge succeeded, ready to push {pr_string} to GitLab for CI pipeline testing") |
| 216 |
except subprocess.CalledProcessError: |
| 217 |
print(f"Failed to merge PR {pull.number} ({pull.head.ref}) with latest tested " |
| 218 |
f"{self.main_branch} ({self.latest_tested_main_commit}). Skipping") |
| 219 |
self.unmergeable_shas.append(pull.head.sha) |
| 220 |
subprocess.run(["git", "merge", "--abort"]) |
| 221 |
backlogged = "merge conflicts with {}".format(self.main_branch) |
| 222 |
push = False |
| 223 |
continue |
| 224 |
else: |
| 225 |
print(f"Skip pushing {pr_string} because its merge base is NOT an ancestor of " |
| 226 |
f"latest_tested_main {merge_base_sha} vs. {self.latest_tested_main_commit}") |
| 227 |
backlogged = "base" |
| 228 |
push = False |
| 229 |
else: |
| 230 |
# If the --main-branch CLI argument wasn't passed, or if this PR doesn't target that branch, |
| 231 |
# then we will push the merge commit that was automatically created by GitHub to GitLab |
| 232 |
# where it will kick off a CI pipeline. |
| 233 |
try: |
| 234 |
subprocess.run(["git", "fetch", "github", |
| 235 |
f"{pull.merge_commit_sha}:{pr_string}"], check=True) |
| 236 |
except subprocess.CalledProcessError: |
| 237 |
print("Failed to locally checkout PR {0} ({1}). Skipping" |
| 238 |
.format(pull.number, pull.merge_commit_sha)) |
| 239 |
backlogged = "GitLab failed to checkout this branch" |
| 240 |
push = False |
| 241 |
continue |
| 242 |
|
| 243 |
pr_dict[pr_string] = { |
| 244 |
'base_sha': pull.base.sha, |
| 245 |
'head_sha': pull.head.sha, |
| 246 |
'push': push, |
| 247 |
'backlogged': backlogged, |
| 248 |
} |
| 249 |
|
| 250 |
def listify_dict(d): |
| 251 |
pr_strings = sorted(d.keys()) |
| 252 |
base_shas = [d[s]['base_sha'] for s in pr_strings] |
| 253 |
head_shas = [d[s]['head_sha'] for s in pr_strings] |
| 254 |
b_logged = [d[s]['backlogged'] for s in pr_strings] |
| 255 |
return { |
| 256 |
"pr_strings": pr_strings, |
| 257 |
"base_shas": base_shas, |
| 258 |
"head_shas": head_shas, |
| 259 |
"backlogged": b_logged, |
| 260 |
} |
| 261 |
all_open_prs = listify_dict(pr_dict) |
| 262 |
filtered_pr_dict = {k: v for (k, v) in pr_dict.items() if v['push']} |
| 263 |
filtered_open_prs = listify_dict(filtered_pr_dict) |
| 264 |
print("All Open PRs:") |
| 265 |
for pr_string in all_open_prs['pr_strings']: |
| 266 |
print(" {0}".format(pr_string)) |
| 267 |
print("Filtered Open PRs:") |
| 268 |
for pr_string in filtered_open_prs['pr_strings']: |
| 269 |
print(" {0}".format(pr_string)) |
| 270 |
print("Rate limit at the end of list_github_prs(): {}".format(self.py_github.rate_limiting[0])) |
| 271 |
return [all_open_prs, filtered_open_prs] |
| 272 |
|
| 273 |
def list_github_protected_branches(self): |
| 274 |
""" Return a list of protected branch names from GitHub.""" |
| 275 |
branches = self.py_gh_repo.get_branches() |
| 276 |
print("Rate limit after get_branches(): {}".format(self.py_github.rate_limiting[0])) |
| 277 |
protected_branches = [br.name for br in branches if br.protected] |
| 278 |
protected_branches = sorted(protected_branches) |
| 279 |
if self.currently_running_sha: |
| 280 |
print("Skip pushing {0} because it already has a pipeline running ({1})" |
| 281 |
.format(self.main_branch, self.currently_running_sha)) |
| 282 |
protected_branches.remove(self.main_branch) |
| 283 |
print("Protected branches:") |
| 284 |
for protected_branch in protected_branches: |
| 285 |
print(" {0}".format(protected_branch)) |
| 286 |
return protected_branches |
| 287 |
|
| 288 |
def list_github_tags(self): |
| 289 |
""" Return a list of tag names from GitHub.""" |
| 290 |
tag_list = self.py_gh_repo.get_tags() |
| 291 |
print("Rate limit after get_tags(): {}".format(self.py_github.rate_limiting[0])) |
| 292 |
tags = sorted([tag.name for tag in tag_list]) |
| 293 |
print("Tags:") |
| 294 |
for tag in tags: |
| 295 |
print(" {0}".format(tag)) |
| 296 |
return tags |
| 297 |
|
| 298 |
def setup_git_repo(self): |
| 299 |
"""Initialize a bare git repository with two remotes: |
| 300 |
one for GitHub and one for GitLab. |
| 301 |
If main_branch was specified, we also fetch that branch from GitHub. |
| 302 |
""" |
| 303 |
subprocess.run(["git", "init"], check=True) |
| 304 |
subprocess.run(["git", "config", "user.email", "[email protected]"], check=True) |
| 305 |
subprocess.run(["git", "config", "user.name", "spackbot"], check=True) |
| 306 |
subprocess.run(["git", "config", "advice.detachedHead", "false"], check=True) |
| 307 |
subprocess.run(["git", "remote", "add", "github", self.github_repo], check=True) |
| 308 |
subprocess.run(["git", "remote", "add", "gitlab", self.gitlab_repo], check=True) |
| 309 |
|
| 310 |
# Shallow fetch from GitLab. |
| 311 |
self.gitlab_shallow_fetch() |
| 312 |
|
| 313 |
if self.main_branch: |
| 314 |
subprocess.run(["git", "fetch", "github", self.main_branch], check=True) |
| 315 |
|
| 316 |
def get_gitlab_pr_branches(self): |
| 317 |
"""Query GitLab for branches that have already been copied over from GitHub PRs. |
| 318 |
Return the string output of `git branch --remotes --list gitlab/pr*`. |
| 319 |
""" |
| 320 |
branch_args = ["git", "branch", "--remotes", "--list", "gitlab/pr*"] |
| 321 |
self.gitlab_pr_output = \ |
| 322 |
subprocess.run(branch_args, check=True, stdout=subprocess.PIPE).stdout |
| 323 |
|
| 324 |
def gitlab_shallow_fetch(self): |
| 325 |
"""Perform a shallow fetch from GitLab""" |
| 326 |
fetch_args = ["git", "fetch", "-q", "--depth=1", "gitlab"] |
| 327 |
subprocess.run(fetch_args, check=True, stdout=subprocess.PIPE).stdout |
| 328 |
|
| 329 |
def get_open_refspecs(self, open_prs): |
| 330 |
"""Return a list of refspecs to push given a list of open PRs.""" |
| 331 |
print("Building initial lists of refspecs to fetch and push") |
| 332 |
pr_strings = open_prs["pr_strings"] |
| 333 |
base_shas = open_prs["base_shas"] |
| 334 |
backlogged = open_prs["backlogged"] |
| 335 |
open_refspecs = [] |
| 336 |
for open_pr, base_sha, backlog in zip(pr_strings, base_shas, backlogged): |
| 337 |
open_refspecs.append("{0}:{0}".format(open_pr)) |
| 338 |
print(" pushing {0} (based on {1})".format(open_pr, base_sha)) |
| 339 |
return open_refspecs |
| 340 |
|
| 341 |
def update_refspecs_for_protected_branches(self, protected_branches, open_refspecs, fetch_refspecs): |
| 342 |
"""Update our refspecs lists for protected branches from GitHub.""" |
| 343 |
for protected_branch in protected_branches: |
| 344 |
fetch_refspecs.append("+refs/heads/{0}:refs/remotes/{0}".format(protected_branch)) |
| 345 |
open_refspecs.append("refs/heads/{0}:refs/heads/{0}".format(protected_branch)) |
| 346 |
return open_refspecs, fetch_refspecs |
| 347 |
|
| 348 |
def update_refspecs_for_tags(self, tags, open_refspecs, fetch_refspecs): |
| 349 |
"""Update our refspecs lists for tags from GitHub.""" |
| 350 |
for tag in tags: |
| 351 |
fetch_refspecs.append("+refs/tags/{0}:refs/tags/{0}".format(tag)) |
| 352 |
open_refspecs.append("refs/tags/{0}:refs/tags/{0}".format(tag)) |
| 353 |
return open_refspecs, fetch_refspecs |
| 354 |
|
| 355 |
def fetch_github_branches(self, fetch_refspecs): |
| 356 |
"""Perform `git fetch` for a given list of refspecs.""" |
| 357 |
print("Fetching GitHub refs for open PRs") |
| 358 |
fetch_args = ["git", "fetch", "-q", "github"] + fetch_refspecs |
| 359 |
subprocess.run(fetch_args, check=True) |
| 360 |
|
| 361 |
def build_local_branches(self, protected_branches): |
| 362 |
"""Create local branches for a list of protected branches.""" |
| 363 |
print("Building local branches for protected branches") |
| 364 |
for branch in protected_branches: |
| 365 |
local_branch_name = "{0}".format(branch) |
| 366 |
remote_branch_name = "refs/remotes/{0}".format(branch) |
| 367 |
subprocess.run(["git", "branch", "-q", local_branch_name, remote_branch_name], check=True) |
| 368 |
|
| 369 |
def make_status_for_pipeline(self, pipeline): |
| 370 |
"""Generate POST data to create a GitHub status from a GitLab pipeline |
| 371 |
API response |
| 372 |
""" |
| 373 |
post_data = {} |
| 374 |
if "status" not in pipeline: |
| 375 |
return post_data |
| 376 |
|
| 377 |
if pipeline["status"] == "created": |
| 378 |
post_data["state"] = "pending" |
| 379 |
post_data["description"] = "Pipeline has been created" |
| 380 |
|
| 381 |
elif pipeline["status"] == "waiting_for_resource": |
| 382 |
post_data["state"] = "pending" |
| 383 |
post_data["description"] = "Pipeline is waiting for resources" |
| 384 |
|
| 385 |
elif pipeline["status"] == "preparing": |
| 386 |
post_data["state"] = "pending" |
| 387 |
post_data["description"] = "Pipeline is preparing" |
| 388 |
|
| 389 |
elif pipeline["status"] == "pending": |
| 390 |
post_data["state"] = "pending" |
| 391 |
post_data["description"] = "Pipeline is pending" |
| 392 |
|
| 393 |
elif pipeline["status"] == "running": |
| 394 |
post_data["state"] = "pending" |
| 395 |
post_data["description"] = "Pipeline is running" |
| 396 |
|
| 397 |
elif pipeline["status"] == "manual": |
| 398 |
post_data["state"] = "pending" |
| 399 |
post_data["description"] = "Pipeline is running manually" |
| 400 |
|
| 401 |
elif pipeline["status"] == "scheduled": |
| 402 |
post_data["state"] = "pending" |
| 403 |
post_data["description"] = "Pipeline is scheduled" |
| 404 |
|
| 405 |
elif pipeline["status"] == "failed": |
| 406 |
post_data["state"] = "error" |
| 407 |
post_data["description"] = "Pipeline failed" |
| 408 |
|
| 409 |
elif pipeline["status"] == "canceled": |
| 410 |
# Do not post canceled pipeline status to GitHub, it's confusing to our users. |
| 411 |
# This usually happens when a PR gets force-pushed. The next time the sync script runs |
| 412 |
# it will post a status for the newly force-pushed commit. |
| 413 |
return {} |
| 414 |
|
| 415 |
elif pipeline["status"] == "skipped": |
| 416 |
post_data["state"] = "failure" |
| 417 |
post_data["description"] = "Pipeline was skipped" |
| 418 |
|
| 419 |
elif pipeline["status"] == "success": |
| 420 |
post_data["state"] = "success" |
| 421 |
post_data["description"] = "Pipeline succeeded" |
| 422 |
|
| 423 |
post_data["target_url"] = pipeline["web_url"] |
| 424 |
return post_data |
| 425 |
|
| 426 |
def dedupe_pipelines(self, api_response): |
| 427 |
"""Prune pipelines API response to only include the most recent result for each SHA""" |
| 428 |
pipelines = {} |
| 429 |
for response in api_response: |
| 430 |
sha = response['sha'] |
| 431 |
if sha not in pipelines: |
| 432 |
pipelines[sha] = response |
| 433 |
else: |
| 434 |
existing_datetime = dateutil.parser.parse(pipelines[sha]['updated_at']) |
| 435 |
current_datetime = dateutil.parser.parse(response['updated_at']) |
| 436 |
if current_datetime > existing_datetime: |
| 437 |
pipelines[sha] = response |
| 438 |
return pipelines |
| 439 |
|
| 440 |
def find_pr_sha(self, tested_sha): |
| 441 |
api_url = self.commit_api_template.format(tested_sha) |
| 442 |
|
| 443 |
try: |
| 444 |
request = urllib.request.Request(api_url) |
| 445 |
if "GITLAB_TOKEN" in os.environ: |
| 446 |
request.add_header("Authorization", "Bearer %s" % os.environ["GITLAB_TOKEN"]) |
| 447 |
response = urllib.request.urlopen(request) |
| 448 |
except OSError: |
| 449 |
print('Failed to fetch commit for tested sha {0}'.format(tested_sha)) |
| 450 |
return None |
| 451 |
|
| 452 |
response_data = response.read() |
| 453 |
|
| 454 |
try: |
| 455 |
tested_commit_info = json.loads(response_data) |
| 456 |
except json.decoder.JSONDecodeError: |
| 457 |
print('Failed to parse response as json ({0})'.format(response_data)) |
| 458 |
return None |
| 459 |
|
| 460 |
if 'title' not in tested_commit_info: |
| 461 |
print('Returned commit object missing "Title" field') |
| 462 |
return None |
| 463 |
|
| 464 |
merge_commit_msg = tested_commit_info['title'] |
| 465 |
m = self.merge_msg_regex.match(merge_commit_msg) |
| 466 |
|
| 467 |
if m is None: |
| 468 |
print('Failed to find pr_sha in merge commit message') |
| 469 |
return None |
| 470 |
|
| 471 |
return m.group(1) |
| 472 |
|
| 473 |
def get_pipelines_for_branch(self, branch, time_threshold=None): |
| 474 |
# Use gitlab's API to get pipeline results for the corresponding ref. |
| 475 |
api_url = self.pipeline_api_template.format( |
| 476 |
urllib.parse.quote_plus(branch) |
| 477 |
) |
| 478 |
|
| 479 |
# Optionally constrain the query with the provided time_threshold |
| 480 |
if time_threshold: |
| 481 |
api_url = "{0}&updated_after={1}".format(api_url, time_threshold) |
| 482 |
|
| 483 |
try: |
| 484 |
request = urllib.request.Request(api_url) |
| 485 |
if "GITLAB_TOKEN" in os.environ: |
| 486 |
request.add_header("Authorization", "Bearer %s" % os.environ["GITLAB_TOKEN"]) |
| 487 |
response = urllib.request.urlopen(request) |
| 488 |
except OSError as inst: |
| 489 |
print("GitLab API request error accessing {0}".format(api_url)) |
| 490 |
print(inst) |
| 491 |
return None |
| 492 |
try: |
| 493 |
pipelines = json.loads(response.read()) |
| 494 |
except json.decoder.JSONDecodeError as inst: |
| 495 |
print("Error parsing response to {0}".format(api_url)) |
| 496 |
print(inst) |
| 497 |
return None |
| 498 |
|
| 499 |
return self.dedupe_pipelines(pipelines) |
| 500 |
|
| 501 |
def post_pipeline_status(self, open_prs, protected_branches): |
| 502 |
print("Rate limit at the beginning of post_pipeline_status(): {}".format(self.py_github.rate_limiting[0])) |
| 503 |
pipeline_branches = [] |
| 504 |
backlog_branches = [] |
| 505 |
# Split up the open_prs branches into two piles: branches we force-pushed to gitlab |
| 506 |
# and branches we deferred pushing. |
| 507 |
for pr_branch, base_sha, head_sha, backlog in zip(open_prs["pr_strings"], |
| 508 |
open_prs["base_shas"], |
| 509 |
open_prs["head_shas"], |
| 510 |
open_prs["backlogged"]): |
| 511 |
if not backlog: |
| 512 |
pipeline_branches.append(pr_branch) |
| 513 |
else: |
| 514 |
backlog_branches.append((pr_branch, head_sha, backlog)) |
| 515 |
|
| 516 |
pipeline_branches.extend(protected_branches) |
| 517 |
|
| 518 |
print('Querying pipelines to post status for:') |
| 519 |
for branch in pipeline_branches: |
| 520 |
# Post status to GitHub for each pipeline found. |
| 521 |
pipelines = self.get_pipelines_for_branch(branch, self.time_threshold_brief) |
| 522 |
if not pipelines: |
| 523 |
continue |
| 524 |
for sha, pipeline in pipelines.items(): |
| 525 |
post_data = self.make_status_for_pipeline(pipeline) |
| 526 |
if not post_data: |
| 527 |
continue |
| 528 |
# TODO: associate shas with protected branches, so we do not have to |
| 529 |
# hit an endpoint here, but just use the sha we already know just like |
| 530 |
# we do below for backlogged PR statuses. |
| 531 |
pr_sha = self.find_pr_sha(sha) |
| 532 |
if not pr_sha: |
| 533 |
print('Could not find github PR sha for tested commit: {0}'.format(sha)) |
| 534 |
print('Using tested commit to post status') |
| 535 |
pr_sha = sha |
| 536 |
self.create_status_for_commit(pr_sha, |
| 537 |
branch, |
| 538 |
post_data["state"], |
| 539 |
post_data["target_url"], |
| 540 |
post_data["description"]) |
| 541 |
|
| 542 |
# Post a status of pending/backlogged for branches we deferred pushing |
| 543 |
print("Posting backlogged status to the following:") |
| 544 |
base_backlog_desc = \ |
| 545 |
"This branch's merge-base with {} is newer than the latest commit tested by GitLab".format(self.main_branch) |
| 546 |
for branch, head_sha, reason in backlog_branches: |
| 547 |
if reason == "stale": |
| 548 |
print("Skip posting status for {} because it has not been updated recently".format(branch)) |
| 549 |
continue |
| 550 |
elif reason == "base": |
| 551 |
desc = base_backlog_desc |
| 552 |
url = "https://github.com/spack/spack-infrastructure/blob/main/docs/deferred_pipelines.md" |
| 553 |
elif reason == "draft": |
| 554 |
desc = "GitLab CI is disabled for draft PRs" |
| 555 |
url = "" |
| 556 |
else: |
| 557 |
desc = reason |
| 558 |
url = "" |
| 559 |
self.create_status_for_commit(head_sha, branch, "pending", url, desc) |
| 560 |
|
| 561 |
# Post errors to any PRs that we couldn't merge to latest_tested_main_commit. |
| 562 |
print('Posting unmergeable status to the following:') |
| 563 |
for sha in self.unmergeable_shas: |
| 564 |
print(' {0}'.format(sha)) |
| 565 |
self.create_status_for_commit(sha, "", "error", "", f"PR could not be merged with {self.main_branch}") |
| 566 |
print("Rate limit at the end of post_pipeline_status(): {}".format(self.py_github.rate_limiting[0])) |
| 567 |
|
| 568 |
def create_status_for_commit(self, sha, branch, state, target_url, description): |
| 569 |
context = self.status_context |
| 570 |
commit = self.get_commit(sha) |
| 571 |
existing_statuses = commit.get_combined_status() |
| 572 |
for status in existing_statuses.statuses: |
| 573 |
if (status.context == context and |
| 574 |
status.state == state and |
| 575 |
status.description == description and |
| 576 |
status.target_url == target_url): |
| 577 |
print("Not posting duplicate status to {} / {}".format(branch, sha)) |
| 578 |
return |
| 579 |
try: |
| 580 |
status_response = self.get_commit(sha).create_status( |
| 581 |
state=state, |
| 582 |
target_url=target_url, |
| 583 |
description=description, |
| 584 |
context=context |
| 585 |
) |
| 586 |
if status_response.state != state: |
| 587 |
print("Expected CommitStatus state {0}, got {1}".format( |
| 588 |
state, status_response.state)) |
| 589 |
except Exception as e_inst: |
| 590 |
print('Caught exception posting status for {0}/{1}'.format(branch, sha)) |
| 591 |
print(e_inst) |
| 592 |
print(" {0} -> {1}".format(branch, sha)) |
| 593 |
|
| 594 |
def sync(self): |
| 595 |
"""Synchronize pull requests from GitHub as branches on GitLab.""" |
| 596 |
|
| 597 |
print("Initial rate limit: {}".format(self.py_github.rate_limiting[0])) |
| 598 |
reset_time = datetime.utcfromtimestamp(self.py_github.rate_limiting_resettime).strftime('%Y-%m-%d %H:%M:%S') |
| 599 |
print("Rate limit will refresh at: {} UTC".format(reset_time)) |
| 600 |
|
| 601 |
# Setup SSH command for communicating with GitLab. |
| 602 |
os.environ["GIT_SSH_COMMAND"] = "ssh -F /dev/null -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" |
| 603 |
|
| 604 |
# Work inside a temporary directory that will be deleted when this script terminates. |
| 605 |
with tempfile.TemporaryDirectory() as tmpdirname: |
| 606 |
os.chdir(tmpdirname) |
| 607 |
|
| 608 |
# Setup the local repo with two remotes. |
| 609 |
self.setup_git_repo() |
| 610 |
|
| 611 |
if self.main_branch: |
| 612 |
# Find the currently running main branch pipeline, if any, and get the sha. |
| 613 |
# Also get the latest commit on the main branch that has a completed pipeline. |
| 614 |
main_branch_pipelines = self.get_pipelines_for_branch(self.main_branch) |
| 615 |
for sha, pipeline in main_branch_pipelines.items(): |
| 616 |
if self.latest_tested_main_commit is None and \ |
| 617 |
(pipeline['status'] == "success" or pipeline['status'] == "failed"): |
| 618 |
self.latest_tested_main_commit = sha |
| 619 |
|
| 620 |
if self.currently_running_sha is None and pipeline['status'] == "running": |
| 621 |
self.currently_running_sha = sha |
| 622 |
|
| 623 |
if self.latest_tested_main_commit and self.currently_running_sha: |
| 624 |
break |
| 625 |
|
| 626 |
print("Latest completed {0} pipeline: {1}".format(self.main_branch, self.latest_tested_main_commit)) |
| 627 |
print("Currently running {0} pipeline: {1}".format(self.main_branch, self.currently_running_sha)) |
| 628 |
|
| 629 |
# Retrieve open PRs from GitHub. |
| 630 |
all_open_prs, open_prs = self.list_github_prs() |
| 631 |
|
| 632 |
# Get protected branches on GitHub. |
| 633 |
protected_branches = self.list_github_protected_branches() |
| 634 |
|
| 635 |
# Get tags on GitHub. |
| 636 |
tags = self.list_github_tags() |
| 637 |
|
| 638 |
# Get refspecs for open PRs and protected branches. |
| 639 |
open_refspecs = self.get_open_refspecs(open_prs) |
| 640 |
fetch_refspecs = [] |
| 641 |
self.update_refspecs_for_protected_branches(protected_branches, open_refspecs, fetch_refspecs) |
| 642 |
self.update_refspecs_for_tags(tags, open_refspecs, fetch_refspecs) |
| 643 |
|
| 644 |
# Sync open GitHub PRs and protected branches to GitLab. |
| 645 |
self.fetch_github_branches(fetch_refspecs) |
| 646 |
self.build_local_branches(protected_branches) |
| 647 |
if open_refspecs: |
| 648 |
print("Syncing to GitLab") |
| 649 |
push_args = ["git", "push", "--porcelain", "-f", "gitlab"] + open_refspecs |
| 650 |
subprocess.run(push_args, check=True) |
| 651 |
|
| 652 |
# Post pipeline status to GitHub for each open PR, if enabled |
| 653 |
if self.post_status: |
| 654 |
print('Posting pipeline status for open PRs and protected branches') |
| 655 |
self.post_pipeline_status(all_open_prs, protected_branches) |
| 656 |
|
| 657 |
|
| 658 |
if __name__ == "__main__": |
| 659 |
# Parse command-line arguments. |
| 660 |
parser = argparse.ArgumentParser(description="Sync GitHub PRs to GitLab") |
| 661 |
parser.add_argument("github_project", help="GitHub project (org/repo or user/repo)") |
| 662 |
parser.add_argument("gitlab_repo", help="Full clone URL for GitLab") |
| 663 |
parser.add_argument("gitlab_host", help="GitLab web host") |
| 664 |
parser.add_argument("gitlab_project", help="GitLab project (org/repo or user/repo)") |
| 665 |
parser.add_argument("--disable-status-post", action="store_true", default=False, |
| 666 |
help="Do not post pipeline status to each GitHub PR") |
| 667 |
parser.add_argument("--status-context", type=str, default="External Gitlab pipeline", |
| 668 |
help="The name of the status in the github status check") |
| 669 |
parser.add_argument("--sync-draft-prs", action="store_true", default=False, |
| 670 |
help="Copy draft PRs from GitHub to GitLab") |
| 671 |
parser.add_argument("--pr-mirror-bucket", default=None, |
| 672 |
help="Delete mirrors for closed PRs from the specified S3 bucket") |
| 673 |
parser.add_argument("--main-branch", default=None, |
| 674 |
help="""If provided, we check if there is a currently running |
| 675 |
pipeline for this branch. If so, we defer syncing any subsequent commits in an effort |
| 676 |
to not interrupt this pipeline. We also defer pushing any PR branches that are based |
| 677 |
on a commit of the main branch that is newer than the latest commit tested by GitLab.""") |
| 678 |
parser.add_argument("--prereq-check", nargs="+", default=False, |
| 679 |
help="Only push branches that have already passed this GitHub check") |
| 680 |
|
| 681 |
args = parser.parse_args() |
| 682 |
|
| 683 |
ssh_key_base64 = os.getenv("GITLAB_SSH_KEY_BASE64") |
| 684 |
if ssh_key_base64 is None: |
| 685 |
raise Exception("GITLAB_SSH_KEY_BASE64 environment is not set") |
| 686 |
|
| 687 |
if "GITHUB_TOKEN" not in os.environ: |
| 688 |
raise Exception("GITHUB_TOKEN environment is not set") |
| 689 |
|
| 690 |
bridge = SpackCIBridge(gitlab_repo=args.gitlab_repo, |
| 691 |
gitlab_host=args.gitlab_host, |
| 692 |
gitlab_project=args.gitlab_project, |
| 693 |
github_project=args.github_project, |
| 694 |
disable_status_post=args.disable_status_post, |
| 695 |
status_context=args.status_context, |
| 696 |
sync_draft_prs=args.sync_draft_prs, |
| 697 |
main_branch=args.main_branch, |
| 698 |
prereq_checks=args.prereq_check) |
| 699 |
bridge.setup_ssh(ssh_key_base64) |
| 700 |
bridge.sync() |