| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
# SPDX-FileCopyrightText: 2026 Oak Ridge National Laboratory and Contributors |
| 4 |
# |
| 5 |
# SPDX-License-Identifier: Apache-2.0 |
| 6 |
|
| 7 |
|
| 8 |
from datetime import datetime |
| 9 |
import argparse |
| 10 |
import itertools |
| 11 |
import requests |
| 12 |
import time |
| 13 |
import re |
| 14 |
import urllib3 |
| 15 |
import git |
| 16 |
import gitdb |
| 17 |
|
| 18 |
# Remove annoying warning about insecure connection (self-signed cert). |
| 19 |
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) |
| 20 |
|
| 21 |
|
| 22 |
class skip_after_n_successes: |
| 23 |
def __init__(self, default_value, n): |
| 24 |
self.runs_max = n |
| 25 |
self.runs_current = 0 |
| 26 |
self.default_value = default_value |
| 27 |
|
| 28 |
def __call__(self, fn, *args, **kwargs): |
| 29 |
if self.runs_current >= self.runs_max: |
| 30 |
return self.default_value |
| 31 |
|
| 32 |
ret = fn(*args, **kwargs) |
| 33 |
if ret: |
| 34 |
self.runs_current += 1 |
| 35 |
return ret |
| 36 |
|
| 37 |
|
| 38 |
def http_get_request(*args, **kwargs): |
| 39 |
kwargs["verify"] = False |
| 40 |
return requests.get(*args, **kwargs) |
| 41 |
|
| 42 |
|
| 43 |
def request_as_list(url, *args, **kwargs): |
| 44 |
current_url = url |
| 45 |
body_json = [] |
| 46 |
while current_url: |
| 47 |
response = http_get_request(current_url, *args, **kwargs) |
| 48 |
body_json += response.json() |
| 49 |
|
| 50 |
header = response.headers |
| 51 |
current_url = None |
| 52 |
if "link" in header: |
| 53 |
links = re.search( |
| 54 |
r'(?<=\<)([\S]*)(?=>; rel="next")', header["link"], flags=re.IGNORECASE |
| 55 |
) |
| 56 |
if links is not None: |
| 57 |
current_url = links.group(0) |
| 58 |
|
| 59 |
return body_json |
| 60 |
|
| 61 |
|
| 62 |
def add_timestamp(branch): |
| 63 |
date_str = branch["commit"]["committed_date"] |
| 64 |
# We ignore the TZ since Gitlab/GitHub always reports in UTC |
| 65 |
branch["dt"] = int(datetime.strptime(date_str.split(".")[0], "%Y-%m-%dT%H:%M:%S").timestamp()) |
| 66 |
return branch |
| 67 |
|
| 68 |
|
| 69 |
def is_descendant_of_root(branch, root_commit, repo): |
| 70 |
if root_commit is None: |
| 71 |
return True |
| 72 |
try: |
| 73 |
commit_sha = branch["commit"]["id"] |
| 74 |
repo.commit(commit_sha) |
| 75 |
merge_base = repo.merge_base(args.root_commit, commit_sha) |
| 76 |
if len(merge_base) >= 1 and str(merge_base[0]) == args.root_commit: |
| 77 |
return True |
| 78 |
except gitdb.exc.BadObject: |
| 79 |
return False |
| 80 |
|
| 81 |
|
| 82 |
def is_recent(branch): |
| 83 |
deadline_sec = int(time.time()) - (args.days * 86400) |
| 84 |
return branch["dt"] > deadline_sec |
| 85 |
|
| 86 |
|
| 87 |
def has_no_status(branch): |
| 88 |
gh_commit_sha = branch["commit"]["id"] |
| 89 |
# Backported branches use the merge head |
| 90 |
if re.fullmatch(r"^pr\d+_.*$", branch["name"]): |
| 91 |
gh_commit_sha = branch["commit"]["parent_ids"][1] |
| 92 |
|
| 93 |
# Query GitHub for the status of this commit |
| 94 |
response = http_get_request(gh_url + "/commits/" + gh_commit_sha + "/status") |
| 95 |
if int(response.headers["x-ratelimit-remaining"]) <= 0: |
| 96 |
raise ConnectionError(response.json()) |
| 97 |
|
| 98 |
commit = response.json() |
| 99 |
if commit is None or "sha" not in commit: |
| 100 |
return False |
| 101 |
|
| 102 |
for status in commit["statuses"]: |
| 103 |
if status["context"] == args.gh_context: |
| 104 |
return False |
| 105 |
|
| 106 |
return True |
| 107 |
|
| 108 |
|
| 109 |
parser = argparse.ArgumentParser( |
| 110 |
prog="generate_pipeline.py", description="Generate Dynamic pipelines for Gitlab" |
| 111 |
) |
| 112 |
parser.add_argument( |
| 113 |
"-u", |
| 114 |
"--gl-url", |
| 115 |
required=True, |
| 116 |
help="Base URL for Gitlab remote. Ex: https://code.olcf.ornl.gov/", |
| 117 |
) |
| 118 |
parser.add_argument( |
| 119 |
"-n", "--gh-name", required=True, help="Full name of the GitHub project. Ex: ornladios/ADIOS2" |
| 120 |
) |
| 121 |
parser.add_argument( |
| 122 |
"-p", "--project_id", required=True, help="Gitlab internal project ID of the project." |
| 123 |
) |
| 124 |
parser.add_argument( |
| 125 |
"-c", |
| 126 |
"--gh-context", |
| 127 |
default="OLCF Frontier", |
| 128 |
help="Name of the status in GitHub (A.K.A context)", |
| 129 |
) |
| 130 |
parser.add_argument( |
| 131 |
"-d", "--days", type=int, default=1, help="How many days back to search for commits" |
| 132 |
) |
| 133 |
parser.add_argument("-m", "--max", type=int, default=2, help="Maximum amount of pipelines computed") |
| 134 |
parser.add_argument( |
| 135 |
"-r", |
| 136 |
"--root-commit", |
| 137 |
default=None, |
| 138 |
type=str, |
| 139 |
help="Template file of the pipeline `{branch}` will be substituted", |
| 140 |
) |
| 141 |
parser.add_argument( |
| 142 |
"-f", |
| 143 |
"--template_file", |
| 144 |
required=True, |
| 145 |
help="Template file of the pipeline `{branch}` will be substituted", |
| 146 |
) |
| 147 |
args = parser.parse_args() |
| 148 |
|
| 149 |
|
| 150 |
gl_url = args.gl_url + "/api/v4/projects/" + str(args.project_id) |
| 151 |
gh_url = "https://api.github.com/repos/" + args.gh_name |
| 152 |
|
| 153 |
with open(args.template_file, "r") as fd: |
| 154 |
template_str = fd.read() |
| 155 |
|
| 156 |
repo = git.Repo(".", odbt=git.GitDB) |
| 157 |
|
| 158 |
branches = request_as_list(gl_url + "/repository/branches") |
| 159 |
branches = [b for b in branches if is_descendant_of_root(b, args.root_commit, repo)] |
| 160 |
branches = [add_timestamp(branch) for branch in branches] |
| 161 |
branches = [b for b in branches if is_recent(b)] |
| 162 |
branches = sorted(branches, key=lambda x: x["dt"]) |
| 163 |
|
| 164 |
# Skip running (and return true) has_no_status after returning True args.max times. |
| 165 |
# We need this not to hog the Github Rest API draconian ratelimit. |
| 166 |
run_n_times = skip_after_n_successes(default_value=False, n=args.max) |
| 167 |
branches = [b for b in branches if run_n_times(has_no_status, b)] |
| 168 |
|
| 169 |
for branch in branches: |
| 170 |
print(template_str.format(branch=branch["name"], commit=branch["commit"]["id"])) |