Debian Codegraph @

.gitlab/config/generate_pipelines.py

raw
1 #!/usr/bin/env python3
2
3 # SPDX-FileCopyrightText: 2026 Oak Ridge National Laboratory and Contributors
4 #
5 # SPDX-License-Identifier: Apache-2.0
6
7
8 from datetime import datetime
9 import argparse
10 import itertools
11 import requests
12 import time
13 import re
14 import urllib3
15 import git
16 import gitdb
17
18 # Remove annoying warning about insecure connection (self-signed cert).
19 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
20
21
22 class skip_after_n_successes:
23 def __init__(self, default_value, n):
24 self.runs_max = n
25 self.runs_current = 0
26 self.default_value = default_value
27
28 def __call__(self, fn, *args, **kwargs):
29 if self.runs_current >= self.runs_max:
30 return self.default_value
31
32 ret = fn(*args, **kwargs)
33 if ret:
34 self.runs_current += 1
35 return ret
36
37
38 def http_get_request(*args, **kwargs):
39 kwargs["verify"] = False
40 return requests.get(*args, **kwargs)
41
42
43 def request_as_list(url, *args, **kwargs):
44 current_url = url
45 body_json = []
46 while current_url:
47 response = http_get_request(current_url, *args, **kwargs)
48 body_json += response.json()
49
50 header = response.headers
51 current_url = None
52 if "link" in header:
53 links = re.search(
54 r'(?<=\<)([\S]*)(?=>; rel="next")', header["link"], flags=re.IGNORECASE
55 )
56 if links is not None:
57 current_url = links.group(0)
58
59 return body_json
60
61
62 def add_timestamp(branch):
63 date_str = branch["commit"]["committed_date"]
64 # We ignore the TZ since Gitlab/GitHub always reports in UTC
65 branch["dt"] = int(datetime.strptime(date_str.split(".")[0], "%Y-%m-%dT%H:%M:%S").timestamp())
66 return branch
67
68
69 def is_descendant_of_root(branch, root_commit, repo):
70 if root_commit is None:
71 return True
72 try:
73 commit_sha = branch["commit"]["id"]
74 repo.commit(commit_sha)
75 merge_base = repo.merge_base(args.root_commit, commit_sha)
76 if len(merge_base) >= 1 and str(merge_base[0]) == args.root_commit:
77 return True
78 except gitdb.exc.BadObject:
79 return False
80
81
82 def is_recent(branch):
83 deadline_sec = int(time.time()) - (args.days * 86400)
84 return branch["dt"] > deadline_sec
85
86
87 def has_no_status(branch):
88 gh_commit_sha = branch["commit"]["id"]
89 # Backported branches use the merge head
90 if re.fullmatch(r"^pr\d+_.*$", branch["name"]):
91 gh_commit_sha = branch["commit"]["parent_ids"][1]
92
93 # Query GitHub for the status of this commit
94 response = http_get_request(gh_url + "/commits/" + gh_commit_sha + "/status")
95 if int(response.headers["x-ratelimit-remaining"]) <= 0:
96 raise ConnectionError(response.json())
97
98 commit = response.json()
99 if commit is None or "sha" not in commit:
100 return False
101
102 for status in commit["statuses"]:
103 if status["context"] == args.gh_context:
104 return False
105
106 return True
107
108
109 parser = argparse.ArgumentParser(
110 prog="generate_pipeline.py", description="Generate Dynamic pipelines for Gitlab"
111 )
112 parser.add_argument(
113 "-u",
114 "--gl-url",
115 required=True,
116 help="Base URL for Gitlab remote. Ex: https://code.olcf.ornl.gov/",
117 )
118 parser.add_argument(
119 "-n", "--gh-name", required=True, help="Full name of the GitHub project. Ex: ornladios/ADIOS2"
120 )
121 parser.add_argument(
122 "-p", "--project_id", required=True, help="Gitlab internal project ID of the project."
123 )
124 parser.add_argument(
125 "-c",
126 "--gh-context",
127 default="OLCF Frontier",
128 help="Name of the status in GitHub (A.K.A context)",
129 )
130 parser.add_argument(
131 "-d", "--days", type=int, default=1, help="How many days back to search for commits"
132 )
133 parser.add_argument("-m", "--max", type=int, default=2, help="Maximum amount of pipelines computed")
134 parser.add_argument(
135 "-r",
136 "--root-commit",
137 default=None,
138 type=str,
139 help="Template file of the pipeline `{branch}` will be substituted",
140 )
141 parser.add_argument(
142 "-f",
143 "--template_file",
144 required=True,
145 help="Template file of the pipeline `{branch}` will be substituted",
146 )
147 args = parser.parse_args()
148
149
150 gl_url = args.gl_url + "/api/v4/projects/" + str(args.project_id)
151 gh_url = "https://api.github.com/repos/" + args.gh_name
152
153 with open(args.template_file, "r") as fd:
154 template_str = fd.read()
155
156 repo = git.Repo(".", odbt=git.GitDB)
157
158 branches = request_as_list(gl_url + "/repository/branches")
159 branches = [b for b in branches if is_descendant_of_root(b, args.root_commit, repo)]
160 branches = [add_timestamp(branch) for branch in branches]
161 branches = [b for b in branches if is_recent(b)]
162 branches = sorted(branches, key=lambda x: x["dt"])
163
164 # Skip running (and return true) has_no_status after returning True args.max times.
165 # We need this not to hog the Github Rest API draconian ratelimit.
166 run_n_times = skip_after_n_successes(default_value=False, n=args.max)
167 branches = [b for b in branches if run_n_times(has_no_status, b)]
168
169 for branch in branches:
170 print(template_str.format(branch=branch["name"], commit=branch["commit"]["id"]))