Debian Codegraph @

vmtest/githubapi.py

raw
1 # Copyright (c) Meta Platforms, Inc. and affiliates.
2 # SPDX-License-Identifier: LGPL-2.1-or-later
3
4 import json
5 import os
6 from pathlib import Path
7 import typing
8 from typing import Any, Dict, Mapping, Optional, Union
9 import urllib.error
10 import urllib.parse
11 import urllib.request
12
13 if typing.TYPE_CHECKING:
14 import aiohttp
15
16
17 _CACHE = Optional[Union[str, bytes, Path]]
18
19
20 # Hacky base class because we want the GitHub API from async and non-async
21 # code.
22 #
23 # This provides a slapdash interface for caching a response in a file so that
24 # we can do conditional requests
25 # (https://docs.github.com/en/rest/overview/resources-in-the-rest-api#conditional-requests).
26 # A more complete implementation would be something like a SQLite database
27 # indexed by endpoint, but this is simpler and good enough for now.
28 class _GitHubApiBase:
29 _HOST = "https://api.github.com"
30
31 def __init__(self, token: Optional[str]) -> None:
32 self._headers = {
33 "Accept": "application/vnd.github.v3+json",
34 "User-Agent": "osandov/drgn vmtest",
35 }
36 if token is not None:
37 self._headers["Authorization"] = "token " + token
38
39 def _request(
40 self,
41 method: str,
42 url: str,
43 *,
44 params: Optional[Mapping[str, str]] = None,
45 headers: Optional[Dict[str, str]] = None,
46 data: Any = None,
47 ) -> Any:
48 raise NotImplementedError()
49
50 def _cached_get_json(self, endpoint: str, cache: _CACHE) -> Any:
51 raise NotImplementedError()
52
53 def _read_cache(self, cache: _CACHE) -> Optional[Mapping[str, Any]]:
54 if not cache:
55 return None
56 try:
57 with open(cache, "r") as f:
58 return json.load(f)
59 except FileNotFoundError:
60 return None
61
62 def _cached_get_headers(
63 self, cached: Optional[Mapping[str, Any]]
64 ) -> Dict[str, str]:
65 if cached is not None:
66 if "etag" in cached:
67 return {**self._headers, "If-None-Match": cached["etag"]}
68 elif "last_modified" in cached:
69 return {**self._headers, "If-Modified-Since": cached["last_modified"]}
70 return self._headers
71
72 def _write_cache(
73 self, cache: _CACHE, body: Any, headers: Mapping[str, str]
74 ) -> None:
75 if cache is not None and ("ETag" in headers or "Last-Modified" in headers):
76 to_cache = {"body": body}
77 if "ETag" in headers:
78 to_cache["etag"] = headers["ETag"]
79 if "Last-Modified" in headers:
80 to_cache["last_modified"] = headers["Last-Modified"]
81 with open(cache, "w") as f:
82 json.dump(to_cache, f)
83
84 def get_release_by_tag(
85 self, owner: str, repo: str, tag: str, *, cache: _CACHE = None
86 ) -> Any:
87 return self._cached_get_json(f"repos/{owner}/{repo}/releases/tags/{tag}", cache)
88
89 def download(self, url: str) -> Any:
90 return self._request(
91 "GET", url, headers={**self._headers, "Accept": "application/octet-stream"}
92 )
93
94 def upload(
95 self,
96 url: str,
97 data: Any,
98 content_type: str,
99 content_length: Optional[int] = None,
100 ) -> Any:
101 headers = {**self._headers, "Content-Type": content_type}
102 if content_length is not None:
103 headers["Content-Length"] = str(content_length)
104 return self._request(
105 "POST",
106 url,
107 headers=headers,
108 data=data,
109 )
110
111 def delete(self, url: str) -> Any:
112 return self._request("DELETE", url, headers=self._headers)
113
114
115 class GitHubApi(_GitHubApiBase):
116 def _request(
117 self,
118 method: str,
119 url: str,
120 *,
121 params: Optional[Mapping[str, str]] = None,
122 headers: Optional[Dict[str, str]] = None,
123 data: Any = None,
124 ) -> Any:
125 if params:
126 url += "?" + urllib.parse.urlencode(params)
127 req = urllib.request.Request(
128 url,
129 data=data,
130 headers={} if headers is None else headers,
131 method=method,
132 )
133 # Work around python/cpython#77842.
134 authorization = req.get_header("Authorization")
135 if authorization is not None:
136 req.remove_header("Authorization")
137 req.add_unredirected_header("Authorization", authorization)
138 return urllib.request.urlopen(req)
139
140 def _cached_get_json(self, endpoint: str, cache: _CACHE) -> Any:
141 cached = self._read_cache(cache)
142 # If the request was cached and the VMTEST_TRUST_CACHE environment
143 # variable is set, assume the cache is still valid.
144 if cached is not None and "VMTEST_TRUST_CACHE" in os.environ:
145 return cached["body"]
146 req = urllib.request.Request(
147 self._HOST + "/" + endpoint,
148 headers=self._cached_get_headers(cached),
149 )
150 # Work around python/cpython#77842.
151 authorization = req.get_header("Authorization")
152 if authorization is not None:
153 req.remove_header("Authorization")
154 req.add_unredirected_header("Authorization", authorization)
155 try:
156 with urllib.request.urlopen(req) as resp:
157 body = json.load(resp)
158 self._write_cache(cache, body, resp.headers)
159 return body
160 except urllib.error.HTTPError as e:
161 if e.code == 304 and cached is not None:
162 return cached["body"]
163 else:
164 raise
165
166
167 class AioGitHubApi(_GitHubApiBase):
168 def __init__(self, session: "aiohttp.ClientSession", token: Optional[str]) -> None:
169 super().__init__(token)
170 self._session = session
171
172 def _request(
173 self,
174 method: str,
175 url: str,
176 *,
177 params: Optional[Mapping[str, str]] = None,
178 headers: Optional[Dict[str, str]] = None,
179 data: Any = None,
180 ) -> Any:
181 return self._session.request(
182 method,
183 url,
184 params=params,
185 headers=headers,
186 data=data,
187 raise_for_status=True,
188 )
189
190 async def _cached_get_json(self, endpoint: str, cache: _CACHE) -> Any:
191 cached = self._read_cache(cache)
192 if cached is not None and "VMTEST_TRUST_CACHE" in os.environ:
193 return cached["body"]
194 async with self._session.get(
195 self._HOST + "/" + endpoint,
196 headers=self._cached_get_headers(cached),
197 raise_for_status=True,
198 ) as resp:
199 if resp.status == 304:
200 if cached is None:
201 raise Exception("got HTTP 304 but response was not cached")
202 return cached["body"]
203 else:
204 body = await resp.json()
205 self._write_cache(cache, body, resp.headers)
206 return body