Debian Codegraph @

mediawiki.py

raw
1 import requests
2 import time
3 from datetime import datetime
4 from email.utils import parsedate_to_datetime
5 from tenacity import retry, retry_if_result, stop_after_attempt
6
7 def is_none_p(value):
8 """Return True if value is None"""
9 return value is None
10
11 def get_retry_after_delay(response: requests.Response) -> int:
12 """
13 Parses the Retry-After header from an HTTP response and returns the
14 delay in seconds.
15 """
16 retry_after = response.headers.get("Retry-After")
17 if not retry_after:
18 return 0
19
20 try:
21 # Attempt to parse as an integer (delay in seconds)
22 delay_seconds = int(retry_after)
23 return delay_seconds
24 except ValueError:
25 # If not an integer, attempt to parse as an HTTP date
26 try:
27 retry_date = parsedate_to_datetime(retry_after)
28 now = datetime.now(retry_date.tzinfo)
29 delay_seconds = (retry_date - now).total_seconds()
30 return max(0, int(delay_seconds))
31 except (ValueError, TypeError):
32 raise
33
34 def init_session():
35 # https://wikitech.wikimedia.org/wiki/Robot_policy
36 global session
37 session = requests.Session()
38 headers = {
39 "User-Agent": f"User-Agent: FcitxZhwikiDictBot/1.0 (https://github.com/felixonmars/fcitx5-pinyin-zhwiki) python-requests/{requests.__version__}",
40 "Accept-Encoding": "gzip, deflate, br, zstd",
41 }
42 session.headers.update(headers)
43
44 @retry(stop=stop_after_attempt(10), retry=retry_if_result(is_none_p))
45 def do_request(url: str, params=None) -> requests.Response:
46 r = session.get(url, params=params)
47 if r.status_code == 200:
48 return r
49 elif r.status_code == 429:
50 delay = get_retry_after_delay(r)
51 time.sleep(delay)
52 return None
53 else:
54 r.raise_for_status()
55 return None