| 1 |
import requests |
| 2 |
import time |
| 3 |
from datetime import datetime |
| 4 |
from email.utils import parsedate_to_datetime |
| 5 |
from tenacity import retry, retry_if_result, stop_after_attempt |
| 6 |
|
| 7 |
def is_none_p(value): |
| 8 |
"""Return True if value is None""" |
| 9 |
return value is None |
| 10 |
|
| 11 |
def get_retry_after_delay(response: requests.Response) -> int: |
| 12 |
""" |
| 13 |
Parses the Retry-After header from an HTTP response and returns the |
| 14 |
delay in seconds. |
| 15 |
""" |
| 16 |
retry_after = response.headers.get("Retry-After") |
| 17 |
if not retry_after: |
| 18 |
return 0 |
| 19 |
|
| 20 |
try: |
| 21 |
# Attempt to parse as an integer (delay in seconds) |
| 22 |
delay_seconds = int(retry_after) |
| 23 |
return delay_seconds |
| 24 |
except ValueError: |
| 25 |
# If not an integer, attempt to parse as an HTTP date |
| 26 |
try: |
| 27 |
retry_date = parsedate_to_datetime(retry_after) |
| 28 |
now = datetime.now(retry_date.tzinfo) |
| 29 |
delay_seconds = (retry_date - now).total_seconds() |
| 30 |
return max(0, int(delay_seconds)) |
| 31 |
except (ValueError, TypeError): |
| 32 |
raise |
| 33 |
|
| 34 |
def init_session(): |
| 35 |
# https://wikitech.wikimedia.org/wiki/Robot_policy |
| 36 |
global session |
| 37 |
session = requests.Session() |
| 38 |
headers = { |
| 39 |
"User-Agent": f"User-Agent: FcitxZhwikiDictBot/1.0 (https://github.com/felixonmars/fcitx5-pinyin-zhwiki) python-requests/{requests.__version__}", |
| 40 |
"Accept-Encoding": "gzip, deflate, br, zstd", |
| 41 |
} |
| 42 |
session.headers.update(headers) |
| 43 |
|
| 44 |
@retry(stop=stop_after_attempt(10), retry=retry_if_result(is_none_p)) |
| 45 |
def do_request(url: str, params=None) -> requests.Response: |
| 46 |
r = session.get(url, params=params) |
| 47 |
if r.status_code == 200: |
| 48 |
return r |
| 49 |
elif r.status_code == 429: |
| 50 |
delay = get_retry_after_delay(r) |
| 51 |
time.sleep(delay) |
| 52 |
return None |
| 53 |
else: |
| 54 |
r.raise_for_status() |
| 55 |
return None |