| 1 |
import re |
| 2 |
import time as time_mod |
| 3 |
from datetime import datetime |
| 4 |
|
| 5 |
from calibre.utils.date import utcnow |
| 6 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 7 |
|
| 8 |
|
| 9 |
class TimeMagazineUSA(BasicNewsRecipe): |
| 10 |
title = 'TIME Magazine' |
| 11 |
__author__ = 'Monkfishare' |
| 12 |
description = 'Weekly US magazine.' |
| 13 |
language = 'en' |
| 14 |
timefmt = '' |
| 15 |
no_stylesheets = True |
| 16 |
remove_javascript = True |
| 17 |
remove_empty_feeds = True |
| 18 |
masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/b/b3/Time_Magazine_logo.svg/960px-Time_Magazine_logo.svg.png' |
| 19 |
|
| 20 |
recipe_specific_options = { |
| 21 |
'edition': { |
| 22 |
'short': 'Edition URL', |
| 23 |
'long': ( |
| 24 |
'For example, https://time.com/magazine/us/7362702/february-9th-2026-vol-207-no-3-u-s/\n' |
| 25 |
'Browse past editions at https://time.com/vault/year/2026 (change the year as needed).\n' |
| 26 |
'Leave the URL field blank to automatically download the latest US edition.' |
| 27 |
), |
| 28 |
'default': '', |
| 29 |
}, |
| 30 |
'res': { |
| 31 |
'short': 'Image resolution (width in pixels)', |
| 32 |
'long': ('Recommended for e-ink devices: 1200, 1600, or 2000 (default)\nUse 400 or 800 for smaller file sizes or non e-ink devices.'), |
| 33 |
'default': '2000', |
| 34 |
}, |
| 35 |
} |
| 36 |
|
| 37 |
keep_only_tags = [ |
| 38 |
dict(name='section', attrs={'aria-labelledby': lambda v: v and (v == 'intro-section' or re.match(r'^[a-z][a-z0-9_]*$', v))}), |
| 39 |
dict(name='article'), |
| 40 |
] |
| 41 |
|
| 42 |
remove_tags = [ |
| 43 |
dict( |
| 44 |
attrs={ |
| 45 |
'class': lambda c: ( |
| 46 |
c |
| 47 |
and any( |
| 48 |
x in c |
| 49 |
for x in [ |
| 50 |
'bg-warm-grey-ads', |
| 51 |
'native-ad', |
| 52 |
'advertisement', |
| 53 |
'sr-only', |
| 54 |
'google-source', |
| 55 |
'print:hidden', |
| 56 |
'rightrail', |
| 57 |
] |
| 58 |
) |
| 59 |
) |
| 60 |
} |
| 61 |
), |
| 62 |
dict(attrs={'data-testid': 'authors-list'}), |
| 63 |
dict(name='button'), |
| 64 |
dict(name='svg'), |
| 65 |
dict(attrs={'aria-hidden': 'true'}), |
| 66 |
dict(attrs={'role': 'tooltip'}), |
| 67 |
dict(attrs={'class': lambda c: c and 'items-center' in c and 'flex-row' in c}), |
| 68 |
dict(attrs={'class': lambda c: c and 'relative' in c and 'inline-flex' in c and 'w-max' in c}), |
| 69 |
] |
| 70 |
|
| 71 |
extra_css = ''' |
| 72 |
h1 { font-size: 1.6em; } |
| 73 |
h2, h3 { font-size: 1.2em; } |
| 74 |
time { color: #666; font-size: 0.85em; } |
| 75 |
img { display: block; max-width: 100%; height: auto; |
| 76 |
position: static !important; float: none !important; } |
| 77 |
figure { display: block; position: static !important; |
| 78 |
margin: 1em 0; overflow: visible; } |
| 79 |
''' |
| 80 |
|
| 81 |
def get_browser(self): |
| 82 |
br = BasicNewsRecipe.get_browser(self) |
| 83 |
br.addheaders += [ |
| 84 |
('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'), |
| 85 |
('Accept-Language', 'en-US,en;q=0.9'), |
| 86 |
('Sec-Fetch-Mode', 'navigate'), |
| 87 |
('Sec-Fetch-Dest', 'document'), |
| 88 |
('Sec-Fetch-Site', 'none'), |
| 89 |
('Upgrade-Insecure-Requests', '1'), |
| 90 |
] |
| 91 |
return br |
| 92 |
|
| 93 |
def _img_width(self): |
| 94 |
w = self.recipe_specific_options.get('res') |
| 95 |
if w and isinstance(w, str) and w.isdigit(): |
| 96 |
return w |
| 97 |
return '2000' |
| 98 |
|
| 99 |
def _set_img_width(self, src): |
| 100 |
src = re.sub(r'[?&]width=\d+', '', src).rstrip('?&') |
| 101 |
sep = '&' if '?' in src else '?' |
| 102 |
return src + sep + 'width=' + self._img_width() |
| 103 |
|
| 104 |
def _sanitize_nested_anchors(self, raw_html): |
| 105 |
return re.sub( |
| 106 |
r'<a\b([^>]*\bhref="/section/[^"]*"[^>]*)>(.*?)</a>', |
| 107 |
r'<span\1>\2</span>', |
| 108 |
raw_html, |
| 109 |
flags=re.DOTALL, |
| 110 |
) |
| 111 |
|
| 112 |
def _find_latest_us_edition(self, soup): |
| 113 |
candidates = [] |
| 114 |
for art in soup.find_all('article'): |
| 115 |
a = art.find('a', href=True) |
| 116 |
if not a: |
| 117 |
continue |
| 118 |
href = a['href'] |
| 119 |
if '/magazine/us/' not in href: |
| 120 |
continue |
| 121 |
time_tag = art.find('time', attrs={'datetime': True}) |
| 122 |
if not time_tag: |
| 123 |
continue |
| 124 |
try: |
| 125 |
dt = datetime.fromisoformat(time_tag['datetime']) |
| 126 |
candidates.append((dt, href, a.get_text(strip=True))) |
| 127 |
except Exception: |
| 128 |
pass |
| 129 |
if not candidates: |
| 130 |
return None, None |
| 131 |
candidates.sort(key=lambda x: x[0], reverse=True) |
| 132 |
return candidates[0][1], candidates[0][2] |
| 133 |
|
| 134 |
def _parse_toc_from_soup(self, soup): |
| 135 |
toc_container = soup.find('div', class_=lambda c: c and 'columns-1' in c) |
| 136 |
if not toc_container: |
| 137 |
return None |
| 138 |
|
| 139 |
sections = [] |
| 140 |
for section_div in toc_container.find_all('div', class_=lambda c: c and 'break-inside-avoid' in c): |
| 141 |
h3 = section_div.find('h3', class_=lambda c: c and 'text-4xl' in c) |
| 142 |
section_name = h3.get_text(strip=True) if h3 else None |
| 143 |
if not section_name: |
| 144 |
continue |
| 145 |
|
| 146 |
articles = [] |
| 147 |
for title_h3 in section_div.find_all('h3', class_=lambda c: c and 'font-editorial' in c): |
| 148 |
title = title_h3.get_text(strip=True) |
| 149 |
a = title_h3.find_parent('a', href=True) |
| 150 |
if not a: |
| 151 |
continue |
| 152 |
href = a['href'] |
| 153 |
if not re.match(r'^(/\d{7,}/|/article/|/collection/)', href): |
| 154 |
continue |
| 155 |
if title: |
| 156 |
p = a.find('p', class_=lambda c: c and 'text-t4' in c) |
| 157 |
desc = p.get_text(strip=True) if p else '' |
| 158 |
self.log(' ', title) |
| 159 |
articles.append({'title': title, 'url': 'https://time.com' + href, 'description': desc}) |
| 160 |
|
| 161 |
if articles: |
| 162 |
self.log(section_name) |
| 163 |
sections.append((section_name, articles)) |
| 164 |
|
| 165 |
return sections or None |
| 166 |
|
| 167 |
def _slug_to_title(self, path): |
| 168 |
parts = path.strip('/').split('/') |
| 169 |
slug = parts[1] if len(parts) >= 2 else parts[0] |
| 170 |
return slug.replace('-', ' ').title() |
| 171 |
|
| 172 |
def _parse_toc_from_raw(self, raw_html): |
| 173 |
text = raw_html.replace('\\/', '/') |
| 174 |
text = re.sub(r'\\u002[Ff]', '/', text) |
| 175 |
|
| 176 |
combined = re.compile( |
| 177 |
r'(?:' |
| 178 |
r'/\d{7,}/[a-z0-9][a-z0-9\-]*/' |
| 179 |
r'|/article/\d{4}/\d{2}/\d{2}/[a-z0-9][a-z0-9\-]*/' |
| 180 |
r'|/collection/[a-z0-9\-]+/\d{4}/[a-z0-9\-]*/' |
| 181 |
r')' |
| 182 |
) |
| 183 |
|
| 184 |
seen = set() |
| 185 |
paths = [] |
| 186 |
for m in combined.finditer(text): |
| 187 |
path = m.group(0) |
| 188 |
if re.search(r'/magazine/|/section/|/vault/', path): |
| 189 |
continue |
| 190 |
if path not in seen: |
| 191 |
seen.add(path) |
| 192 |
paths.append(path) |
| 193 |
|
| 194 |
if not paths: |
| 195 |
return None |
| 196 |
|
| 197 |
articles = [] |
| 198 |
for path in paths: |
| 199 |
url = 'https://time.com' + path |
| 200 |
try: |
| 201 |
art_soup = self.index_to_soup(url) |
| 202 |
h1 = art_soup.find('h1') |
| 203 |
title = h1.get_text(strip=True) if h1 else self._slug_to_title(path) |
| 204 |
except Exception: |
| 205 |
title = self._slug_to_title(path) |
| 206 |
self.log('\t', title) |
| 207 |
articles.append({'title': title, 'url': url}) |
| 208 |
time_mod.sleep(0.5) |
| 209 |
|
| 210 |
return [('Articles', articles)] |
| 211 |
|
| 212 |
def _fetch_raw(self, url): |
| 213 |
resp = self.browser.open(url) |
| 214 |
return resp.read().decode('utf-8', errors='replace') |
| 215 |
|
| 216 |
def _title_from_path(self, path): |
| 217 |
slug = path.strip('/').split('/')[-1] |
| 218 |
m = re.match(r'([a-z]+)-(\d+)[a-z]*-(\d{4})', slug) |
| 219 |
if m: |
| 220 |
month, day, year = m.group(1).capitalize(), m.group(2), m.group(3) |
| 221 |
return '%s %s, %s' % (month, day, year) |
| 222 |
return slug.replace('-', ' ').title() |
| 223 |
|
| 224 |
def parse_index(self): |
| 225 |
opts = self.recipe_specific_options |
| 226 |
custom_path = (opts.get('edition') or '').strip() |
| 227 |
|
| 228 |
if custom_path: |
| 229 |
edition_url = custom_path if custom_path.startswith('http') else 'https://time.com' + custom_path |
| 230 |
date_str = self._title_from_path(custom_path) |
| 231 |
self.log('Using custom edition:', edition_url) |
| 232 |
else: |
| 233 |
year = utcnow().year |
| 234 |
archive_soup = self.index_to_soup('https://time.com/vault/year/%d/' % year) |
| 235 |
edition_url, edition_title = self._find_latest_us_edition(archive_soup) |
| 236 |
if not edition_url: |
| 237 |
archive_soup = self.index_to_soup('https://time.com/vault/year/%d/' % (year - 1)) |
| 238 |
edition_url, edition_title = self._find_latest_us_edition(archive_soup) |
| 239 |
if not edition_url: |
| 240 |
raise ValueError('Could not find latest US edition') |
| 241 |
self.log('Found edition:', edition_url) |
| 242 |
date_str = edition_title.split('|')[0].strip() if edition_title else 'USA' |
| 243 |
|
| 244 |
self.title = 'TIME Magazine' |
| 245 |
self.timefmt = ' [' + date_str + ']' |
| 246 |
|
| 247 |
raw_html = self._fetch_raw(edition_url) |
| 248 |
raw_html = self._sanitize_nested_anchors(raw_html) |
| 249 |
|
| 250 |
from calibre.ebooks.BeautifulSoup import BeautifulSoup |
| 251 |
|
| 252 |
soup = BeautifulSoup(raw_html, 'html.parser') |
| 253 |
|
| 254 |
for span in soup.find_all('span', class_=lambda c: c and 'text-t3' in c): |
| 255 |
if span.get_text(strip=True).lower() == 'issue name': |
| 256 |
p = span.find_next_sibling('p') |
| 257 |
if p: |
| 258 |
full_issue_name = p.get_text(strip=True) |
| 259 |
date_str = full_issue_name.split('|')[0].strip() |
| 260 |
if date_str: |
| 261 |
self.timefmt = ' [' + date_str + ']' |
| 262 |
self.log('Issue name:', full_issue_name) |
| 263 |
break |
| 264 |
|
| 265 |
figure = soup.find('figure') |
| 266 |
if figure: |
| 267 |
img = figure.find('img') |
| 268 |
if img: |
| 269 |
src = img.get('src', '') |
| 270 |
self.cover_url = self._set_img_width(src) |
| 271 |
self.log('Cover:', self.cover_url) |
| 272 |
|
| 273 |
sections = self._parse_toc_from_soup(soup) |
| 274 |
if sections: |
| 275 |
self.log('Parsed TOC from HTML structure (%d sections)' % len(sections)) |
| 276 |
return sections |
| 277 |
|
| 278 |
self.log('TOC not in HTML structure, trying raw payload extraction') |
| 279 |
sections = self._parse_toc_from_raw(raw_html) |
| 280 |
if sections: |
| 281 |
self.log('Extracted %d article URLs from raw payload' % len(sections[0][1])) |
| 282 |
return sections |
| 283 |
|
| 284 |
raise ValueError('Could not parse edition TOC from either HTML or raw payload. Response length: %d bytes' % len(raw_html)) |
| 285 |
|
| 286 |
def preprocess_html(self, soup): |
| 287 |
for picture in soup.find_all('picture'): |
| 288 |
img = picture.find('img') |
| 289 |
if img: |
| 290 |
for attr in ('srcset', 'sizes', 'class', 'style', 'loading', 'decoding'): |
| 291 |
img.attrs.pop(attr, None) |
| 292 |
img['src'] = self._set_img_width(img.get('src', '')) |
| 293 |
picture.replace_with(img) |
| 294 |
else: |
| 295 |
picture.decompose() |
| 296 |
|
| 297 |
for a in soup.find_all('a', href=True): |
| 298 |
if 'magazineshop' in a['href']: |
| 299 |
parent = a.find_parent() |
| 300 |
if parent and parent.name in ('p', 'div', 'span'): |
| 301 |
parent.decompose() |
| 302 |
else: |
| 303 |
a.decompose() |
| 304 |
|
| 305 |
for figure in soup.find_all('figure'): |
| 306 |
figcaption = figure.find('figcaption') |
| 307 |
caption_text = figcaption.get_text(strip=True) if figcaption else '' |
| 308 |
img = figure.find('img') |
| 309 |
if img: |
| 310 |
img_src = self._set_img_width(img.get('src', '')) |
| 311 |
img_tag = soup.new_tag('img', src=img_src, alt=caption_text) |
| 312 |
new_fig = soup.new_tag('figure') |
| 313 |
new_fig.append(img_tag) |
| 314 |
if caption_text: |
| 315 |
new_cap = soup.new_tag('figcaption', style='font-size:0.8em; color:#555; font-style:italic; text-align:center; margin-top:0.3em;') |
| 316 |
new_cap.string = caption_text |
| 317 |
new_fig.append(new_cap) |
| 318 |
figure.replace_with(new_fig) |
| 319 |
else: |
| 320 |
figure.decompose() |
| 321 |
|
| 322 |
for tag in soup.find_all(['img']): |
| 323 |
for attr in ('style', 'class'): |
| 324 |
if tag.has_attr(attr): |
| 325 |
del tag[attr] |
| 326 |
tag['src'] = self._set_img_width(tag.get('src', '')) |
| 327 |
|
| 328 |
for div in soup.find_all('div', class_=lambda c: c and 'justify-items-center' in c): |
| 329 |
div.unwrap() |
| 330 |
for div in soup.find_all('div', class_=lambda c: c and 'container' in c and 'mx-auto' in c): |
| 331 |
div.unwrap() |
| 332 |
for div in soup.find_all('div', class_=lambda c: c and 'grid' in c): |
| 333 |
div.unwrap() |
| 334 |
|
| 335 |
for ul in soup.find_all('ul'): |
| 336 |
tag_links = [a for a in ul.find_all('a', href=True) if '/section/' in a['href'] or '/tag/' in a['href']] |
| 337 |
if tag_links: |
| 338 |
p = soup.new_tag('p') |
| 339 |
for i, a in enumerate(tag_links): |
| 340 |
if i > 0: |
| 341 |
p.append(' | ') |
| 342 |
p.append(a) |
| 343 |
ul.replace_with(p) |
| 344 |
|
| 345 |
for p in soup.find_all('p'): |
| 346 |
if p.get_text(strip=True) == 'Advertisement': |
| 347 |
parent = p.find_parent(attrs={'data-ad-wrapper': True}) or p.find_parent(class_=lambda c: c and 'bg-warm-grey-ads' in c) |
| 348 |
if parent: |
| 349 |
parent.decompose() |
| 350 |
else: |
| 351 |
p.decompose() |
| 352 |
|
| 353 |
return soup |