Debian Codegraph @

recipes/time_magazine.recipe

raw
1 import re
2 import time as time_mod
3 from datetime import datetime
4
5 from calibre.utils.date import utcnow
6 from calibre.web.feeds.news import BasicNewsRecipe
7
8
9 class TimeMagazineUSA(BasicNewsRecipe):
10 title = 'TIME Magazine'
11 __author__ = 'Monkfishare'
12 description = 'Weekly US magazine.'
13 language = 'en'
14 timefmt = ''
15 no_stylesheets = True
16 remove_javascript = True
17 remove_empty_feeds = True
18 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/b/b3/Time_Magazine_logo.svg/960px-Time_Magazine_logo.svg.png'
19
20 recipe_specific_options = {
21 'edition': {
22 'short': 'Edition URL',
23 'long': (
24 'For example, https://time.com/magazine/us/7362702/february-9th-2026-vol-207-no-3-u-s/\n'
25 'Browse past editions at https://time.com/vault/year/2026 (change the year as needed).\n'
26 'Leave the URL field blank to automatically download the latest US edition.'
27 ),
28 'default': '',
29 },
30 'res': {
31 'short': 'Image resolution (width in pixels)',
32 'long': ('Recommended for e-ink devices: 1200, 1600, or 2000 (default)\nUse 400 or 800 for smaller file sizes or non e-ink devices.'),
33 'default': '2000',
34 },
35 }
36
37 keep_only_tags = [
38 dict(name='section', attrs={'aria-labelledby': lambda v: v and (v == 'intro-section' or re.match(r'^[a-z][a-z0-9_]*$', v))}),
39 dict(name='article'),
40 ]
41
42 remove_tags = [
43 dict(
44 attrs={
45 'class': lambda c: (
46 c
47 and any(
48 x in c
49 for x in [
50 'bg-warm-grey-ads',
51 'native-ad',
52 'advertisement',
53 'sr-only',
54 'google-source',
55 'print:hidden',
56 'rightrail',
57 ]
58 )
59 )
60 }
61 ),
62 dict(attrs={'data-testid': 'authors-list'}),
63 dict(name='button'),
64 dict(name='svg'),
65 dict(attrs={'aria-hidden': 'true'}),
66 dict(attrs={'role': 'tooltip'}),
67 dict(attrs={'class': lambda c: c and 'items-center' in c and 'flex-row' in c}),
68 dict(attrs={'class': lambda c: c and 'relative' in c and 'inline-flex' in c and 'w-max' in c}),
69 ]
70
71 extra_css = '''
72 h1 { font-size: 1.6em; }
73 h2, h3 { font-size: 1.2em; }
74 time { color: #666; font-size: 0.85em; }
75 img { display: block; max-width: 100%; height: auto;
76 position: static !important; float: none !important; }
77 figure { display: block; position: static !important;
78 margin: 1em 0; overflow: visible; }
79 '''
80
81 def get_browser(self):
82 br = BasicNewsRecipe.get_browser(self)
83 br.addheaders += [
84 ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
85 ('Accept-Language', 'en-US,en;q=0.9'),
86 ('Sec-Fetch-Mode', 'navigate'),
87 ('Sec-Fetch-Dest', 'document'),
88 ('Sec-Fetch-Site', 'none'),
89 ('Upgrade-Insecure-Requests', '1'),
90 ]
91 return br
92
93 def _img_width(self):
94 w = self.recipe_specific_options.get('res')
95 if w and isinstance(w, str) and w.isdigit():
96 return w
97 return '2000'
98
99 def _set_img_width(self, src):
100 src = re.sub(r'[?&]width=\d+', '', src).rstrip('?&')
101 sep = '&' if '?' in src else '?'
102 return src + sep + 'width=' + self._img_width()
103
104 def _sanitize_nested_anchors(self, raw_html):
105 return re.sub(
106 r'<a\b([^>]*\bhref="/section/[^"]*"[^>]*)>(.*?)</a>',
107 r'<span\1>\2</span>',
108 raw_html,
109 flags=re.DOTALL,
110 )
111
112 def _find_latest_us_edition(self, soup):
113 candidates = []
114 for art in soup.find_all('article'):
115 a = art.find('a', href=True)
116 if not a:
117 continue
118 href = a['href']
119 if '/magazine/us/' not in href:
120 continue
121 time_tag = art.find('time', attrs={'datetime': True})
122 if not time_tag:
123 continue
124 try:
125 dt = datetime.fromisoformat(time_tag['datetime'])
126 candidates.append((dt, href, a.get_text(strip=True)))
127 except Exception:
128 pass
129 if not candidates:
130 return None, None
131 candidates.sort(key=lambda x: x[0], reverse=True)
132 return candidates[0][1], candidates[0][2]
133
134 def _parse_toc_from_soup(self, soup):
135 toc_container = soup.find('div', class_=lambda c: c and 'columns-1' in c)
136 if not toc_container:
137 return None
138
139 sections = []
140 for section_div in toc_container.find_all('div', class_=lambda c: c and 'break-inside-avoid' in c):
141 h3 = section_div.find('h3', class_=lambda c: c and 'text-4xl' in c)
142 section_name = h3.get_text(strip=True) if h3 else None
143 if not section_name:
144 continue
145
146 articles = []
147 for title_h3 in section_div.find_all('h3', class_=lambda c: c and 'font-editorial' in c):
148 title = title_h3.get_text(strip=True)
149 a = title_h3.find_parent('a', href=True)
150 if not a:
151 continue
152 href = a['href']
153 if not re.match(r'^(/\d{7,}/|/article/|/collection/)', href):
154 continue
155 if title:
156 p = a.find('p', class_=lambda c: c and 'text-t4' in c)
157 desc = p.get_text(strip=True) if p else ''
158 self.log(' ', title)
159 articles.append({'title': title, 'url': 'https://time.com' + href, 'description': desc})
160
161 if articles:
162 self.log(section_name)
163 sections.append((section_name, articles))
164
165 return sections or None
166
167 def _slug_to_title(self, path):
168 parts = path.strip('/').split('/')
169 slug = parts[1] if len(parts) >= 2 else parts[0]
170 return slug.replace('-', ' ').title()
171
172 def _parse_toc_from_raw(self, raw_html):
173 text = raw_html.replace('\\/', '/')
174 text = re.sub(r'\\u002[Ff]', '/', text)
175
176 combined = re.compile(
177 r'(?:'
178 r'/\d{7,}/[a-z0-9][a-z0-9\-]*/'
179 r'|/article/\d{4}/\d{2}/\d{2}/[a-z0-9][a-z0-9\-]*/'
180 r'|/collection/[a-z0-9\-]+/\d{4}/[a-z0-9\-]*/'
181 r')'
182 )
183
184 seen = set()
185 paths = []
186 for m in combined.finditer(text):
187 path = m.group(0)
188 if re.search(r'/magazine/|/section/|/vault/', path):
189 continue
190 if path not in seen:
191 seen.add(path)
192 paths.append(path)
193
194 if not paths:
195 return None
196
197 articles = []
198 for path in paths:
199 url = 'https://time.com' + path
200 try:
201 art_soup = self.index_to_soup(url)
202 h1 = art_soup.find('h1')
203 title = h1.get_text(strip=True) if h1 else self._slug_to_title(path)
204 except Exception:
205 title = self._slug_to_title(path)
206 self.log('\t', title)
207 articles.append({'title': title, 'url': url})
208 time_mod.sleep(0.5)
209
210 return [('Articles', articles)]
211
212 def _fetch_raw(self, url):
213 resp = self.browser.open(url)
214 return resp.read().decode('utf-8', errors='replace')
215
216 def _title_from_path(self, path):
217 slug = path.strip('/').split('/')[-1]
218 m = re.match(r'([a-z]+)-(\d+)[a-z]*-(\d{4})', slug)
219 if m:
220 month, day, year = m.group(1).capitalize(), m.group(2), m.group(3)
221 return '%s %s, %s' % (month, day, year)
222 return slug.replace('-', ' ').title()
223
224 def parse_index(self):
225 opts = self.recipe_specific_options
226 custom_path = (opts.get('edition') or '').strip()
227
228 if custom_path:
229 edition_url = custom_path if custom_path.startswith('http') else 'https://time.com' + custom_path
230 date_str = self._title_from_path(custom_path)
231 self.log('Using custom edition:', edition_url)
232 else:
233 year = utcnow().year
234 archive_soup = self.index_to_soup('https://time.com/vault/year/%d/' % year)
235 edition_url, edition_title = self._find_latest_us_edition(archive_soup)
236 if not edition_url:
237 archive_soup = self.index_to_soup('https://time.com/vault/year/%d/' % (year - 1))
238 edition_url, edition_title = self._find_latest_us_edition(archive_soup)
239 if not edition_url:
240 raise ValueError('Could not find latest US edition')
241 self.log('Found edition:', edition_url)
242 date_str = edition_title.split('|')[0].strip() if edition_title else 'USA'
243
244 self.title = 'TIME Magazine'
245 self.timefmt = ' [' + date_str + ']'
246
247 raw_html = self._fetch_raw(edition_url)
248 raw_html = self._sanitize_nested_anchors(raw_html)
249
250 from calibre.ebooks.BeautifulSoup import BeautifulSoup
251
252 soup = BeautifulSoup(raw_html, 'html.parser')
253
254 for span in soup.find_all('span', class_=lambda c: c and 'text-t3' in c):
255 if span.get_text(strip=True).lower() == 'issue name':
256 p = span.find_next_sibling('p')
257 if p:
258 full_issue_name = p.get_text(strip=True)
259 date_str = full_issue_name.split('|')[0].strip()
260 if date_str:
261 self.timefmt = ' [' + date_str + ']'
262 self.log('Issue name:', full_issue_name)
263 break
264
265 figure = soup.find('figure')
266 if figure:
267 img = figure.find('img')
268 if img:
269 src = img.get('src', '')
270 self.cover_url = self._set_img_width(src)
271 self.log('Cover:', self.cover_url)
272
273 sections = self._parse_toc_from_soup(soup)
274 if sections:
275 self.log('Parsed TOC from HTML structure (%d sections)' % len(sections))
276 return sections
277
278 self.log('TOC not in HTML structure, trying raw payload extraction')
279 sections = self._parse_toc_from_raw(raw_html)
280 if sections:
281 self.log('Extracted %d article URLs from raw payload' % len(sections[0][1]))
282 return sections
283
284 raise ValueError('Could not parse edition TOC from either HTML or raw payload. Response length: %d bytes' % len(raw_html))
285
286 def preprocess_html(self, soup):
287 for picture in soup.find_all('picture'):
288 img = picture.find('img')
289 if img:
290 for attr in ('srcset', 'sizes', 'class', 'style', 'loading', 'decoding'):
291 img.attrs.pop(attr, None)
292 img['src'] = self._set_img_width(img.get('src', ''))
293 picture.replace_with(img)
294 else:
295 picture.decompose()
296
297 for a in soup.find_all('a', href=True):
298 if 'magazineshop' in a['href']:
299 parent = a.find_parent()
300 if parent and parent.name in ('p', 'div', 'span'):
301 parent.decompose()
302 else:
303 a.decompose()
304
305 for figure in soup.find_all('figure'):
306 figcaption = figure.find('figcaption')
307 caption_text = figcaption.get_text(strip=True) if figcaption else ''
308 img = figure.find('img')
309 if img:
310 img_src = self._set_img_width(img.get('src', ''))
311 img_tag = soup.new_tag('img', src=img_src, alt=caption_text)
312 new_fig = soup.new_tag('figure')
313 new_fig.append(img_tag)
314 if caption_text:
315 new_cap = soup.new_tag('figcaption', style='font-size:0.8em; color:#555; font-style:italic; text-align:center; margin-top:0.3em;')
316 new_cap.string = caption_text
317 new_fig.append(new_cap)
318 figure.replace_with(new_fig)
319 else:
320 figure.decompose()
321
322 for tag in soup.find_all(['img']):
323 for attr in ('style', 'class'):
324 if tag.has_attr(attr):
325 del tag[attr]
326 tag['src'] = self._set_img_width(tag.get('src', ''))
327
328 for div in soup.find_all('div', class_=lambda c: c and 'justify-items-center' in c):
329 div.unwrap()
330 for div in soup.find_all('div', class_=lambda c: c and 'container' in c and 'mx-auto' in c):
331 div.unwrap()
332 for div in soup.find_all('div', class_=lambda c: c and 'grid' in c):
333 div.unwrap()
334
335 for ul in soup.find_all('ul'):
336 tag_links = [a for a in ul.find_all('a', href=True) if '/section/' in a['href'] or '/tag/' in a['href']]
337 if tag_links:
338 p = soup.new_tag('p')
339 for i, a in enumerate(tag_links):
340 if i > 0:
341 p.append(' | ')
342 p.append(a)
343 ul.replace_with(p)
344
345 for p in soup.find_all('p'):
346 if p.get_text(strip=True) == 'Advertisement':
347 parent = p.find_parent(attrs={'data-ad-wrapper': True}) or p.find_parent(class_=lambda c: c and 'bg-warm-grey-ads' in c)
348 if parent:
349 parent.decompose()
350 else:
351 p.decompose()
352
353 return soup