Debian Codegraph @

recipes/nytimes_sub.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2018, Kovid Goyal <kovid at kovidgoyal.net>
4
5 import datetime
6 import json
7 import re
8 from pprint import pprint
9
10 from calibre import strftime
11 from calibre.ebooks.BeautifulSoup import BeautifulSoup, Tag
12 from calibre.utils.date import strptime
13 from calibre.web.feeds.news import BasicNewsRecipe
14 from polyglot.functools import lru_cache
15
16 is_web_edition = False
17 use_wayback_machine = False
18
19 # This is an Apollo persisted query hash which you can get
20 # from looking at the XHR requests made by: https://www.nytimes.com/section/todayspaper
21 # or by https://www.nytimes.com/section/world
22 persistedQuery = '1f99120a11e94dd62a9474f68ee1255537ee3cf7eac20a0377819edb2fa1fef7'
23
24 # The sections to download when downloading the web edition, comment out
25 # the section you are not interested in
26 web_sections = [
27 'world',
28 'us',
29 'politics',
30 'nyregion',
31 'business',
32 'technology',
33 'sports',
34 'science',
35 'health',
36 'opinion',
37 'arts',
38 'books',
39 'movies',
40 'arts/music',
41 'arts/television',
42 'style',
43 'food',
44 'fashion',
45 'travel',
46 'education',
47 'multimedia',
48 'obituaries',
49 'magazine',
50 ]
51 # web_sections = [ 'business' ]
52 url_date_pat = re.compile(r'/(2\d\d\d)/(\d\d)/(\d\d)/')
53
54
55 def date_from_url(url):
56 m = url_date_pat.search(url)
57 if m is not None:
58 return datetime.date(*map(int, m.groups()))
59
60
61 def format_date(d):
62 try:
63 return strftime(' [%a, %d %b %Y]', d)
64 except Exception:
65 return strftime(' [%Y/%m/%d]', d)
66
67
68 def classes(classes):
69 q = frozenset(classes.split(' '))
70 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
71
72
73 def new_tag(soup, name, attrs=()):
74 impl = getattr(soup, 'new_tag', None)
75 if impl is not None:
76 return impl(name, attrs=dict(attrs))
77 return Tag(soup, name, attrs=attrs or None)
78
79
80 def absolutize_href(href):
81 if not href.startswith('http'):
82 href = 'https://www.nytimes.com/' + href.lstrip('/')
83 return href
84
85
86 @lru_cache(2)
87 def parser_module():
88 from calibre.live import load_module
89
90 return load_module('calibre.web.site_parsers.nytimes')
91
92
93 class NewYorkTimes(BasicNewsRecipe):
94 if is_web_edition:
95 title = 'The New York Times (Web)'
96 description = (
97 'New York Times (Web). You can edit the recipe to remove sections you are not interested in. '
98 'Use advanced menu to make changes to fetch Todays Paper'
99 )
100 else:
101 title = 'The New York Times'
102 description = 'New York Times. Todays Paper Use advanced menu to make changes to fetch Web Edition'
103 encoding = 'utf-8'
104 __author__ = 'Kovid Goyal'
105 language = 'en_US'
106 ignore_duplicate_articles = {'title', 'url'}
107 no_stylesheets = True
108 oldest_web_edition_article = 7 # days
109 browser_type = 'webengine'
110
111 extra_css = '''
112 .byl, .time { font-size:small; color:#202020; }
113 .cap { font-size:small; text-align:center; }
114 .cred { font-style:italic; font-size:small; }
115 em, blockquote { color: #202020; }
116 .sc { font-variant: small-caps; }
117 .lbl { font-size:small; color:#404040; }
118 img { display:block; margin:0 auto; }
119 '''
120
121 @property
122 def nyt_parser(self):
123 return parser_module()
124
125 def get_nyt_page(self, url, skip_wayback=False):
126 if use_wayback_machine and not skip_wayback:
127 from calibre import browser
128
129 return self.nyt_parser.download_url(url, browser())
130 return self.index_to_soup(url, raw=True)
131
132 def preprocess_raw_html(self, raw_html, url):
133 return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
134
135 articles_are_obfuscated = use_wayback_machine
136
137 if use_wayback_machine:
138
139 def get_obfuscated_article(self, url):
140 from calibre.ptempfile import PersistentTemporaryFile
141
142 with PersistentTemporaryFile() as tf:
143 tf.write(self.get_nyt_page(url))
144 return tf.name
145
146 recipe_specific_options = {
147 'web': {
148 'short': 'Type in yes, if you want ' + ('Todays Paper' if is_web_edition else 'Web Edition'),
149 'default': 'Web Edition' if is_web_edition else 'Todays Paper',
150 },
151 'days': {
152 'short': 'Oldest article to download from this news source. In days ',
153 'long': 'For example, 1, gives you articles from the past 24 hours\n(Works only for Web_Edition)',
154 'default': str(oldest_web_edition_article),
155 },
156 'date': {
157 'short': 'The date of the edition to download (YYYY/MM/DD format)\nUsed to fetch past editions of NYT newspaper',
158 'long': 'For example, 2024/07/16',
159 },
160 'res': {
161 'short': ('For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo'),
162 'long': (
163 'This is useful for non e-ink devices, and for a lower file size\nthan '
164 'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
165 ),
166 },
167 'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'},
168 }
169
170 def __init__(self, *args, **kwargs):
171 BasicNewsRecipe.__init__(self, *args, **kwargs)
172 c = self.recipe_specific_options.get('comp')
173 d = self.recipe_specific_options.get('days')
174 w = self.recipe_specific_options.get('web')
175 self.is_web_edition = is_web_edition
176 if w and isinstance(w, str):
177 if w == 'yes':
178 self.is_web_edition = not is_web_edition
179 if d and isinstance(d, str):
180 self.oldest_web_edition_article = float(d)
181 if c and isinstance(c, str):
182 if c.lower() == 'yes':
183 self.compress_news_images = True
184
185 def todays_paper_url(self):
186 pdate = self.recipe_specific_options.get('date')
187 if pdate and isinstance(pdate, str):
188 return 'https://www.nytimes.com/issue/todayspaper/{}/todays-new-york-times'.format(pdate)
189 return 'https://www.nytimes.com/section/todayspaper'
190
191 def parse_todays_page(self):
192 url = self.todays_paper_url()
193 soup = self.index_to_soup(url)
194 return parse_todays_page(soup)
195
196 def parse_web_sections(self):
197 feeds = []
198 for slug in web_sections:
199 url = 'https://www.nytimes.com/section/' + slug
200 self.log('Download section index:', url)
201 soup = self.index_to_soup(url)
202 # with open('/t/raw.html', 'w') as f:
203 # f.write(str(soup))
204 section_title, articles = parse_web_section(soup)
205 self.log('Section:', section_title)
206 if articles:
207 feeds.append((section_title, articles))
208 for a in articles:
209 self.log('\t', a['title'], a['url'])
210 else:
211 self.log(' No articles found in section:', section_title)
212 if self.test and len(feeds) >= self.test[0]:
213 break
214 return feeds
215
216 def parse_index(self):
217 # return [('All articles', [
218 # {'title': 'XXXXX', 'url': 'https://www.nytimes.com/2020/11/27/world/americas/coronavirus-migrants-venezuela.html'},
219 # ])]
220 date, feeds = self.parse_todays_page()
221 pdate = date.strftime('%Y/%m/%d')
222 self.cover_url = 'https://static01.nyt.com/images/{}/nytfrontpage/scan.jpg'.format(pdate)
223 self.timefmt = strftime(' [%d %b, %Y]', date)
224 if self.is_web_edition:
225 return self.parse_web_sections()
226 for s, articles in feeds:
227 self.log('Section:', s)
228 for a in articles:
229 self.log('\t', a['title'], a['url'])
230 return feeds
231
232 def get_browser(self, *args, **kwargs):
233 # kwargs['user_agent'] = 'Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)'
234 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
235 return br
236
237 def preprocess_html(self, soup):
238 w = self.recipe_specific_options.get('res')
239 if w and isinstance(w, str):
240 res = '-' + w
241 for img in soup.findAll('img', attrs={'src': True}):
242 if '-article' in img['src']:
243 ext = img['src'].split('?')[0].split('.')[-1]
244 img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
245 for c in soup.findAll('div', attrs={'class': 'cap'}):
246 for p in c.findAll(['p', 'div']):
247 p.name = 'span'
248 return soup
249
250 def get_article_url(self, article):
251 url = BasicNewsRecipe.get_article_url(self, article)
252 if not re.search(r'/video/|/athletic/|/card/', url):
253 return url
254 self.log('\tSkipping ', url)
255
256
257 def preloaded_data(soup):
258 candidates = soup.find_all('script', string=lambda x: x and 'window.__preloadedData' in x)
259 script = candidates[0]
260 script = str(script)
261 raw = script[script.find('{') : script.rfind(';')].strip().rstrip(';') # }
262 raw = parser_module().clean_js_json(raw)
263 # with open('/t/raw.json', 'w') as f:
264 # f.write(raw)
265 return json.JSONDecoder(strict=False).raw_decode(raw)[0]['initialState']
266
267
268 def asset_to_article(asset):
269 title = asset['headline']['default']
270 return {'title': title, 'url': asset['url'], 'description': asset['summary']}
271
272
273 def parse_web_section(soup):
274 data = preloaded_data(soup)
275 article_map = {}
276 for k, v in data.items():
277 if v['__typename'] == 'Article':
278 article_map[k] = asset_to_article(v)
279 articles = []
280 for k, v in data['ROOT_QUERY'].items():
281 if k.startswith('workOrLocation'):
282 c = data[v['__ref']]
283 section_title = c['name']
284 for k, v in c['collectionsPage'].items():
285 if k.startswith('stream'):
286 for k, v in v.items():
287 if k.startswith('edges'):
288 for q in v:
289 r = q['node']['__ref']
290 if r.startswith('Article:'):
291 articles.append(article_map[r])
292 if not articles:
293 for c in c['collectionsPage']['embeddedCollections']:
294 for e in c['stream']['edges']:
295 for k, v in e.items():
296 if k.startswith('node'):
297 articles.append(article_map[v['__ref']])
298 return section_title, articles
299
300
301 def parse_todays_page(soup):
302 m = soup.find('meta', attrs={'name': 'nyt-collection:uri'})['content'].split('/')
303 pdate = strptime('{}/{}/{}'.format(*m[-4:-1]), '%Y/%m/%d', assume_utc=False, as_utc=False)
304 article_map = {}
305 data = preloaded_data(soup)
306 for k, v in data.items():
307 if v['__typename'] == 'Article':
308 article_map[k] = asset_to_article(v)
309 feeds = []
310 for k, v in data['ROOT_QUERY'].items():
311 if k.startswith('workOrLocation'):
312 for g in data[v['__ref']]['groupings']:
313 for c in g['containers']:
314 articles = []
315 for r in c['relations']:
316 ref = r['asset']['__ref']
317 if ref in article_map:
318 articles.append(article_map[ref])
319 if articles:
320 feeds.append((c['label'], articles))
321 return pdate, feeds
322
323
324 if __name__ == '__main__':
325 import sys
326
327 with open(sys.argv[-1]) as f:
328 html = f.read()
329 soup = BeautifulSoup(html)
330 if is_web_edition:
331 section_title, articles = parse_web_section(soup)
332 print(section_title)
333 pprint(articles)
334 else:
335 pdate, feeds = parse_todays_page(soup)
336 print(pdate)
337 pprint(feeds)