Debian Codegraph @

recipes/nytimesbook.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
4
5
6 import json
7 from pprint import pprint
8
9 from calibre.ebooks.BeautifulSoup import BeautifulSoup
10 from calibre.web.feeds.news import BasicNewsRecipe
11 from polyglot.functools import lru_cache
12
13 use_wayback_machine = False
14
15
16 def absolutize(url):
17 if url.startswith('/'):
18 url = 'https://www.nytimes.com' + url
19 return url
20
21
22 @lru_cache(2)
23 def parser_module():
24 from calibre.live import load_module
25
26 return load_module('calibre.web.site_parsers.nytimes')
27
28
29 class NewYorkTimesBookReview(BasicNewsRecipe):
30 title = 'New York Times Book Review'
31 language = 'en_US'
32 description = 'The New York Times Sunday Book Review'
33 __author__ = 'Kovid Goyal'
34
35 no_stylesheets = True
36 no_javascript = True
37 ignore_duplicate_articles = {'title', 'url'}
38 encoding = 'utf-8'
39
40 extra_css = '''
41 .byl, .time { font-size:small; color:#202020; }
42 .cap { font-size:small; text-align:center; }
43 .cred { font-style:italic; font-size:small; }
44 em, blockquote { color: #202020; }
45 .sc { font-variant: small-caps; }
46 .lbl { font-size:small; color:#404040; }
47 img { display:block; margin:0 auto; }
48 '''
49
50 articles_are_obfuscated = use_wayback_machine
51
52 if use_wayback_machine:
53
54 def get_obfuscated_article(self, url):
55 from calibre.ptempfile import PersistentTemporaryFile
56
57 with PersistentTemporaryFile() as tf:
58 tf.write(self.get_nyt_page(url))
59 return tf.name
60
61 @property
62 def nyt_parser(self):
63 return parser_module()
64
65 def get_nyt_page(self, url, skip_wayback=False):
66 if use_wayback_machine and not skip_wayback:
67 from calibre import browser
68
69 return self.nyt_parser.download_url(url, browser())
70 return self.index_to_soup(url, raw=True)
71
72 def preprocess_raw_html(self, raw_html, url):
73 return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url)
74
75 recipe_specific_options = {
76 'res': {
77 'short': ('For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo'),
78 'long': (
79 'This is useful for non e-ink devices, and for a lower file size\nthan '
80 'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.'
81 ),
82 },
83 'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'},
84 }
85
86 def __init__(self, *args, **kwargs):
87 BasicNewsRecipe.__init__(self, *args, **kwargs)
88 c = self.recipe_specific_options.get('comp')
89 if c and isinstance(c, str):
90 if c.lower() == 'yes':
91 self.compress_news_images = True
92
93 def parse_index(self):
94 # return [('Articles', [{'url': 'https://www.nytimes.com/2022/09/08/books/review/karen-armstrong-by-the-book-interview.html', 'title':'test'}])]
95 soup = self.index_to_soup('https://www.nytimes.com/pages/books/review/index.html')
96 # with open('/t/raw.html', 'w') as f: f.write(str(soup))
97 feeds = parse_toc(soup)
98 for section_title, articles in feeds:
99 self.log(section_title)
100 for a in articles:
101 self.log('\t' + a['title'], a['url'])
102 return feeds
103
104 def get_browser(self, *args, **kwargs):
105 kwargs['user_agent'] = 'User-Agent: Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)'
106 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
107 return br
108
109 def preprocess_html(self, soup):
110 w = self.recipe_specific_options.get('res')
111 if w and isinstance(w, str):
112 res = '-' + w
113 for img in soup.findAll('img', attrs={'src': True}):
114 if '-article' in img['src']:
115 ext = img['src'].split('?')[0].split('.')[-1]
116 img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext
117 for c in soup.findAll('div', attrs={'class': 'cap'}):
118 for p in c.findAll(['p', 'div']):
119 p.name = 'span'
120 return soup
121
122
123 def asset_to_article(asset):
124 title = asset['headline']['default']
125 return {'title': title, 'url': asset['url'], 'description': asset['summary']}
126
127
128 def preloaded_data(soup):
129 candidates = soup.find_all('script', string=lambda x: x and 'window.__preloadedData' in x)
130 script = candidates[0]
131 script = str(script)
132 raw = script[script.find('{') : script.rfind(';')].strip().rstrip(';') # }
133 raw = parser_module().clean_js_json(raw)
134 return json.JSONDecoder(strict=False).raw_decode(raw)[0]['initialState']
135
136
137 def parse_toc(soup):
138 data = preloaded_data(soup)
139 # with open('/t/raw.json', 'w') as f: pprint(data, stream=f)
140 article_map = {}
141 for k, v in data.items():
142 if v['__typename'] == 'Article':
143 article_map[k] = asset_to_article(v)
144 feeds = []
145 for k, v in data['ROOT_QUERY'].items():
146 if k.startswith('workOrLocation'):
147 for g in data[v['__ref']]['groupings']:
148 for c in g['containers']:
149 articles = []
150 for r in c['relations']:
151 ref = r['asset']['__ref']
152 if ref in article_map:
153 articles.append(article_map[ref])
154 if articles:
155 feeds.append(('Highlights', articles))
156
157 articles = []
158 for k, v in data['ROOT_QUERY'].items():
159 if k.startswith('workOrLocation'):
160 c = data[v['__ref']]
161 section_title = c['name']
162 for k, v in c['collectionsPage'].items():
163 if k.startswith('stream'):
164 for k, v in v.items():
165 if k.startswith('edges'):
166 for q in v:
167 r = q['node']['__ref']
168 if r.startswith('Article:'):
169 articles.append(article_map[r])
170 if not articles:
171 for c in c['collectionsPage']['embeddedCollections']:
172 for e in c['stream']['edges']:
173 for k, v in e.items():
174 if k.startswith('node'):
175 articles.append(article_map[v['__ref']])
176 feeds.append((section_title, articles))
177 return feeds
178
179
180 if __name__ == '__main__':
181 import sys
182
183 with open(sys.argv[-1]) as f:
184 html = f.read()
185 soup = BeautifulSoup(html)
186 feeds = parse_toc(soup)
187 pprint(feeds)