| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
# License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net> |
| 4 |
|
| 5 |
|
| 6 |
import json |
| 7 |
from pprint import pprint |
| 8 |
|
| 9 |
from calibre.ebooks.BeautifulSoup import BeautifulSoup |
| 10 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 11 |
from polyglot.functools import lru_cache |
| 12 |
|
| 13 |
use_wayback_machine = False |
| 14 |
|
| 15 |
|
| 16 |
def absolutize(url): |
| 17 |
if url.startswith('/'): |
| 18 |
url = 'https://www.nytimes.com' + url |
| 19 |
return url |
| 20 |
|
| 21 |
|
| 22 |
@lru_cache(2) |
| 23 |
def parser_module(): |
| 24 |
from calibre.live import load_module |
| 25 |
|
| 26 |
return load_module('calibre.web.site_parsers.nytimes') |
| 27 |
|
| 28 |
|
| 29 |
class NewYorkTimesBookReview(BasicNewsRecipe): |
| 30 |
title = 'New York Times Book Review' |
| 31 |
language = 'en_US' |
| 32 |
description = 'The New York Times Sunday Book Review' |
| 33 |
__author__ = 'Kovid Goyal' |
| 34 |
|
| 35 |
no_stylesheets = True |
| 36 |
no_javascript = True |
| 37 |
ignore_duplicate_articles = {'title', 'url'} |
| 38 |
encoding = 'utf-8' |
| 39 |
|
| 40 |
extra_css = ''' |
| 41 |
.byl, .time { font-size:small; color:#202020; } |
| 42 |
.cap { font-size:small; text-align:center; } |
| 43 |
.cred { font-style:italic; font-size:small; } |
| 44 |
em, blockquote { color: #202020; } |
| 45 |
.sc { font-variant: small-caps; } |
| 46 |
.lbl { font-size:small; color:#404040; } |
| 47 |
img { display:block; margin:0 auto; } |
| 48 |
''' |
| 49 |
|
| 50 |
articles_are_obfuscated = use_wayback_machine |
| 51 |
|
| 52 |
if use_wayback_machine: |
| 53 |
|
| 54 |
def get_obfuscated_article(self, url): |
| 55 |
from calibre.ptempfile import PersistentTemporaryFile |
| 56 |
|
| 57 |
with PersistentTemporaryFile() as tf: |
| 58 |
tf.write(self.get_nyt_page(url)) |
| 59 |
return tf.name |
| 60 |
|
| 61 |
@property |
| 62 |
def nyt_parser(self): |
| 63 |
return parser_module() |
| 64 |
|
| 65 |
def get_nyt_page(self, url, skip_wayback=False): |
| 66 |
if use_wayback_machine and not skip_wayback: |
| 67 |
from calibre import browser |
| 68 |
|
| 69 |
return self.nyt_parser.download_url(url, browser()) |
| 70 |
return self.index_to_soup(url, raw=True) |
| 71 |
|
| 72 |
def preprocess_raw_html(self, raw_html, url): |
| 73 |
return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url) |
| 74 |
|
| 75 |
recipe_specific_options = { |
| 76 |
'res': { |
| 77 |
'short': ('For hi-res images, select a resolution from the following\noptions: popup, jumbo, mobileMasterAt3x, superJumbo'), |
| 78 |
'long': ( |
| 79 |
'This is useful for non e-ink devices, and for a lower file size\nthan ' |
| 80 |
'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.' |
| 81 |
), |
| 82 |
}, |
| 83 |
'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'}, |
| 84 |
} |
| 85 |
|
| 86 |
def __init__(self, *args, **kwargs): |
| 87 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 88 |
c = self.recipe_specific_options.get('comp') |
| 89 |
if c and isinstance(c, str): |
| 90 |
if c.lower() == 'yes': |
| 91 |
self.compress_news_images = True |
| 92 |
|
| 93 |
def parse_index(self): |
| 94 |
# return [('Articles', [{'url': 'https://www.nytimes.com/2022/09/08/books/review/karen-armstrong-by-the-book-interview.html', 'title':'test'}])] |
| 95 |
soup = self.index_to_soup('https://www.nytimes.com/pages/books/review/index.html') |
| 96 |
# with open('/t/raw.html', 'w') as f: f.write(str(soup)) |
| 97 |
feeds = parse_toc(soup) |
| 98 |
for section_title, articles in feeds: |
| 99 |
self.log(section_title) |
| 100 |
for a in articles: |
| 101 |
self.log('\t' + a['title'], a['url']) |
| 102 |
return feeds |
| 103 |
|
| 104 |
def get_browser(self, *args, **kwargs): |
| 105 |
kwargs['user_agent'] = 'User-Agent: Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)' |
| 106 |
br = BasicNewsRecipe.get_browser(self, *args, **kwargs) |
| 107 |
return br |
| 108 |
|
| 109 |
def preprocess_html(self, soup): |
| 110 |
w = self.recipe_specific_options.get('res') |
| 111 |
if w and isinstance(w, str): |
| 112 |
res = '-' + w |
| 113 |
for img in soup.findAll('img', attrs={'src': True}): |
| 114 |
if '-article' in img['src']: |
| 115 |
ext = img['src'].split('?')[0].split('.')[-1] |
| 116 |
img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext |
| 117 |
for c in soup.findAll('div', attrs={'class': 'cap'}): |
| 118 |
for p in c.findAll(['p', 'div']): |
| 119 |
p.name = 'span' |
| 120 |
return soup |
| 121 |
|
| 122 |
|
| 123 |
def asset_to_article(asset): |
| 124 |
title = asset['headline']['default'] |
| 125 |
return {'title': title, 'url': asset['url'], 'description': asset['summary']} |
| 126 |
|
| 127 |
|
| 128 |
def preloaded_data(soup): |
| 129 |
candidates = soup.find_all('script', string=lambda x: x and 'window.__preloadedData' in x) |
| 130 |
script = candidates[0] |
| 131 |
script = str(script) |
| 132 |
raw = script[script.find('{') : script.rfind(';')].strip().rstrip(';') # } |
| 133 |
raw = parser_module().clean_js_json(raw) |
| 134 |
return json.JSONDecoder(strict=False).raw_decode(raw)[0]['initialState'] |
| 135 |
|
| 136 |
|
| 137 |
def parse_toc(soup): |
| 138 |
data = preloaded_data(soup) |
| 139 |
# with open('/t/raw.json', 'w') as f: pprint(data, stream=f) |
| 140 |
article_map = {} |
| 141 |
for k, v in data.items(): |
| 142 |
if v['__typename'] == 'Article': |
| 143 |
article_map[k] = asset_to_article(v) |
| 144 |
feeds = [] |
| 145 |
for k, v in data['ROOT_QUERY'].items(): |
| 146 |
if k.startswith('workOrLocation'): |
| 147 |
for g in data[v['__ref']]['groupings']: |
| 148 |
for c in g['containers']: |
| 149 |
articles = [] |
| 150 |
for r in c['relations']: |
| 151 |
ref = r['asset']['__ref'] |
| 152 |
if ref in article_map: |
| 153 |
articles.append(article_map[ref]) |
| 154 |
if articles: |
| 155 |
feeds.append(('Highlights', articles)) |
| 156 |
|
| 157 |
articles = [] |
| 158 |
for k, v in data['ROOT_QUERY'].items(): |
| 159 |
if k.startswith('workOrLocation'): |
| 160 |
c = data[v['__ref']] |
| 161 |
section_title = c['name'] |
| 162 |
for k, v in c['collectionsPage'].items(): |
| 163 |
if k.startswith('stream'): |
| 164 |
for k, v in v.items(): |
| 165 |
if k.startswith('edges'): |
| 166 |
for q in v: |
| 167 |
r = q['node']['__ref'] |
| 168 |
if r.startswith('Article:'): |
| 169 |
articles.append(article_map[r]) |
| 170 |
if not articles: |
| 171 |
for c in c['collectionsPage']['embeddedCollections']: |
| 172 |
for e in c['stream']['edges']: |
| 173 |
for k, v in e.items(): |
| 174 |
if k.startswith('node'): |
| 175 |
articles.append(article_map[v['__ref']]) |
| 176 |
feeds.append((section_title, articles)) |
| 177 |
return feeds |
| 178 |
|
| 179 |
|
| 180 |
if __name__ == '__main__': |
| 181 |
import sys |
| 182 |
|
| 183 |
with open(sys.argv[-1]) as f: |
| 184 |
html = f.read() |
| 185 |
soup = BeautifulSoup(html) |
| 186 |
feeds = parse_toc(soup) |
| 187 |
pprint(feeds) |