| 1 |
#!/usr/bin/env python |
| 2 |
import json |
| 3 |
import time |
| 4 |
from datetime import datetime, timedelta |
| 5 |
from urllib.parse import quote, urlencode |
| 6 |
from uuid import uuid4 |
| 7 |
|
| 8 |
from html5_parser import parse |
| 9 |
from lxml import etree |
| 10 |
from mechanize import Request |
| 11 |
|
| 12 |
from calibre import browser |
| 13 |
from calibre.ptempfile import PersistentTemporaryFile |
| 14 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 15 |
|
| 16 |
|
| 17 |
def safe_dict(data, *names): |
| 18 |
ans = data |
| 19 |
for x in names: |
| 20 |
ans = ans.get(x) or {} |
| 21 |
return ans |
| 22 |
|
| 23 |
|
| 24 |
def process_web_list(li_node): |
| 25 |
li_html = '' |
| 26 |
for li in li_node['items']: |
| 27 |
if li.get('textHtml'): |
| 28 |
li_html += f'<li>{li["textHtml"]}</li>' |
| 29 |
elif li.get('textJson'): |
| 30 |
li_html += f'<li>{parse_textjson(li["textJson"])}</li>' |
| 31 |
else: |
| 32 |
li_html += f'<li>{li.get("text", "")}</li>' |
| 33 |
return '<ul>' + li_html + '</ul>' |
| 34 |
|
| 35 |
|
| 36 |
def process_info_box(bx): |
| 37 |
info = '' |
| 38 |
for x in safe_dict(bx, 'components'): |
| 39 |
info += f'<blockquote>{process_web_node(x)}</blockquote>' |
| 40 |
return info |
| 41 |
|
| 42 |
|
| 43 |
def parse_txt(ty): |
| 44 |
typ = ty.get('type', '') |
| 45 |
children = ty.get('children', []) |
| 46 |
href = '#' |
| 47 |
attributes = ty.get('attributes') or () |
| 48 |
for a in attributes: |
| 49 |
if a.get('name') == 'href': |
| 50 |
href = a.get('value', href) |
| 51 |
break |
| 52 |
|
| 53 |
tag_map = { |
| 54 |
'text': lambda: [ty.get('value', '')], |
| 55 |
'scaps': lambda: [ |
| 56 |
f'<span style="text-transform: uppercase; font-size: 0.85em; letter-spacing: 0.05em;">{"".join(parse_txt(c))}</span>' for c in children |
| 57 |
], |
| 58 |
'bold': lambda: [f'<b>{"".join(parse_txt(c))}</b>' for c in children], |
| 59 |
'drop_caps': lambda: [f'<b>{"".join(parse_txt(c))}</b>' for c in children], |
| 60 |
'italic': lambda: [f'<i>{"".join(parse_txt(c))}</i>' for c in children], |
| 61 |
'linebreak': lambda: ['<br>'], |
| 62 |
'external_link': lambda: [f'<a href="{href}">{"".join(parse_txt(c))}</a>' for c in children], |
| 63 |
'internal_link': lambda: [f'<a href="{href}">{"".join(parse_txt(c))}</a>' for c in children], |
| 64 |
'ufinish': lambda: [text for c in children for text in parse_txt(c)], |
| 65 |
'subscript': lambda: [f'<sub>{"".join(parse_txt(c))}</sub>' for c in children], |
| 66 |
'superscript': lambda: [f'<sup>{"".join(parse_txt(c))}</sup>' for c in children], |
| 67 |
} |
| 68 |
|
| 69 |
if typ in tag_map: |
| 70 |
yield from tag_map[typ]() |
| 71 |
else: |
| 72 |
print('** ', typ) |
| 73 |
|
| 74 |
|
| 75 |
def parse_textjson(nt): |
| 76 |
return ''.join(''.join(parse_txt(n)) for n in nt) |
| 77 |
|
| 78 |
|
| 79 |
def process_web_node(node): |
| 80 |
ntype = node.get('type', '') |
| 81 |
if ntype == 'CROSSHEAD': |
| 82 |
if node.get('textHtml'): |
| 83 |
return f'<h4>{node.get("textHtml")}</h4>' |
| 84 |
return f'<h4>{node.get("text", "")}</h4>' |
| 85 |
elif ntype in ['PARAGRAPH', 'BOOK_INFO']: |
| 86 |
if node.get('textHtml'): |
| 87 |
return f'\n<p>{node.get("textHtml")}</p>' |
| 88 |
if node.get('textJson'): |
| 89 |
return f'\n<p>{parse_textjson(node["textJson"])}</p>' |
| 90 |
return f'\n<p>{node.get("text", "")}</p>' |
| 91 |
elif (ntype == 'IMAGE') or (node.get('__typename', '') == 'ImageComponent'): |
| 92 |
alt = '' if node.get('altText') is None else node.get('altText') |
| 93 |
cap = '' |
| 94 |
if node.get('caption'): |
| 95 |
if node['caption'].get('textHtml') is not None: |
| 96 |
cap = node['caption']['textHtml'] |
| 97 |
elif node['caption'].get('textJson') is not None: |
| 98 |
cap = parse_textjson(node['caption']['textJson']) |
| 99 |
elif node['caption'].get('text') is not None: |
| 100 |
cap = node['caption']['text'] |
| 101 |
return f'<div><img src="{node["url"]}" title="{alt}"></div><div style="text-align:center; font-size:small;">{cap}</div>' |
| 102 |
elif ntype == 'PULL_QUOTE': |
| 103 |
if node.get('textHtml'): |
| 104 |
return f'<blockquote>{node.get("textHtml")}</blockquote>' |
| 105 |
if node.get('textJson'): |
| 106 |
return f'<blockquote>{parse_textjson(node["textJson"])}</blockquote>' |
| 107 |
return f'<blockquote>{node.get("text", "")}</blockquote>' |
| 108 |
elif ntype == 'BLOCK_QUOTE': |
| 109 |
if node.get('textHtml'): |
| 110 |
return f'<blockquote><i>{node.get("textHtml")}</i></blockquote>' |
| 111 |
if node.get('textJson'): |
| 112 |
return f'<blockquote><i>{parse_textjson(node["textJson"])}</i></blockquote>' |
| 113 |
return f'<blockquote><i>{node.get("text", "")}</i></blockquote>' |
| 114 |
elif ntype == 'DIVIDER': |
| 115 |
return '<hr>' |
| 116 |
elif ntype == 'INFOGRAPHIC': |
| 117 |
if node.get('fallback'): |
| 118 |
return process_web_node(node['fallback']) |
| 119 |
elif ntype == 'INFOBOX': |
| 120 |
return process_info_box(node) |
| 121 |
elif ntype == 'UNORDERED_LIST': |
| 122 |
if node.get('items'): |
| 123 |
return process_web_list(node) |
| 124 |
elif ntype: |
| 125 |
print('** ', ntype) |
| 126 |
return '' |
| 127 |
|
| 128 |
|
| 129 |
def load_article_from_web_json(raw): |
| 130 |
# open('/t/raw.json', 'w').write(raw) |
| 131 |
body = '' |
| 132 |
data = json.loads(raw)['data']['findArticleByUrl'] |
| 133 |
body += f'<div style="color: red; font-size:small; font-weight:bold;">{data.get("flyTitle", "")}</div>' |
| 134 |
body += f'<h1>{data["headline"]}</h1>' |
| 135 |
if data.get('rubric') and data.get('rubric') is not None: |
| 136 |
body += f'<div style="font-style: italic; color:#202020;">{data.get("rubric", "")}</div>' |
| 137 |
try: |
| 138 |
date = data['dateModified'] |
| 139 |
except Exception: |
| 140 |
date = data['datePublished'] |
| 141 |
dt = datetime.fromisoformat(date[:-1]) + timedelta(seconds=time.timezone) |
| 142 |
dt = dt.strftime('%b %d, %Y %I:%M %p') |
| 143 |
if data.get('dateline') is None: |
| 144 |
body += f'<p style="color: gray; font-size: small;">{dt}</p>' |
| 145 |
else: |
| 146 |
body += f'<p style="color: gray; font-size: small;">{dt + " | " + (data["dateline"])}</p>' |
| 147 |
main_image_url = safe_dict(data, 'leadComponent') or '' |
| 148 |
if main_image_url: |
| 149 |
body += process_web_node(data['leadComponent']) |
| 150 |
if data.get('byline'): |
| 151 |
if data['byline'] is not None: |
| 152 |
body += f'<p style="color: gray; font-size: small;"><i>{"By " + data["byline"]}</i></p>' |
| 153 |
for node in data.get('body'): |
| 154 |
body += process_web_node(node) |
| 155 |
return '<html><body><article>' + body + '</article></body></html>' |
| 156 |
|
| 157 |
|
| 158 |
class NoArticles(Exception): |
| 159 |
pass |
| 160 |
|
| 161 |
|
| 162 |
def get_content(url_): |
| 163 |
|
| 164 |
headers = { |
| 165 |
'User-Agent': 'TheEconomist-Liskov-android', |
| 166 |
'accept': 'multipart/mixed; deferSpec=20220824, application/json', |
| 167 |
'accept-encoding': 'gzip', |
| 168 |
'content-type': 'application/json', |
| 169 |
'x-app-trace-id': str(uuid4()), |
| 170 |
'x-economist-consumer': 'TheEconomist-Liskov-android', |
| 171 |
'x-teg-client-name': 'Economist-Android', |
| 172 |
'x-teg-client-os': 'Android', |
| 173 |
'x-teg-client-version': '4.40.0', |
| 174 |
} |
| 175 |
br = browser() |
| 176 |
req = Request( |
| 177 |
url_, |
| 178 |
headers=headers, |
| 179 |
) |
| 180 |
res = br.open(req) |
| 181 |
return res.read() |
| 182 |
|
| 183 |
|
| 184 |
def process_url(url): |
| 185 |
if url.startswith('/'): |
| 186 |
url = 'https://www.economist.com' + url |
| 187 |
return url |
| 188 |
|
| 189 |
|
| 190 |
class Econ1843(BasicNewsRecipe): |
| 191 |
title = 'The Economist 1843' |
| 192 |
language = 'en_GB' |
| 193 |
encoding = 'utf-8' |
| 194 |
masthead_url = 'https://www.livemint.com/lm-img/dev/economist-logo-oneline.png' |
| 195 |
|
| 196 |
__author__ = 'unkn0wn' |
| 197 |
description = ( |
| 198 |
'Published since September 1843 to take part in “a severe contest between intelligence, which presses forward, and ' |
| 199 |
'an unworthy, timid ignorance obstructing our progress.”' |
| 200 |
) |
| 201 |
cover_url = 'https://pbs.twimg.com/media/FQtV6T0WYAYDxKY?format=jpg&name=orig' |
| 202 |
extra_css = ''' |
| 203 |
em { color:#202020; } |
| 204 |
img {display:block; margin:0 auto;} |
| 205 |
''' |
| 206 |
resolve_internal_links = True |
| 207 |
|
| 208 |
# economist.com has started throttling after about 60% of the total has |
| 209 |
# downloaded with connection reset by peer (104) errors. |
| 210 |
delay = 1 |
| 211 |
browser_type = 'webengine' |
| 212 |
|
| 213 |
recipe_specific_options = { |
| 214 |
'res': { |
| 215 |
'short': 'For hi-res images, select a resolution from the\nfollowing options: 834, 960, 1096, 1280, 1424', |
| 216 |
'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use from 480, 384, 360, 256.', |
| 217 |
'default': '600', |
| 218 |
}, |
| 219 |
} |
| 220 |
|
| 221 |
def get_browser(self, *args, **kwargs): |
| 222 |
from calibre.utils.random_ua import common_english_word_ua |
| 223 |
|
| 224 |
kwargs['user_agent'] = common_english_word_ua() |
| 225 |
br = BasicNewsRecipe.get_browser(self, *args, **kwargs) |
| 226 |
return br |
| 227 |
|
| 228 |
def economist_test_article(self): |
| 229 |
return [('Articles', [{'title': 'test', 'url': 'https://www.economist.com/1843/2025/03/05/the-great-pretender-how-ahmed-al-sharaa-won-syria'}])] |
| 230 |
|
| 231 |
def economist_return_index(self, ans): |
| 232 |
if not ans: |
| 233 |
raise NoArticles( |
| 234 |
'Could not find any articles, either the ' |
| 235 |
'economist.com server is having trouble and you should ' |
| 236 |
'try later or the website format has changed and the ' |
| 237 |
'recipe needs to be updated.' |
| 238 |
) |
| 239 |
return ans |
| 240 |
|
| 241 |
def parse_index(self): |
| 242 |
# return self.economist_test_article() |
| 243 |
self.title = '1843' |
| 244 |
raw = self.index_to_soup('https://www.economist.com/1843') |
| 245 |
ans = self.economist_parse_index(raw) |
| 246 |
return self.economist_return_index(ans) |
| 247 |
|
| 248 |
def economist_parse_index(self, soup): |
| 249 |
script_tag = soup.find('script', id='__NEXT_DATA__') |
| 250 |
if script_tag is not None: |
| 251 |
data = json.loads(script_tag.string) |
| 252 |
# open('/t/raw.json', 'w').write(json.dumps(data, indent=2, sort_keys=True)) |
| 253 |
|
| 254 |
feeds = [] |
| 255 |
|
| 256 |
for part in safe_dict(data, 'props', 'pageProps', 'content', 'articles'): |
| 257 |
title = safe_dict(part, 'headline') or '' |
| 258 |
url = process_url(safe_dict(part, 'url') or '') |
| 259 |
desc = safe_dict(part, 'rubric') or '' |
| 260 |
sub = safe_dict(part, 'flyTitle') or '' |
| 261 |
if sub: |
| 262 |
desc = sub + ' :: ' + desc |
| 263 |
self.log('\t', title, '\n\t', desc, '\n\t\t', url) |
| 264 |
feeds.append({'title': title, 'description': desc, 'url': url}) |
| 265 |
return [('Articles', feeds)] |
| 266 |
|
| 267 |
def preprocess_html(self, soup): |
| 268 |
width = '600' |
| 269 |
w = self.recipe_specific_options.get('res') |
| 270 |
if w and isinstance(w, str): |
| 271 |
width = w |
| 272 |
for img in soup.findAll('img', src=True): |
| 273 |
qua = 'economist.com/cdn-cgi/image/width=' + width + ',quality=80,format=auto/' |
| 274 |
img['src'] = img['src'].replace('economist.com/', qua) |
| 275 |
return soup |
| 276 |
|
| 277 |
def preprocess_raw_html(self, raw, url): |
| 278 |
# open('/t/raw.html', 'wb').write(raw.encode('utf-8')) |
| 279 |
html = load_article_from_web_json(raw) |
| 280 |
root = parse(html) |
| 281 |
# the economist uses <small> for small caps with a custom font |
| 282 |
for init in root.xpath('//span[@data-caps="initial"]'): |
| 283 |
init.set('style', 'font-weight:bold;') |
| 284 |
for x in root.xpath('//small'): |
| 285 |
if x.text and len(x) == 0: |
| 286 |
x.text = x.text.upper() |
| 287 |
x.tag = 'span' |
| 288 |
x.set( |
| 289 |
'style', |
| 290 |
'text-transform: uppercase; font-size: 0.85em; letter-spacing: 0.05em;', |
| 291 |
) |
| 292 |
for h2 in root.xpath('//h2'): |
| 293 |
h2.tag = 'h4' |
| 294 |
for x in root.xpath('//figcaption'): |
| 295 |
x.set('style', 'text-align:center; font-size:small;') |
| 296 |
for x in root.xpath('//cite'): |
| 297 |
x.tag = 'blockquote' |
| 298 |
x.set('style', 'color:#404040;') |
| 299 |
raw = etree.tostring(root, encoding='unicode') |
| 300 |
return raw |
| 301 |
|
| 302 |
def get_article(self, url): |
| 303 |
query = { |
| 304 |
'operationName': 'ArticleDeeplinkQuery', |
| 305 |
'variables': '{{"ref":"{}"}}'.format(url), |
| 306 |
'query': 'query ArticleDeeplinkQuery($ref: String!, $includeRelatedArticles: Boolean = true ) { findArticleByUrl(url: $ref) { __typename ...ArticleDataFragment } } fragment ContentIdentityFragment on ContentIdentity { articleType forceAppWebView leadMediaType } fragment NarrationFragment on Narration { album bitrate duration filename id provider url isAiGenerated fileHash } fragment ImageTeaserFragment on ImageComponent { altText height imageType source url width } fragment PodcastAudioFragment on PodcastEpisode { id audio { url durationInSeconds } } fragment ArticleTeaserFragment on Article { id tegId url rubric headline flyTitle brand byline dateFirstPublished dateline dateModified datePublished dateRevised estimatedReadTime wordCount printHeadline contentIdentity { __typename ...ContentIdentityFragment } section { tegId name } teaserImage { __typename type ...ImageTeaserFragment } leadComponent { __typename type ...ImageTeaserFragment } narration(selectionMethod: PREFER_ACTOR_NARRATION) { __typename ...NarrationFragment } podcast { __typename ...PodcastAudioFragment } } fragment AnnotatedTextFragment on AnnotatedText { text textJson annotations { type length index attributes { name value } } } fragment ImageComponentFragment on ImageComponent { altText caption { __typename ...AnnotatedTextFragment } credit height imageType mode source url width } fragment BlockQuoteComponentFragment on BlockQuoteComponent { text textJson annotations { type length index attributes { name value } } } fragment BookInfoComponentFragment on BookInfoComponent { text textJson annotations { type length index attributes { name value } } } fragment ParagraphComponentFragment on ParagraphComponent { text textJson annotations { type length index attributes { name value } } } fragment PullQuoteComponentFragment on PullQuoteComponent { text textJson annotations { type length index attributes { name value } } } fragment CrossheadComponentFragment on CrossheadComponent { text } fragment OrderedListComponentFragment on OrderedListComponent { items { __typename ...AnnotatedTextFragment } } fragment UnorderedListComponentFragment on UnorderedListComponent { items { __typename ...AnnotatedTextFragment } } fragment VideoComponentFragment on VideoComponent { url title thumbnailImage } fragment InfoboxComponentFragment on InfoboxComponent { components { __typename type ...BlockQuoteComponentFragment ...BookInfoComponentFragment ...ParagraphComponentFragment ...PullQuoteComponentFragment ...CrossheadComponentFragment ...OrderedListComponentFragment ...UnorderedListComponentFragment ...VideoComponentFragment } } fragment InfographicComponentFragment on InfographicComponent { url title width fallback { __typename ...ImageComponentFragment } altText height width } fragment ArticleDataFragment on Article { id url brand byline rubric headline layout { headerStyle } contentIdentity { __typename ...ContentIdentityFragment } dateline dateFirstPublished dateModified datePublished dateRevised estimatedReadTime narration(selectionMethod: PREFER_ACTOR_NARRATION) { __typename ...NarrationFragment } printFlyTitle printHeadline printRubric flyTitle wordCount section { tegId name articles(pagingInfo: { pagingType: OFFSET pageSize: 6 pageNumber: 1 } ) @include(if: $includeRelatedArticles) { edges { node { __typename ...ArticleTeaserFragment } } } } teaserImage { __typename type ...ImageComponentFragment } tegId leadComponent { __typename type ...ImageComponentFragment } body { __typename type ...BlockQuoteComponentFragment ...BookInfoComponentFragment ...ParagraphComponentFragment ...PullQuoteComponentFragment ...CrossheadComponentFragment ...OrderedListComponentFragment ...UnorderedListComponentFragment ...InfoboxComponentFragment ...ImageComponentFragment ...VideoComponentFragment ...InfographicComponentFragment } footer { __typename type ...ParagraphComponentFragment } tags { name } ads { adData } podcast { __typename ...PodcastAudioFragment } }', # noqa: E501 |
| 307 |
} |
| 308 |
deep_url = 'https://cp2-graphql-gateway.p.aws.economist.com/graphql?' + urlencode(query, safe='()!', quote_via=quote) |
| 309 |
raw = get_content(deep_url) |
| 310 |
return raw |
| 311 |
|
| 312 |
def print_version(self, url): |
| 313 |
art_cont = self.get_article(url) |
| 314 |
pt = PersistentTemporaryFile('.html') |
| 315 |
pt.write(art_cont) |
| 316 |
pt.close() |
| 317 |
return 'file:///' + pt.name |