| 1 |
#!/usr/bin/env python |
| 2 |
""" |
| 3 |
scmp.com |
| 4 |
""" |
| 5 |
|
| 6 |
import json |
| 7 |
import time |
| 8 |
from datetime import datetime |
| 9 |
|
| 10 |
from html5_parser import parse |
| 11 |
from lxml import etree |
| 12 |
|
| 13 |
from calibre import replace_entities |
| 14 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 15 |
|
| 16 |
|
| 17 |
def E(parent, name, text='', **attrs): |
| 18 |
ans = parent.makeelement(name, **attrs) |
| 19 |
ans.text = text |
| 20 |
parent.append(ans) |
| 21 |
return ans |
| 22 |
|
| 23 |
|
| 24 |
def process_node(node, html_parent): |
| 25 |
ntype = node.get('type') |
| 26 |
|
| 27 |
if ntype not in { |
| 28 |
'track-viewed-percentage', |
| 29 |
'inline-ad-slot', |
| 30 |
'inline-widget', |
| 31 |
'text', |
| 32 |
}: |
| 33 |
c = html_parent.makeelement(ntype) |
| 34 |
if ntype != 'p': |
| 35 |
c.attrib.update({k: v or '' for k, v in node.get('attribs', {}).items()}) |
| 36 |
html_parent.append(c) |
| 37 |
for nc in node.get('children', ()): |
| 38 |
process_node(nc, c) |
| 39 |
elif ntype == 'text': |
| 40 |
text = node.get('data') |
| 41 |
if text: |
| 42 |
text = replace_entities(text) |
| 43 |
if len(html_parent): |
| 44 |
t = html_parent[-1] |
| 45 |
t.tail = (t.tail or '') + text |
| 46 |
else: |
| 47 |
html_parent.text = (html_parent.text or '') + text |
| 48 |
|
| 49 |
|
| 50 |
def ts_date(x): |
| 51 |
dt = datetime.fromtimestamp(x / 1000 + time.timezone) |
| 52 |
return dt.strftime('%b %d, %Y at %I:%M %p') |
| 53 |
|
| 54 |
|
| 55 |
def load_article_from_json(raw, root): |
| 56 |
# open('/t/raw.json', 'w').write(raw) |
| 57 |
data = json.loads(raw)['props']['pageProps']['payload']['data']['article'] |
| 58 |
body = root.xpath('//body')[0] |
| 59 |
for child in tuple(body): |
| 60 |
body.remove(child) |
| 61 |
article = E(body, 'article') |
| 62 |
E( |
| 63 |
article, |
| 64 |
'div', |
| 65 |
replace_entities(data['firstTopic']['name']), |
| 66 |
style='color: gray; font-size:small; font-weight:bold;', |
| 67 |
) |
| 68 |
E(article, 'h1', replace_entities(data['headline'])) |
| 69 |
# E(article, 'p', replace_entities(data['subHeadline']['text']), style='font-style: italic; color:#202020;') |
| 70 |
for subh in data['subHeadline']['json']: |
| 71 |
process_node(subh, article) |
| 72 |
auth = ts_date(data['publishedDate']) + ' | ' + str(data.get('readingTime', '')) + ' min read | ' + ', '.join([a['name'] for a in data['authors']]) |
| 73 |
E(article, 'p', auth, style='color: #202020; font-size:small;') |
| 74 |
main_image_url = sub_img = '' |
| 75 |
for l in data['images']: |
| 76 |
if l['type'] == 'leading': |
| 77 |
main_image_url = l['url'] |
| 78 |
sub_img = l['title'] |
| 79 |
if main_image_url != '': |
| 80 |
div = E(article, 'div') |
| 81 |
E(div, 'img', src=main_image_url) |
| 82 |
E(div, 'div', sub_img, style='text-align:center; font-size:small;') |
| 83 |
for node in data['body']['json']: |
| 84 |
process_node(node, article) |
| 85 |
|
| 86 |
|
| 87 |
class SCMP(BasicNewsRecipe): |
| 88 |
title = 'South China Morning Post' |
| 89 |
__author__ = 'unkn0wn' |
| 90 |
description = ( |
| 91 |
'The South China Morning Post is a leading news media company that has reported on China and Asia ' |
| 92 |
'for more than a century with global impact. Founded in 1903, SCMP is headquartered in Hong Kong, ' |
| 93 |
'where it is the city’s newspaper of record. Our teams span across Asia and the United States, ' |
| 94 |
'working together to connect with news consumers around the world. We are committed to informing ' |
| 95 |
'and inspiring through journalism of the highest standards. Our vision is to “Elevate Thought”, ' |
| 96 |
'and our mission is to “Lead the global conversation about China”.' |
| 97 |
) |
| 98 |
publisher = 'South China Morning Post Publishers Ltd.' |
| 99 |
oldest_article = 1 |
| 100 |
no_stylesheets = True |
| 101 |
remove_javascript = True |
| 102 |
remove_attributes = ['width', 'height'] |
| 103 |
encoding = 'utf-8' |
| 104 |
use_embedded_content = False |
| 105 |
language = 'en_HK' |
| 106 |
remove_empty_feeds = True |
| 107 |
resolve_internal_links = True |
| 108 |
publication_type = 'newspaper' |
| 109 |
ignore_duplicate_articles = {'title', 'url'} |
| 110 |
extra_css = 'blockquote, em { color: #202020; }' |
| 111 |
masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/c/c3/SCMP_logo.svg' |
| 112 |
|
| 113 |
def get_cover_url(self): |
| 114 |
soup = self.index_to_soup('https://www.frontpages.com/china-newspapers/') |
| 115 |
for img in soup.findAll('img', src=True): |
| 116 |
if 'south-china-morning-post' in img['src']: |
| 117 |
return 'https://www.frontpages.com' + img['src'] |
| 118 |
|
| 119 |
recipe_specific_options = { |
| 120 |
'days': { |
| 121 |
'short': 'Oldest article to download from this news source. In days ', |
| 122 |
'long': 'For example, 0.5, gives you articles from the past 12 hours', |
| 123 |
'default': str(oldest_article), |
| 124 |
}, |
| 125 |
'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'}, |
| 126 |
'rev': { |
| 127 |
'short': 'Reverse the order of articles in each feed?', |
| 128 |
'long': 'enter yes', |
| 129 |
'default': 'no', |
| 130 |
}, |
| 131 |
} |
| 132 |
|
| 133 |
def __init__(self, *args, **kwargs): |
| 134 |
BasicNewsRecipe.__init__(self, *args, **kwargs) |
| 135 |
d = self.recipe_specific_options.get('days') |
| 136 |
if d and isinstance(d, str): |
| 137 |
self.oldest_article = float(d) |
| 138 |
r = self.recipe_specific_options.get('rev') |
| 139 |
if r and isinstance(r, str): |
| 140 |
if r.lower() == 'yes': |
| 141 |
self.reverse_article_order = True |
| 142 |
c = self.recipe_specific_options.get('comp') |
| 143 |
if c and isinstance(c, str): |
| 144 |
if c.lower() == 'yes': |
| 145 |
self.compress_news_images = True |
| 146 |
|
| 147 |
# used when unable to extract article from <script>, particularly in the Sports section |
| 148 |
remove_tags = [ |
| 149 |
dict( |
| 150 |
classes( |
| 151 |
'sticky-wrap relative social-media social-media--extended__shares' |
| 152 |
' article-body-comment scmp_button_comment_wrapper social-media--extended__in-site' |
| 153 |
' footer scmp-advert-tile sidebar-col related-article share-widget' |
| 154 |
) |
| 155 |
), |
| 156 |
dict(attrs={'addthis_title': True}), |
| 157 |
dict(name=['script', 'style']), |
| 158 |
] |
| 159 |
|
| 160 |
# https://www.scmp.com/rss |
| 161 |
feeds = [ |
| 162 |
('Hong Kong', 'https://www.scmp.com/rss/2/feed'), |
| 163 |
('China', 'https://www.scmp.com/rss/4/feed'), |
| 164 |
('Asia', 'https://www.scmp.com/rss/3/feed'), |
| 165 |
('World', 'https://www.scmp.com/rss/5/feed'), |
| 166 |
('Business', 'https://www.scmp.com/rss/92/feed'), |
| 167 |
('Tech', 'https://www.scmp.com/rss/36/feed'), |
| 168 |
('Life', 'https://www.scmp.com/rss/94/feed'), |
| 169 |
('Culture', 'https://www.scmp.com/rss/322296/feed'), |
| 170 |
('Sport', 'https://www.scmp.com/rss/95/feed'), |
| 171 |
('Post Mag', 'https://www.scmp.com/rss/71/feed'), |
| 172 |
('Style', 'https://www.scmp.com/rss/72/feed'), |
| 173 |
('News', 'https://www.scmp.com/rss/91/feed'), |
| 174 |
] |
| 175 |
|
| 176 |
def print_version(self, url): |
| 177 |
return url.split('?')[0] |
| 178 |
|
| 179 |
def preprocess_raw_html(self, raw_html, url): |
| 180 |
body = '<html><body><article></article></body></html>' |
| 181 |
b_root = parse(body) |
| 182 |
root = parse(raw_html) |
| 183 |
script = root.xpath('//script[@id="__NEXT_DATA__"]') |
| 184 |
if script: |
| 185 |
try: |
| 186 |
load_article_from_json(script[0].text, b_root) |
| 187 |
except Exception: |
| 188 |
self.log('** Failed parse: ', url) |
| 189 |
return raw_html |
| 190 |
head = b_root.xpath('//h2') + b_root.xpath('//h3') |
| 191 |
for h2 in head: |
| 192 |
h2.tag = 'h4' |
| 193 |
raw = etree.tostring(b_root, encoding='unicode') |
| 194 |
return raw |
| 195 |
return raw_html |
| 196 |
|
| 197 |
def preprocess_html(self, soup): |
| 198 |
from urllib.parse import urlparse |
| 199 |
|
| 200 |
for img in soup.findAll('img', attrs={'src': True}): |
| 201 |
y = 'https://img.i-scmp.com/cdn-cgi/image/fit=contain,width=768,format=auto' |
| 202 |
img['src'] = y + urlparse(img['src']).path |
| 203 |
for img in soup.findAll('img', attrs={'title': True}): |
| 204 |
div = soup.new_tag('div', attrs={'style': 'text-align:center; font-size:small;'}) |
| 205 |
div.string = img.get('title', '') |
| 206 |
if img.find_parent('div'): |
| 207 |
img.find_parent('div').append(div) |
| 208 |
else: |
| 209 |
img.append(div) |
| 210 |
return soup |