Debian Codegraph @

recipes/scmp.recipe

raw
1 #!/usr/bin/env python
2 """
3 scmp.com
4 """
5
6 import json
7 import time
8 from datetime import datetime
9
10 from html5_parser import parse
11 from lxml import etree
12
13 from calibre import replace_entities
14 from calibre.web.feeds.news import BasicNewsRecipe, classes
15
16
17 def E(parent, name, text='', **attrs):
18 ans = parent.makeelement(name, **attrs)
19 ans.text = text
20 parent.append(ans)
21 return ans
22
23
24 def process_node(node, html_parent):
25 ntype = node.get('type')
26
27 if ntype not in {
28 'track-viewed-percentage',
29 'inline-ad-slot',
30 'inline-widget',
31 'text',
32 }:
33 c = html_parent.makeelement(ntype)
34 if ntype != 'p':
35 c.attrib.update({k: v or '' for k, v in node.get('attribs', {}).items()})
36 html_parent.append(c)
37 for nc in node.get('children', ()):
38 process_node(nc, c)
39 elif ntype == 'text':
40 text = node.get('data')
41 if text:
42 text = replace_entities(text)
43 if len(html_parent):
44 t = html_parent[-1]
45 t.tail = (t.tail or '') + text
46 else:
47 html_parent.text = (html_parent.text or '') + text
48
49
50 def ts_date(x):
51 dt = datetime.fromtimestamp(x / 1000 + time.timezone)
52 return dt.strftime('%b %d, %Y at %I:%M %p')
53
54
55 def load_article_from_json(raw, root):
56 # open('/t/raw.json', 'w').write(raw)
57 data = json.loads(raw)['props']['pageProps']['payload']['data']['article']
58 body = root.xpath('//body')[0]
59 for child in tuple(body):
60 body.remove(child)
61 article = E(body, 'article')
62 E(
63 article,
64 'div',
65 replace_entities(data['firstTopic']['name']),
66 style='color: gray; font-size:small; font-weight:bold;',
67 )
68 E(article, 'h1', replace_entities(data['headline']))
69 # E(article, 'p', replace_entities(data['subHeadline']['text']), style='font-style: italic; color:#202020;')
70 for subh in data['subHeadline']['json']:
71 process_node(subh, article)
72 auth = ts_date(data['publishedDate']) + ' | ' + str(data.get('readingTime', '')) + ' min read | ' + ', '.join([a['name'] for a in data['authors']])
73 E(article, 'p', auth, style='color: #202020; font-size:small;')
74 main_image_url = sub_img = ''
75 for l in data['images']:
76 if l['type'] == 'leading':
77 main_image_url = l['url']
78 sub_img = l['title']
79 if main_image_url != '':
80 div = E(article, 'div')
81 E(div, 'img', src=main_image_url)
82 E(div, 'div', sub_img, style='text-align:center; font-size:small;')
83 for node in data['body']['json']:
84 process_node(node, article)
85
86
87 class SCMP(BasicNewsRecipe):
88 title = 'South China Morning Post'
89 __author__ = 'unkn0wn'
90 description = (
91 'The South China Morning Post is a leading news media company that has reported on China and Asia '
92 'for more than a century with global impact. Founded in 1903, SCMP is headquartered in Hong Kong, '
93 'where it is the city’s newspaper of record. Our teams span across Asia and the United States, '
94 'working together to connect with news consumers around the world. We are committed to informing '
95 'and inspiring through journalism of the highest standards. Our vision is to “Elevate Thought”, '
96 'and our mission is to “Lead the global conversation about China”.'
97 )
98 publisher = 'South China Morning Post Publishers Ltd.'
99 oldest_article = 1
100 no_stylesheets = True
101 remove_javascript = True
102 remove_attributes = ['width', 'height']
103 encoding = 'utf-8'
104 use_embedded_content = False
105 language = 'en_HK'
106 remove_empty_feeds = True
107 resolve_internal_links = True
108 publication_type = 'newspaper'
109 ignore_duplicate_articles = {'title', 'url'}
110 extra_css = 'blockquote, em { color: #202020; }'
111 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/c/c3/SCMP_logo.svg'
112
113 def get_cover_url(self):
114 soup = self.index_to_soup('https://www.frontpages.com/china-newspapers/')
115 for img in soup.findAll('img', src=True):
116 if 'south-china-morning-post' in img['src']:
117 return 'https://www.frontpages.com' + img['src']
118
119 recipe_specific_options = {
120 'days': {
121 'short': 'Oldest article to download from this news source. In days ',
122 'long': 'For example, 0.5, gives you articles from the past 12 hours',
123 'default': str(oldest_article),
124 },
125 'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'},
126 'rev': {
127 'short': 'Reverse the order of articles in each feed?',
128 'long': 'enter yes',
129 'default': 'no',
130 },
131 }
132
133 def __init__(self, *args, **kwargs):
134 BasicNewsRecipe.__init__(self, *args, **kwargs)
135 d = self.recipe_specific_options.get('days')
136 if d and isinstance(d, str):
137 self.oldest_article = float(d)
138 r = self.recipe_specific_options.get('rev')
139 if r and isinstance(r, str):
140 if r.lower() == 'yes':
141 self.reverse_article_order = True
142 c = self.recipe_specific_options.get('comp')
143 if c and isinstance(c, str):
144 if c.lower() == 'yes':
145 self.compress_news_images = True
146
147 # used when unable to extract article from <script>, particularly in the Sports section
148 remove_tags = [
149 dict(
150 classes(
151 'sticky-wrap relative social-media social-media--extended__shares'
152 ' article-body-comment scmp_button_comment_wrapper social-media--extended__in-site'
153 ' footer scmp-advert-tile sidebar-col related-article share-widget'
154 )
155 ),
156 dict(attrs={'addthis_title': True}),
157 dict(name=['script', 'style']),
158 ]
159
160 # https://www.scmp.com/rss
161 feeds = [
162 ('Hong Kong', 'https://www.scmp.com/rss/2/feed'),
163 ('China', 'https://www.scmp.com/rss/4/feed'),
164 ('Asia', 'https://www.scmp.com/rss/3/feed'),
165 ('World', 'https://www.scmp.com/rss/5/feed'),
166 ('Business', 'https://www.scmp.com/rss/92/feed'),
167 ('Tech', 'https://www.scmp.com/rss/36/feed'),
168 ('Life', 'https://www.scmp.com/rss/94/feed'),
169 ('Culture', 'https://www.scmp.com/rss/322296/feed'),
170 ('Sport', 'https://www.scmp.com/rss/95/feed'),
171 ('Post Mag', 'https://www.scmp.com/rss/71/feed'),
172 ('Style', 'https://www.scmp.com/rss/72/feed'),
173 ('News', 'https://www.scmp.com/rss/91/feed'),
174 ]
175
176 def print_version(self, url):
177 return url.split('?')[0]
178
179 def preprocess_raw_html(self, raw_html, url):
180 body = '<html><body><article></article></body></html>'
181 b_root = parse(body)
182 root = parse(raw_html)
183 script = root.xpath('//script[@id="__NEXT_DATA__"]')
184 if script:
185 try:
186 load_article_from_json(script[0].text, b_root)
187 except Exception:
188 self.log('** Failed parse: ', url)
189 return raw_html
190 head = b_root.xpath('//h2') + b_root.xpath('//h3')
191 for h2 in head:
192 h2.tag = 'h4'
193 raw = etree.tostring(b_root, encoding='unicode')
194 return raw
195 return raw_html
196
197 def preprocess_html(self, soup):
198 from urllib.parse import urlparse
199
200 for img in soup.findAll('img', attrs={'src': True}):
201 y = 'https://img.i-scmp.com/cdn-cgi/image/fit=contain,width=768,format=auto'
202 img['src'] = y + urlparse(img['src']).path
203 for img in soup.findAll('img', attrs={'title': True}):
204 div = soup.new_tag('div', attrs={'style': 'text-align:center; font-size:small;'})
205 div.string = img.get('title', '')
206 if img.find_parent('div'):
207 img.find_parent('div').append(div)
208 else:
209 img.append(div)
210 return soup