Debian Codegraph @

recipes/bbc.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net>
4
5 import json
6
7 from calibre import prepare_string_for_xml
8 from calibre.web.feeds.recipes import BasicNewsRecipe
9
10
11 # Article JSON parser {{{
12 def serialize_image(block):
13 yield '<div>'
14 block = block['model']
15 img = block['image']
16 alt = prepare_string_for_xml(img.get('alt') or '', True)
17 for q in ('originalSrc', 'src'):
18 if q in img:
19 src = prepare_string_for_xml(img[q])
20 break
21 else:
22 raise ValueError('No src found in img block: {}'.format(img))
23 yield '<img src="{}" alt="{}"/>'.format(src, alt)
24 caption = block.get('caption')
25 if caption and caption.get('type') == 'text':
26 yield '<div>'
27 yield from serialize_paragraph(caption)
28 yield '</div>'
29 yield '</div>'
30
31
32 def block_tag(name, generator):
33 yield '<' + name + '>'
34 yield from generator
35 yield '</' + name + '>'
36
37
38 def serialize_paragraph(block):
39 block = block['model']
40 for x in block['blocks']:
41 xt = x['type']
42 if xt == 'fragment':
43 styles = []
44 model = x['model']
45 for attr in model['attributes']:
46 if attr == 'bold':
47 styles.append('font-weight: bold')
48 elif attr in ('italic', 'italics'):
49 styles.append('font-style: italic')
50 if styles:
51 prefix = '<span style="{}">'.format('; '.join(styles))
52 suffix = '</span>'
53 else:
54 prefix = suffix = ''
55 yield prefix + prepare_string_for_xml(model['text']) + suffix
56 elif xt == 'urlLink':
57 model = x['model']
58 yield '<a href="{}">{}</a>'.format(prepare_string_for_xml(model['locator'], True), prepare_string_for_xml(model['text']))
59
60
61 def serialize_list(block):
62 for x in block['model']['blocks']:
63 if x['type'] == 'listItem':
64 yield from block_tag('li', serialize_paragraph(x))
65
66
67 def serialize_text(block):
68 block = block['model']
69 for x in block['blocks']:
70 xt = x['type']
71 if xt == 'paragraph':
72 yield from block_tag('p', serialize_paragraph(x))
73 elif xt == 'unorderedList':
74 yield from block_tag('ul', serialize_list(x))
75 elif xt == 'orderedList':
76 yield from block_tag('ol', serialize_list(x))
77 else:
78 raise KeyError('Unknown block type: ' + x['type'])
79
80
81 def serialize_contributor(contributor):
82 if 'title' in contributor:
83 yield '<h3>' + prepare_string_for_xml(contributor['title']) + '</h3>'
84 if 'subtitle' in contributor:
85 yield '<div>' + prepare_string_for_xml(contributor['subtitle']) + '</div>'
86
87
88 def extract_text_block_plaintext(text_block):
89 # Pull the plain text out of a nested {type: text, model: {blocks: [...]}}
90 # block. Used to recover a headline string from the 'headline' / 'topper'
91 # block types that BBC Sport articles now use in place of the top-level
92 # article['headline'] field.
93 chunks = []
94 for x in text_block.get('model', {}).get('blocks', []):
95 if x.get('type') == 'paragraph':
96 for f in x.get('model', {}).get('blocks', []):
97 if f.get('type') == 'fragment':
98 chunks.append(f.get('model', {}).get('text', ''))
99 return ''.join(chunks)
100
101
102 def parse_article_json(root, abort_article):
103 data = root['data']
104 has_media_experience = False
105 for key in data:
106 if key.startswith('article?'):
107 article = data[key]['data']
108 break
109 elif key.startswith('media-experience?'):
110 has_media_experience = True
111 else:
112 if has_media_experience:
113 abort_article('Skipping video article')
114 return
115 raise KeyError('No article found in data keys: {}'.format(data.keys()))
116 lines = []
117 if article.get('headline'):
118 lines.append('<h1>{}</h1>'.format(prepare_string_for_xml(article['headline'])))
119 elif article.get('metadata', {}).get('seoHeadline'):
120 # BBC Sport articles no longer populate the top-level 'headline' field;
121 # the visible headline is on metadata.seoHeadline (and is plain text).
122 lines.append('<h1>{}</h1>'.format(prepare_string_for_xml(article['metadata']['seoHeadline'])))
123 if article.get('contributor'):
124 lines.extend(serialize_contributor(article['contributor']))
125 headline_emitted = bool(lines and lines[0].startswith('<h1>'))
126 for block in article['content']['model']['blocks']:
127 bt = block.get('type')
128 if bt == 'image':
129 lines.extend(serialize_image(block))
130 elif bt == 'text':
131 lines.extend(serialize_text(block))
132 elif bt == 'headline' and not headline_emitted:
133 # New BBC Sport block-type containing the headline as a nested text block.
134 inner = block.get('model', {}).get('blocks') or [{}]
135 text = extract_text_block_plaintext(inner[0])
136 if text:
137 lines.insert(0, '<h1>{}</h1>'.format(prepare_string_for_xml(text)))
138 headline_emitted = True
139 elif bt == 'topper' and not headline_emitted:
140 # 'high-impact' BBC Sport article variant: headline lives under
141 # block.model.heading.blocks[0] rather than block.model.blocks[0].
142 heading = block.get('model', {}).get('heading') or {}
143 inner = heading.get('blocks') or [{}]
144 text = extract_text_block_plaintext(inner[0])
145 if text:
146 lines.insert(0, '<h1>{}</h1>'.format(prepare_string_for_xml(text)))
147 headline_emitted = True
148 return '<html><body id="main-content"><article>' + '\n'.join(lines) + '</article></body></html>'
149
150
151 def parse_raw_html(html, abort_article):
152 q = '>window.__INITIAL_DATA__="{'
153 idx = html.find(q)
154 if idx < 0:
155 print('Failed to find JSON')
156 return html
157 data = html[idx + len(q) - 2 :]
158 idx = data.find('}";</script>')
159 data = data[: idx + 2]
160 data = json.loads(data)
161 root = json.loads(data)
162 return parse_article_json(root, abort_article)
163
164
165 if __name__ == '__main__':
166 print(parse_raw_html(open('/t/raw.html').read(), print))
167 # }}}
168
169
170 class BBCNews(BasicNewsRecipe):
171 # Select / de-select the feeds you want in your ebook.
172 feeds = [
173 ('UK', 'https://feeds.bbci.co.uk/news/uk/rss.xml'),
174 ('World', 'https://feeds.bbci.co.uk/news/world/rss.xml'),
175 # ("England", "https://feeds.bbci.co.uk/news/england/rss.xml"),
176 # ("Scotland", "https://feeds.bbci.co.uk/news/scotland/rss.xml"),
177 # ("Wales", "https://feeds.bbci.co.uk/news/wales/rss.xml"),
178 # ("N. Ireland", "https://feeds.bbci.co.uk/news/northern_ireland/rss.xml"),
179 # ("Africa", "https://feeds.bbci.co.uk/news/world/africa/rss.xml"),
180 # ("Asia", "https://feeds.bbci.co.uk/news/world/asia/rss.xml"),
181 # ("Europe", "https://feeds.bbci.co.uk/news/world/europe/rss.xml"),
182 # ("Latin America", "https://feeds.bbci.co.uk/news/world/latin_america/rss.xml"),
183 # ("Middle East", "https://feeds.bbci.co.uk/news/world/middle_east/rss.xml"),
184 ('US & Canada', 'https://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml'),
185 ('Politics', 'https://feeds.bbci.co.uk/news/politics/rss.xml'),
186 ('Science/Environment', 'https://feeds.bbci.co.uk/news/science_and_environment/rss.xml'),
187 ('Technology', 'https://feeds.bbci.co.uk/news/technology/rss.xml'),
188 ('Entertainment/Arts', 'https://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml'),
189 # ("Health", "https://feeds.bbci.co.uk/news/health/rss.xml"),
190 # ("Education/Family", "https://feeds.bbci.co.uk/news/education/rss.xml"),
191 ('Business', 'https://feeds.bbci.co.uk/news/business/rss.xml'),
192 # ("Newsbeat", "https://www.bbc.co.uk/newsbeat/rss.xml"),
193 # ("Click", "http://newsrss.bbc.co.uk/rss/newsonline_uk_edition/programmes/click_online/rss.xml"),
194 # ("Blog: Mark D'Arcy (Parliamentary Correspondent)", "https://feeds.bbci.co.uk/news/correspondents/markdarcy/rss.sxml"),
195 # ("Blog: Robert Peston (Business Editor)", "https://feeds.bbci.co.uk/news/correspondents/robertpeston/rss.sxml"),
196 # ("Blog: Stephanie Flanders (Economics Editor)", "https://feeds.bbci.co.uk/news/correspondents/stephanieflanders/rss.sxml"),
197 ('Sport Front Page', 'http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/front_page/rss.xml'),
198 # ("Football", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/football/rss.xml"),
199 # ("Cricket", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/cricket/rss.xml"),
200 # ("Rugby Union", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/rugby_union/rss.xml"),
201 # ("Rugby League", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/rugby_league/rss.xml"),
202 # ("Tennis", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/tennis/rss.xml"),
203 # ("Golf", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/golf/rss.xml"),
204 # ("Motorsport", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/motorsport/rss.xml"),
205 # ("Boxing", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/boxing/rss.xml"),
206 # ("Athletics", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/athletics/rss.xml"),
207 # ("Snooker", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/other_sports/snooker/rss.xml"),
208 # ("Horse Racing", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/other_sports/horse_racing/rss.xml"),
209 # ("Cycling", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/other_sports/cycling/rss.xml"),
210 # ("Disability Sport", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/other_sports/disability_sport/rss.xml"),
211 # ("Other Sport", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/other_sports/rss.xml"),
212 # ("Olympics 2012", "http://newsrss.bbc.co.uk/rss/sportonline_uk_edition/other_sports/olympics_2012/rss.xml"),
213 # ("N. Ireland Politics", "https://feeds.bbci.co.uk/news/northern_ireland/northern_ireland_politics/rss.xml"),
214 # ("Scotland Politics", "https://feeds.bbci.co.uk/news/scotland/scotland_politics/rss.xml"),
215 # ("Scotland Business", "https://feeds.bbci.co.uk/news/scotland/scotland_business/rss.xml"),
216 # ("E. Scotland, Edinburgh & Fife", "https://feeds.bbci.co.uk/news/scotland/edinburgh_east_and_fife/rss.xml"),
217 # ("W. Scotland & Glasgow", "https://feeds.bbci.co.uk/news/scotland/glasgow_and_west/rss.xml"),
218 # ("Highlands & Islands", "https://feeds.bbci.co.uk/news/scotland/highlands_and_islands/rss.xml"),
219 # ("NE. Scotland, Orkney & Shetland", "https://feeds.bbci.co.uk/news/scotland/north_east_orkney_and_shetland/rss.xml"),
220 # ("South Scotland", "https://feeds.bbci.co.uk/news/scotland/south_scotland/rss.xml"),
221 # ("Central Scotland & Tayside", "https://feeds.bbci.co.uk/news/scotland/tayside_and_central/rss.xml"),
222 # ("Wales Politics", "https://feeds.bbci.co.uk/news/wales/wales_politics/rss.xml"),
223 # ("NW. Wales", "https://feeds.bbci.co.uk/news/wales/north_west_wales/rss.xml"),
224 # ("NE. Wales", "https://feeds.bbci.co.uk/news/wales/north_east_wales/rss.xml"),
225 # ("Mid. Wales", "https://feeds.bbci.co.uk/news/wales/mid_wales/rss.xml"),
226 # ("SW. Wales", "https://feeds.bbci.co.uk/news/wales/south_west_wales/rss.xml"),
227 # ("SE. Wales", "https://feeds.bbci.co.uk/news/wales/south_east_wales/rss.xml"),
228 # ("Newyddion - News in Welsh", "https://feeds.bbci.co.uk/newyddion/rss.xml"),
229 # ("Gwleidyddiaeth", "https://feeds.bbci.co.uk/newyddion/gwleidyddiaeth/rss.xml"),
230 # ("Gogledd-Ddwyrain", "https://feeds.bbci.co.uk/newyddion/gogledd-ddwyrain/rss.xml"),
231 # ("Gogledd-Orllewin", "https://feeds.bbci.co.uk/newyddion/gogledd-orllewin/rss.xml"),
232 # ("Canolbarth", "https://feeds.bbci.co.uk/newyddion/canolbarth/rss.xml"),
233 # ("De-Ddwyrain", "https://feeds.bbci.co.uk/newyddion/de-ddwyrain/rss.xml"),
234 # ("De-Orllewin", "https://feeds.bbci.co.uk/newyddion/de-orllewin/rss.xml"),
235 ('News', 'https://feeds.bbci.co.uk/news/rss.xml'),
236 ]
237
238 # **** SELECT YOUR USER PREFERENCES ****
239
240 # Title to use for the ebook.
241 #
242 title = 'BBC News'
243
244 # A brief description for the ebook.
245 #
246 description = 'BBC web site ebook created using rss feeds.'
247
248 # The max number of articles which may be downloaded from each feed.
249 # I've never seen more than about 70 articles in a single feed in the
250 # BBC feeds.
251 #
252 max_articles_per_feed = 100
253
254 # The max age of articles which may be downloaded from each feed. This is
255 # specified in days - note fractions of days are allowed, Eg. 2.5 (2 and a
256 # half days). My default of 1.5 days is the last 36 hours, the point at
257 # which I've decided 'news' becomes 'old news', but be warned this is not
258 # so good for the blogs, technology, magazine, etc., and sports feeds.
259 # You may wish to extend this to 2-5 but watch out ebook creation time will
260 # increase as well. Setting this to 30 will get everything (AFAICT) as long
261 # as max_articles_per_feed remains set high (except for 'Click' which is
262 # v. low volume and its currently oldest article is 4th Feb 2011).
263 #
264 oldest_article = 1.5
265
266 recipe_specific_options = {
267 'days': {
268 'short': 'Oldest article to download from this news source. In days ',
269 'long': 'For example, 0.5, gives you articles from the past 12 hours',
270 'default': str(oldest_article),
271 }
272 }
273
274 def __init__(self, *args, **kwargs):
275 BasicNewsRecipe.__init__(self, *args, **kwargs)
276 d = self.recipe_specific_options.get('days')
277 if d and isinstance(d, str):
278 self.oldest_article = float(d)
279
280 # Number of simultaneous downloads. 20 is consistently working fine on the
281 # BBC News feeds with no problems. Speeds things up from the default of 5.
282 # If you have a lot of feeds and/or have increased oldest_article above 2
283 # then you may wish to try increasing simultaneous_downloads to 25-30,
284 # Or, of course, if you are in a hurry. [I've not tried beyond 20.]
285 #
286 simultaneous_downloads = 20
287
288 # The format string for the date shown on the ebook's first page.
289 # List of all values: https://docs.python.org/library/time.html
290 # Default in news.py has a leading space so that's mirrored here.
291 # As with 'feeds' select/de-select by adding/removing the initial '#',
292 # only one timefmt should be selected, here's a few to choose from.
293 #
294 # [Fri, 14 Nov 2011] (Calibre default)
295 timefmt = ' [%a, %d %b %Y]'
296 # timefmt = ' [%a, %d %b %Y %H:%M]' # [Fri, 14 Nov 2011 18:30]
297 # timefmt = ' [%a, %d %b %Y %I:%M %p]' # [Fri, 14 Nov 2011 06:30 PM]
298 # timefmt = ' [%d %b %Y]' # [14 Nov 2011]
299 # timefmt = ' [%d %b %Y %H:%M]' # [14 Nov 2011 18.30]
300 # timefmt = ' [%Y-%m-%d]' # [2011-11-14]
301 # timefmt = ' [%Y-%m-%d-%H-%M]' # [2011-11-14-18-30]
302
303 #
304 # **** IMPORTANT ****
305 #
306 # DO NOT EDIT BELOW HERE UNLESS YOU KNOW WHAT YOU ARE DOING.
307 #
308 # DO NOT EDIT BELOW HERE UNLESS YOU KNOW WHAT YOU ARE DOING.
309 #
310 # I MEAN IT, YES I DO, ABSOLUTELY, AT YOU OWN RISK. :)
311 #
312 # **** IMPORTANT ****
313 #
314
315 # Author of this recipe.
316 __author__ = 'Kovid Goyal'
317
318 # Specify English as the language of the RSS feeds (ISO-639 code).
319 language = 'en_GB'
320
321 # Set publisher and publication type.
322 publication_type = 'newspaper'
323 encoding = 'utf-8'
324 use_embedded_content = False
325
326 # Removes empty feeds - why keep them!?
327 remove_empty_feeds = True
328 ignore_duplicate_articles = {'title', 'url'}
329 resolve_internal_links = True
330
331 def preprocess_raw_html(self, raw_html, url):
332 return parse_raw_html(raw_html, self.abort_article)
333
334 keep_only_tags = [dict(name='article')]
335
336 remove_tags = [dict(name=['button', 'svg', 'iframe']), dict(attrs={'data-component': ['ad-slot', 'tags', 'links-block', 'metadata-block', 'topic-list']})]
337
338 remove_attributes = ['style', 'height', 'width']
339 no_stylesheets = True
340 extra_css = '''
341 figure,
342 [data-component="byline-block"],
343 [data-component="caption-block"],
344 [data-component="image-block"] {
345 font-size:small;
346 }
347 '''
348 cover_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/6/62/BBC_News_2019.svg/768px-BBC_News_2019.svg.png'
349 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/4/41/BBC_Logo_2021.svg'
350
351 def preprocess_html(self, soup):
352 for placeholder in soup.findAll('img', attrs={'src': lambda x: x and x.endswith('placeholder.png')}):
353 placeholder.decompose()
354 for img in soup.findAll('img'):
355 img.attrs = {'src': img.get('src', '')}
356 for h2 in soup.findAll(['h2', 'h3']):
357 h2.name = 'h4'
358 return soup