Debian Codegraph @

recipes/bbc_fast.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net>
4
5 import json
6
7 from calibre import prepare_string_for_xml
8 from calibre.web.feeds.recipes import BasicNewsRecipe
9
10
11 # Article JSON parser {{{
12 def serialize_image(block):
13 yield '<div>'
14 block = block['model']
15 img = block['image']
16 alt = prepare_string_for_xml(img.get('alt') or '', True)
17 for q in ('originalSrc', 'src'):
18 if q in img:
19 src = prepare_string_for_xml(img[q])
20 break
21 else:
22 raise ValueError('No src found in img block: {}'.format(img))
23 yield '<img src="{}" alt="{}"/>'.format(src, alt)
24 caption = block.get('caption')
25 if caption and caption.get('type') == 'text':
26 yield '<div>'
27 yield from serialize_paragraph(caption)
28 yield '</div>'
29 yield '</div>'
30
31
32 def block_tag(name, generator):
33 yield '<' + name + '>'
34 yield from generator
35 yield '</' + name + '>'
36
37
38 def serialize_paragraph(block):
39 block = block['model']
40 for x in block['blocks']:
41 xt = x['type']
42 if xt == 'fragment':
43 styles = []
44 model = x['model']
45 for attr in model['attributes']:
46 if attr == 'bold':
47 styles.append('font-weight: bold')
48 elif attr in ('italic', 'italics'):
49 styles.append('font-style: italic')
50 if styles:
51 prefix = '<span style="{}">'.format('; '.join(styles))
52 suffix = '</span>'
53 else:
54 prefix = suffix = ''
55 yield prefix + prepare_string_for_xml(model['text']) + suffix
56 elif xt == 'urlLink':
57 model = x['model']
58 yield '<a href="{}">{}</a>'.format(prepare_string_for_xml(model['locator'], True), prepare_string_for_xml(model['text']))
59
60
61 def serialize_list(block):
62 for x in block['model']['blocks']:
63 if x['type'] == 'listItem':
64 yield from block_tag('li', serialize_paragraph(x))
65
66
67 def serialize_text(block):
68 block = block['model']
69 for x in block['blocks']:
70 xt = x['type']
71 if xt == 'paragraph':
72 yield from block_tag('p', serialize_paragraph(x))
73 elif xt == 'unorderedList':
74 yield from block_tag('ul', serialize_list(x))
75 elif xt == 'orderedList':
76 yield from block_tag('ol', serialize_list(x))
77 else:
78 raise KeyError('Unknown block type: ' + x['type'])
79
80
81 def serialize_contributor(contributor):
82 if 'title' in contributor:
83 yield '<h3>' + prepare_string_for_xml(contributor['title']) + '</h3>'
84 if 'subtitle' in contributor:
85 yield '<div>' + prepare_string_for_xml(contributor['subtitle']) + '</div>'
86
87
88 def extract_text_block_plaintext(text_block):
89 # Pull the plain text out of a nested {type: text, model: {blocks: [...]}}
90 # block. Used to recover a headline string from the 'headline' / 'topper'
91 # block types that BBC Sport articles now use in place of the top-level
92 # article['headline'] field.
93 chunks = []
94 for x in text_block.get('model', {}).get('blocks', []):
95 if x.get('type') == 'paragraph':
96 for f in x.get('model', {}).get('blocks', []):
97 if f.get('type') == 'fragment':
98 chunks.append(f.get('model', {}).get('text', ''))
99 return ''.join(chunks)
100
101
102 def parse_article_json(root, abort_article):
103 data = root['data']
104 has_media_experience = False
105 for key in data:
106 if key.startswith('article?'):
107 article = data[key]['data']
108 break
109 elif key.startswith('media-experience?'):
110 has_media_experience = True
111 else:
112 if has_media_experience:
113 abort_article('Skipping video article')
114 return
115 raise KeyError('No article found in data keys: {}'.format(data.keys()))
116 lines = []
117 if article.get('headline'):
118 lines.append('<h1>{}</h1>'.format(prepare_string_for_xml(article['headline'])))
119 elif article.get('metadata', {}).get('seoHeadline'):
120 # BBC Sport articles no longer populate the top-level 'headline' field;
121 # the visible headline is on metadata.seoHeadline (and is plain text).
122 lines.append('<h1>{}</h1>'.format(prepare_string_for_xml(article['metadata']['seoHeadline'])))
123 if article.get('contributor'):
124 lines.extend(serialize_contributor(article['contributor']))
125 headline_emitted = bool(lines and lines[0].startswith('<h1>'))
126 for block in article['content']['model']['blocks']:
127 bt = block.get('type')
128 if bt == 'image':
129 lines.extend(serialize_image(block))
130 elif bt == 'text':
131 lines.extend(serialize_text(block))
132 elif bt == 'headline' and not headline_emitted:
133 # New BBC Sport block-type containing the headline as a nested text block.
134 inner = block.get('model', {}).get('blocks') or [{}]
135 text = extract_text_block_plaintext(inner[0])
136 if text:
137 lines.insert(0, '<h1>{}</h1>'.format(prepare_string_for_xml(text)))
138 headline_emitted = True
139 elif bt == 'topper' and not headline_emitted:
140 # 'high-impact' BBC Sport article variant: headline lives under
141 # block.model.heading.blocks[0] rather than block.model.blocks[0].
142 heading = block.get('model', {}).get('heading') or {}
143 inner = heading.get('blocks') or [{}]
144 text = extract_text_block_plaintext(inner[0])
145 if text:
146 lines.insert(0, '<h1>{}</h1>'.format(prepare_string_for_xml(text)))
147 headline_emitted = True
148 return '<html><body id="main-content"><article>' + '\n'.join(lines) + '</article></body></html>'
149
150
151 def parse_raw_html(html, abort_article):
152 q = '>window.__INITIAL_DATA__="{'
153 idx = html.find(q)
154 if idx < 0:
155 print('Failed to find JSON')
156 return html
157 data = html[idx + len(q) - 2 :]
158 idx = data.find('}";</script>')
159 data = data[: idx + 2]
160 data = json.loads(data)
161 root = json.loads(data)
162 return parse_article_json(root, abort_article)
163
164
165 if __name__ == '__main__':
166 print(parse_raw_html(open('/t/raw.html').read(), print))
167 # }}}
168
169
170 class BBC(BasicNewsRecipe):
171 title = 'BBC News (fast)'
172 __author__ = 'Kovid Goyal, unkn0wn'
173 description = 'Visit BBC News for up-to-the-minute news, breaking news, video, audio and feature stories. BBC News provides trusted World and UK news as well as local and regional perspectives. Also entertainment, business, science, technology and health news.' # noqa: E501
174 oldest_article = 2
175 max_articles_per_feed = 100
176 no_stylesheets = True
177 use_embedded_content = False
178 encoding = 'utf8'
179 publisher = 'BBC'
180 category = 'news, UK, world'
181 language = 'en_GB'
182 conversion_options = {
183 'comments': description,
184 'tags': category,
185 'language': language,
186 'publisher': publisher,
187 }
188 # Removes empty feeds - why keep them!?
189 remove_empty_feeds = True
190 ignore_duplicate_articles = {'title', 'url'}
191 resolve_internal_links = True
192
193 keep_only_tags = [dict(name='article')]
194
195 remove_tags = [dict(name=['button', 'svg', 'iframe']), dict(attrs={'data-component': ['ad-slot', 'tags', 'links-block', 'metadata-block', 'topic-list']})]
196
197 remove_attributes = ['style', 'height', 'width']
198 extra_css = '''
199 figure,
200 [data-component="byline-block"],
201 [data-component="caption-block"],
202 [data-component="image-block"] {
203 font-size:small;
204 }
205 '''
206 cover_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/6/62/BBC_News_2019.svg/768px-BBC_News_2019.svg.png'
207 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/4/41/BBC_Logo_2021.svg'
208
209 def preprocess_html(self, soup):
210 for placeholder in soup.findAll('img', attrs={'src': lambda x: x and x.endswith('placeholder.png')}):
211 placeholder.decompose()
212 for img in soup.findAll('img'):
213 img.attrs = {'src': img.get('src', '')}
214 for h2 in soup.findAll(['h2', 'h3']):
215 h2.name = 'h4'
216 return soup
217
218 recipe_specific_options = {
219 'days': {
220 'short': 'Oldest article to download from this news source. In days ',
221 'long': 'For example, 0.5, gives you articles from the past 12 hours',
222 'default': str(oldest_article),
223 }
224 }
225
226 def __init__(self, *args, **kwargs):
227 BasicNewsRecipe.__init__(self, *args, **kwargs)
228 d = self.recipe_specific_options.get('days')
229 if d and isinstance(d, str):
230 self.oldest_article = float(d)
231
232 feeds = [
233 ('Science/Environment', 'https://feeds.bbci.co.uk/news/science_and_environment/rss.xml'),
234 ('Technology', 'https://feeds.bbci.co.uk/news/technology/rss.xml'),
235 ('Entertainment/Arts', 'https://feeds.bbci.co.uk/news/entertainment_and_arts/rss.xml'),
236 ('Business', 'https://feeds.bbci.co.uk/news/business/rss.xml'),
237 ('Politics', 'https://feeds.bbci.co.uk/news/politics/rss.xml'),
238 ('Health', 'https://feeds.bbci.co.uk/news/health/rss.xml'),
239 ('US&Canada', 'https://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml'),
240 ('Latin America', 'https://feeds.bbci.co.uk/news/world/latin_america/rss.xml'),
241 ('Europe', 'https://feeds.bbci.co.uk/news/world/europe/rss.xml'),
242 ('South Asia', 'https://feeds.bbci.co.uk/news/world/south_asia/rss.xml'),
243 ('England', 'https://feeds.bbci.co.uk/news/england/rss.xml'),
244 ('Asia-Pacific', 'https://feeds.bbci.co.uk/news/world/asia_pacific/rss.xml'),
245 ('Africa', 'https://feeds.bbci.co.uk/news/world/africa/rss.xml'),
246 ('Top Stories', 'https://feeds.bbci.co.uk/news/rss.xml'),
247 ]
248
249 def preprocess_raw_html(self, raw_html, url):
250 return parse_raw_html(raw_html, self.abort_article)