Debian Codegraph @

recipes/bloomberg.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3
4 import json
5 import time
6 from datetime import datetime, timedelta
7
8 from calibre.ebooks.BeautifulSoup import BeautifulSoup
9 from calibre.web.feeds.news import BasicNewsRecipe, classes
10
11
12 def get_contents(x):
13 if x == '':
14 return ''
15 otype = x.get('role', '')
16 if otype == 'p':
17 return '<p>' + ''.join(map(get_contents, x.get('parts', ''))) + '</p>'
18 elif otype == 'text':
19 if 'style' in x:
20 return '<' + x['style'] + '>' + ''.join(map(get_contents, x.get('parts', ''))) + '</' + x['style'] + '>'
21 return x.get('text', '') + ''.join(map(get_contents, x.get('parts', '')))
22 elif otype == 'br':
23 return '<br>'
24 elif otype == 'anchor':
25 return '<span>' + ''.join(map(get_contents, x.get('parts', ''))) + '</span>'
26 elif otype == 'h3':
27 return '<h4>' + ''.join(map(get_contents, x.get('parts', ''))) + '</h4>'
28 elif otype == 'ul':
29 return '<ul>' + ''.join(map(get_contents, x.get('parts', ''))) + '</ul>'
30 elif otype == 'li':
31 return '<li>' + ''.join(map(get_contents, x.get('parts', ''))) + '</li>'
32 elif otype == 'webview':
33 return '<br>' + x['html'] + ''.join(map(get_contents, x.get('parts', '')))
34 elif otype == 'blockquote':
35 return '<blockquote>' + ''.join(map(get_contents, x.get('parts', ''))) + '</blockquote>'
36 elif otype in {'image', 'video'}:
37 return '<br><img src="{}"><div class="img">{}</div>\n'.format(x['imageURLs']['default'], x['caption'] + '<i> ' + x['credit'] + '</i>')
38 elif otype in {'correction', 'disclaimer'}:
39 return '<p class="corr">' + ''.join(map(get_contents, x.get('parts', ''))) + '</p>'
40
41 elif not any(x == otype for x in ['', 'ad', 'inline-newsletter', 'tabularData']):
42 return '<i>' + ''.join(map(get_contents, x.get('parts', ''))) + '</i>'
43 return ''
44
45
46 class Bloomberg(BasicNewsRecipe):
47 title = 'Bloomberg'
48 language = 'en_US'
49 __author__ = 'unkn0wn'
50 no_stylesheets = True
51 remove_attributes = ['style', 'height', 'width']
52 encoding = 'utf-8'
53 ignore_duplicate_articles = {'url', 'title'}
54 masthead_url = 'https://assets.bbhub.io/company/sites/70/2022/09/logoBBGblck.svg'
55 description = (
56 'Bloomberg delivers business and markets news, data, analysis, and video to the world, featuring stories from Businessweek and Bloomberg News.'
57 )
58 oldest_article = 1.2 # days
59 resolve_internal_links = True
60 remove_empty_feeds = True
61 cover_url = 'https://assets.bwbx.io/images/users/iqjWHBFdfxIU/ivUxvlPidC3M/v0/600x-1.jpg'
62
63 remove_tags = [
64 dict(name=['button', 'svg', 'meta', 'iframe']),
65 dict(name='div', attrs={'id': ['bb-that', 'bb-nav']}),
66 dict(attrs={'data-image-type': 'audio'}),
67 classes('twitter-logo bb-global-footer __sticky__audio__bar__portal__ css--social-wrapper-outer bplayer-container'),
68 ]
69
70 extra_css = '''
71 .auth { font-size:small; font-weight:bold; }
72 .subhead, .cap span { font-style:italic; color:#202020; }
73 em, blockquote { color:#202020; }
74 .cat { font-size:small; color:gray; }
75 .img, .news-figure-caption-text { font-size:small; text-align:center; }
76 .corr { font-size:small; font-style:italic; color:#404040; }
77 .chart { font-size:small; }
78 .news-figure-credit {font-size:small; text-align:center; color:#202020;}
79 '''
80
81 recipe_specific_options = {
82 'days': {
83 'short': 'Oldest article to download from this news source. In days ',
84 'long': 'For example, 0.5, gives you articles for the past 12 hours',
85 'default': str(oldest_article),
86 }
87 }
88
89 def parse_index(self):
90 d = self.recipe_specific_options.get('days')
91 if d and isinstance(d, str):
92 self.oldest_article = float(d)
93 inx = 'https://cdn-mobapi.bloomberg.com'
94 sec = self.index_to_soup(inx + '/wssmobile/v1/navigation/bloomberg_app/search-v2', raw=True)
95 sec_data = json.loads(sec)['searchNav']
96
97 feeds = []
98
99 for i in sec_data:
100 for sects in i['items']:
101 section = sects['title']
102 sec_slug = sects['links']['self']['href']
103 self.log(section)
104
105 articles = []
106
107 art_soup = self.index_to_soup(inx + sec_slug, raw=True)
108 for arts in json.loads(art_soup)['modules']:
109 if arts['stories']:
110 for x in arts['stories']:
111 if x.get('type', '') in {'article', 'interactive'}:
112 dt = datetime.fromtimestamp(x['published'] + time.timezone)
113 if (datetime.now() - dt) > timedelta(self.oldest_article):
114 continue
115 title = x['title']
116 desc = x['autoGeneratedSummary']
117 url = inx + '/wssmobile/v1/stories/' + x['internalID']
118 self.log(' ', title, '\n\t', desc)
119 articles.append({'title': title, 'description': desc, 'url': url})
120 feeds.append((section, articles))
121 return feeds
122
123 # https://cdn-mobapi.bloomberg.com/wssmobile/v1/urllookup/find?variant=android&url=https://www.bloomberg.com/news/articles/'---'?utm_medium=deeplink
124
125 def preprocess_raw_html(self, raw, url):
126 data = json.loads(raw)
127
128 title = '<h1 title="{}">'.format(data['longURL']) + data['title'] + '</h1>'
129
130 cat = subhead = lede = auth = caption = ''
131
132 if 'primaryCategory' in data and data['primaryCategory'] is not None:
133 cat = '<div class="cat">' + data['primaryCategory'] + '</div>'
134
135 if 'abstract' in data and data['abstract'] and data['abstract'] is not None:
136 subhead = '<div class="subhead"><ul><li>' + '</li><li>'.join(list(data['abstract'])) + '</li></ul></div>'
137 elif data.get('summary'):
138 subhead = '<div class="subhead"><p>' + data['summary'] + '</p></div>'
139
140 if 'byline' in data and data['byline'] is not None:
141 dt = datetime.fromtimestamp(data['updatedAt'] + time.timezone)
142 auth = '<p class="auth">' + 'By ' + data['byline'] + ' | Updated on ' + dt.strftime('%b %d, %Y at %I:%M %p') + '</p>'
143
144 body = ''
145 if data.get('type', '') == 'interactive':
146 body += '<p><em>' + 'This is an interactive article, which is supposed to be read in a browser.' + '</p></em>'
147 # body_data = data['components']
148 # for x in body_data:
149 # body += get_contents(x)
150
151 b_data = self.index_to_soup('https://cdn-mobapi.bloomberg.com/wssmobile/v1/bw/news/stories/' + url.split('/')[-1], raw=True)
152 body += json.loads(b_data)['html']
153
154 if 'ledeImage' in data and data['ledeImage'] is not None:
155 x = data['ledeImage']
156 if x['imageURLs']['default'].rsplit('/', 1)[0] not in body:
157 lede = '<br><img src="{}"><div class="img">{}</div>\n'.format(x['imageURLs']['default'], x['caption'] + '<i> ' + x['credit'] + '</i>')
158
159 html = '<html><body>' + cat + title + subhead + auth + lede + caption + '<div>' + body + '</div></body></html>'
160 return BeautifulSoup(html).prettify()
161
162 def preprocess_html(self, soup):
163 for h3 in soup.findAll(['h2', 'h3']):
164 h3.name = 'h4'
165 for icon in soup.findAll('img', attrs={'class': 'video-player__play-icon'}):
166 icon.decompose()
167 for div in soup.findAll('div', attrs={'class': 'chart'}):
168 nos = div.find('noscript')
169 if nos:
170 nos.name = 'span'
171 for img in soup.findAll('img', attrs={'data-native-src': True}):
172 if img['data-native-src'].__contains__('videos') is False:
173 img['src'] = img['data-native-src']
174 else:
175 img['src'] = ''
176 for img in soup.findAll('img', attrs={'src': lambda x: x and x.endswith(('-1x-1.jpg', '-1x-1.png'))}):
177 img['src'] = img['src'].replace('-1x-1', '750x-1')
178 return soup
179
180 def get_browser(self, *args, **kwargs):
181 kwargs['user_agent'] = 'BloombergHorseshoe|Android|16|6.66.0'
182 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
183 return br
184
185 def populate_article_metadata(self, article, soup, first):
186 article.url = soup.find('h1')['title']