Debian Codegraph @

recipes/reuters.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 import time
5 from datetime import datetime, timedelta
6
7 from calibre.web.feeds.news import BasicNewsRecipe
8
9
10 def p_dt(x):
11 dt = datetime.fromisoformat(x[:-1]) + timedelta(seconds=time.timezone)
12 return dt.strftime('%b %d, %Y, %I:%M %p')
13
14
15 class Reuters(BasicNewsRecipe):
16 title = 'Reuters'
17 __author__ = 'unkn0wn'
18 description = (
19 'Reuters, the news and media division of Thomson Reuters, is the world’s largest multimedia news provider, '
20 'reaching billions of people worldwide every day. Reuters provides business, financial, national and international '
21 'news to professionals via desktop terminals, the world’s media organizations, industry events and directly to consumers.'
22 )
23 masthead_url = 'https://upload.wikimedia.org/wikipedia/commons/9/9e/Reuters_logo_2024.svg'
24 cover_url = 'https://yt3.googleusercontent.com/ytc/AIdro_mk43b9eQwN15ZBDyMPDaElxvw4V-oUS9XDUvVnYB3gA9yA=s1024'
25 language = 'en'
26 encoding = 'utf-8'
27 oldest_article = 1.2 # days
28 no_javascript = True
29 no_stylesheets = True
30 remove_attributes = ['style', 'height', 'width']
31 resolve_internal_links = True
32 ignore_duplicate_articles = {'url'}
33 remove_empty_feeds = True
34
35 extra_css = '''
36 .label, .auth { font-size:small; color:#202020; }
37 .desc { font-style: italic; }
38 .figc { font-size:small; }
39 img {display:block; margin:0 auto;}
40 '''
41
42 recipe_specific_options = {
43 'days': {
44 'short': 'Oldest article to download from this news source. In days ',
45 'long': 'For example, 0.5, gives you articles from the past 12 hours',
46 'default': str(oldest_article),
47 },
48 'res': {
49 'short': 'For hi-res images, select a resolution from the\nfollowing options: 960, 1080, 1200',
50 'long': 'This is useful for non e-ink devices',
51 'default': '480',
52 },
53 'spr': {
54 'short': 'Include Sports sections?',
55 'long': 'Yes/No',
56 'default': 'No',
57 },
58 }
59
60 def __init__(self, *args, **kwargs):
61 BasicNewsRecipe.__init__(self, *args, **kwargs)
62 d = self.recipe_specific_options.get('days')
63 if d and isinstance(d, str):
64 self.oldest_article = float(d)
65
66 def parse_index(self):
67 index = 'https://www.reuters.com'
68 today = datetime.now()
69
70 sections = []
71
72 sec_api = json.loads(self.index_to_soup(index + '/mobile/api/v1/menu/?outputType=json', raw=True))
73
74 for s in sec_api[0]['data']['hierarchy']['children']:
75 if s.get('type', '') == 'section':
76 sections.append((s['name'], s['id']))
77 sections.extend((s['name'] + ' - ' + s2['name'], s2['id']) for s2 in s.get('children', []) if s2.get('type', '') == 'section')
78
79 feeds = []
80
81 for sec, link in sections:
82 sp = self.recipe_specific_options.get('spr')
83 if sp and isinstance(sp, str):
84 if sp.lower().strip() != 'yes':
85 if sec.lower().startswith('sport'):
86 continue
87
88 self.log(sec)
89
90 articles = []
91
92 data = json.loads(self.index_to_soup(index + '/mobile/v1' + link + '?outputType=json', raw=True))
93
94 for st in (story for x in data if isinstance(x, dict) for story in x.get('data', {}).get('stories', [])):
95 title = st['title']
96
97 date = datetime.fromisoformat(st['display_time'][:-1]) + timedelta(seconds=time.timezone)
98 if (today - date) > timedelta(self.oldest_article):
99 continue
100
101 desc = st['description']
102 url = index + st['url']
103 self.log(' ', title, '\n\t', desc, '\n\t', url)
104 articles.append({'title': title, 'description': desc, 'url': url})
105 if articles:
106 feeds.append((sec, articles))
107 return feeds
108
109 def preprocess_raw_html(self, raw, url):
110 res = '&width=480'
111 w = self.recipe_specific_options.get('res')
112 if w and isinstance(w, str):
113 res = '&width=' + w
114
115 body = ''
116
117 for det in json.loads(raw):
118 if not det.get('type', '') == 'article_detail':
119 continue
120 data = det['data']['article']
121 body += '<h1>' + data['title'] + '</h1>'
122 if data.get('description'):
123 body += '<p class="desc">' + data['description'] + '</p>'
124 if data.get('authors'):
125 body += '<p class="auth">' + 'By ' + ', '.join(at.get('byline', '') for at in data.get('authors', [])) + '</p>'
126
127 if data.get('thumbnail') and data['thumbnail'].get('type', '') == 'image':
128 th = data['thumbnail']
129 body += '<img src="{}"><div class="figc">{}</div>'.format(
130 th['resizer_url'].split('&')[0] + res,
131 th.get('caption', ''),
132 )
133
134 body += (
135 '<p class="auth">'
136 + str(data.get('read_minutes', '_'))
137 + ' minute read | '
138 + str(data['word_count'])
139 + ' words | '
140 + p_dt(data['updated_time'] if data.get('updated_time') else data['display_time'])
141 + '</p>'
142 )
143
144 if data.get('summary'):
145 body += '<blockquote>' + ''.join(f'<li>{su["description"]}</li>' for su in data['summary']) + '</blockquote>'
146
147 for y in data['content_elements']:
148 ty = y.get('type', '')
149 if ty == 'placeholder':
150 continue
151
152 elif ty == 'paragraph':
153 body += '<p>' + y['content'] + '</p>'
154 elif ty == 'header':
155 body += '<h4>' + y['content'] + '</h4>'
156 elif ty == 'graphic':
157 body += '<img src="{}"><div class="figc">{}</div>'.format(
158 y['resizer_url'].split('&')[0] + res,
159 y.get('description', ''),
160 )
161 else:
162 self.log('**', ty)
163
164 if data.get('sign_off'):
165 body += '<p class="auth">' + data['sign_off'] + '</p>'
166
167 return '<html><body><div>' + body + '</div></body></html>'
168
169 def get_browser(self, *args, **kwargs):
170 kwargs['user_agent'] = (
171 'ReutersNews/7.11.0.1742843009 Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.165 Mobile Safari/537.36'
172 )
173 br = BasicNewsRecipe.get_browser(self, *args, **kwargs)
174 br.addheaders += [('cookie', 'reuters-geo={"country":"-"; "region":"-"}=')]
175 return br
176
177 def print_version(self, url):
178 return url.replace('https://www.reuters.com', 'https://www.reuters.com/mobile/v1') + '?outputType=json'