Debian Codegraph @

recipes/barrons.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3
4 import base64
5 import json
6 import re
7 from collections import defaultdict
8 from datetime import datetime
9 from html import escape
10
11 from calibre.web.feeds.news import BasicNewsRecipe
12
13 keys = [
14 'ZXlKaGJHY2lPaUpTVXpJMU5pSjk=',
15 'V0Zac2FITjNNWGQzU213MFYza3dSWEJ6Y2xR',
16 'cXd3QmVkQVVOWEhUUWNob3dRWjV6TXdtblhxREtlTWhvUkpsa0I3ZHJqV21iMGt0WkNTY0locTVscElpV2FNeU5KQQ==',
17 'T0RZZ0hBZklvaTdES1drUzhnOEd1bkZOQVhwSkRVT0xkSTJydFFrVEVp',
18 'RTNvOTByZFpIdW5QUjdwMFVMalJtSENuRG9mQWRwVFFkSnRUWGpROWVFRFpUMnhvb29WR2RCcG9WS2hF',
19 ]
20
21
22 def _content_to_html(content):
23 html = ''
24 for node in content or []:
25 if not isinstance(node, dict):
26 continue
27 text = escape(node.get('text', ''))
28 if not text:
29 continue
30 if node.get('type') == 'link' and node.get('uri'):
31 html += f'<a href="{node["uri"]}">{text}</a>'
32 else:
33 html += text
34 return html
35
36
37 def _body_to_html(body):
38 html = ''
39 for item in body or []:
40 if not isinstance(item, dict):
41 continue
42 typ = item.get('type', '')
43 if typ == 'paragraph':
44 text = _content_to_html(item.get('content'))
45 if text:
46 html += f'<p>{text}</p>\n'
47 elif typ in ('heading', 'subheading'):
48 text = _content_to_html(item.get('content'))
49 if text:
50 html += f'<h3>{text}</h3>\n'
51 elif typ == 'image':
52 src = item.get('url') or item.get('imageUrl', '')
53 cap_raw = item.get('caption', '')
54 cap = _content_to_html(cap_raw) if isinstance(cap_raw, list) else escape(str(cap_raw or ''))
55 if src:
56 html += f'<figure><img src="{src}"/>'
57 if cap:
58 html += f'<figcaption>{cap}</figcaption>'
59 html += '</figure>\n'
60 elif typ == 'list':
61 tag = 'ol' if item.get('ordered') else 'ul'
62 html += f'<{tag}>\n'
63 for li in item.get('items') or item.get('content') or []:
64 if isinstance(li, dict):
65 html += f'<li>{_content_to_html(li.get("content"))}</li>\n'
66 html += f'</{tag}>\n'
67 return html
68
69
70 class Barrons(BasicNewsRecipe):
71 title = "Barron's Magazine"
72 __author__ = 'unkn0wn'
73 description = (
74 "Barron's is an American weekly magazine/newspaper published by Dow Jones & Company. Founded in 1921 as a sister "
75 "publication to The Wall Street Journal, Barron's covers U.S. financial information, market developments, and "
76 'relevant statistics.'
77 )
78 language = 'en_US'
79 encoding = 'utf-8'
80 ignore_duplicate_articles = {'url'}
81 masthead_url = 'https://www.barrons.com/asset/barrons/images/barrons-logo.png'
82 resolve_internal_links = True
83 delay = 1
84
85 extra_css = '''
86 .auth { font-size:small; }
87 .caption { font-size:small; text-align:center; }
88 img { display:block; margin:0 auto; }
89 '''
90
91 recipe_specific_options = {
92 'date': {
93 'short': 'The date of the edition to download (MMM DD, YYYY format)',
94 'long': 'For example, Dec 30, 2024',
95 },
96 }
97
98 def preprocess_raw_html(self, raw, url):
99 html = raw if isinstance(raw, str) else raw.decode('utf-8', errors='ignore')
100 nd_match = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', html, re.DOTALL)
101 if not nd_match:
102 return html
103 try:
104 j = json.loads(nd_match.group(1))
105 pp = j['props']['pageProps']
106 art = pp.get('articleData', {})
107
108 headline_text = art.get('headline', {})
109 if isinstance(headline_text, dict):
110 headline_text = headline_text.get('text', '')
111
112 byline = art.get('byline', [])
113 byline_text = ''.join(x.get('text', '') for x in byline if isinstance(x, dict)).strip()
114
115 # Subscribers get full body; non-subscribers get snippet (first para only)
116 body = art.get('body') or pp.get('snippet', [])
117
118 out = '<html><body>\n'
119 if headline_text:
120 out += f'<h1>{escape(headline_text)}</h1>\n'
121 if byline_text:
122 out += f'<div class="auth">{escape(byline_text)}</div>\n'
123 out += _body_to_html(body)
124 out += '</body></html>'
125 return out
126 except Exception:
127 return html
128
129 def parse_index(self):
130 # Step 1: Use mobile API for archive listing (cover image + edition dates)
131 index = 'https://barrons.djmedia.djservices.io'
132 theatre = '/apps/barrons/theaters/'
133 archive = self.index_to_soup(
134 index + theatre + 'magazine-archive?screen_ids=magazine-archive',
135 raw=True,
136 )
137 scrn = json.loads(archive)['screens'][0]['frames']
138 self.log(
139 'Available Editions: ',
140 ' | '.join(x['screenIds'][0]['name'][6:] for x in scrn),
141 )
142
143 selected = None
144 for frme in scrn:
145 edition_date = self.recipe_specific_options.get('date')
146 if edition_date and isinstance(edition_date, str):
147 nme = frme['screenIds'][0]['name']
148 if edition_date.lower() not in nme.lower():
149 continue
150 selected = frme
151 break
152
153 if selected is None:
154 selected = scrn[0]
155
156 nme = selected['screenIds'][0]['name']
157 self.cover_url = selected['image']['url']
158 self.log('Downloading ', nme)
159 self.timefmt = ' [' + nme[6:] + ']'
160
161 # Convert mobile API date to website URL
162 date_text = selected['date']['text'] # e.g. "2026-08-31T04:00:00Z"
163 dt = datetime.strptime(date_text[:10], '%Y-%m-%d')
164 date_str = dt.strftime('%Y%m%d')
165
166 # Step 2: Fetch article list from website __NEXT_DATA__
167 web_url = 'https://www.barrons.com/magazine/' + date_str
168 raw_html = self.index_to_soup(web_url, raw=True).decode('utf-8', errors='ignore')
169 nd_match = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', raw_html, re.DOTALL)
170 if not nd_match:
171 raise ValueError('Could not find article data in Barrons magazine page')
172
173 mag_data = json.loads(nd_match.group(1))['props']['pageProps']['magazineData']
174
175 sections_map = [
176 ('coverArticle', 'Cover Story'),
177 ('featureArticles', 'Features'),
178 ('columnsArticles', 'Columns'),
179 ('departmentsArticles', 'Departments'),
180 ('restArticles', 'More'),
181 ]
182
183 feeds_dict = defaultdict(list)
184 for key, section_name in sections_map:
185 for art in mag_data.get(key, []):
186 if not isinstance(art, dict):
187 continue
188 url = art.get('url', '')
189 if 'barrons.com/articles/' not in url:
190 continue
191 url = url.split('?')[0]
192 title = art.get('headline', '')
193 if not title or not url:
194 continue
195 desc = art.get('summary', '') or ''
196 self.log(' ', title, '\n\t', desc)
197 feeds_dict[section_name].append({'title': title, 'url': url, 'description': desc})
198
199 return list(feeds_dict.items())
200
201 def get_browser(self, *args, **kw):
202 kw['user_agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
203 br = BasicNewsRecipe.get_browser(self, *args, **kw)
204 k2 = '.'.join(base64.b64decode(b).decode() for b in keys[:3])
205 k3 = '_'.join(base64.b64decode(v).decode() for v in keys[3:])
206 br.addheaders += [
207 ('Accept-Encoding', 'gzip'),
208 ('Accept-Language', 'en-US,en;q=0.9'),
209 ('Sec-Ch-Ua', '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"'),
210 ('Sec-Ch-Ua-Mobile', '?0'),
211 ('Sec-Ch-Ua-Platform', '"Windows"'),
212 ('Sec-Fetch-Dest', 'document'),
213 ('Sec-Fetch-Mode', 'navigate'),
214 ('Sec-Fetch-Site', 'none'),
215 ('Sec-Fetch-User', '?1'),
216 # Mobile API token (used only for the archive listing)
217 ('App-Identifier', 'com.news.screens'),
218 ('App-Version', '1'),
219 ('Device-Type', 'phone'),
220 ('Os-Name', 'Android'),
221 ('X-Access-Token', k2 + '_' + k3),
222 ]
223 return br
224
225 def populate_article_metadata(self, article, soup, first):
226 if '/articles/' not in article.url:
227 article.url = 'https://www.barrons.com/articles/' + article.url.split('=')[-1]