Debian Codegraph @

recipes/wsj_free.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 import time
5 from urllib.parse import quote, urlencode
6
7 from calibre.ebooks.BeautifulSoup import BeautifulSoup
8 from calibre.web.feeds.news import BasicNewsRecipe
9
10
11 class WSJ(BasicNewsRecipe):
12 title = 'The Wall Street Journal'
13 __author__ = 'unkn0wn'
14 description = (
15 'The Print Edition of WSJ. The Wall Street Journal is your source '
16 'for breaking news, analysis and insights from the U.S. and '
17 "around the world, the world's leading business and finance publication."
18 )
19 language = 'en_US'
20 masthead_url = 'https://s.wsj.net/media/wsj_amp_masthead_lg.png'
21 encoding = 'utf-8'
22 no_javascript = True
23 no_stylesheets = True
24 remove_attributes = ['style', 'height', 'width']
25 resolve_internal_links = True
26
27 recipe_specific_options = {
28 'date': {
29 'short': 'The date of the edition to download (YYYY-MM-DD format)\nOnly the past 6 editions will be available ',
30 'long': 'For example, 2024-05-13',
31 },
32 'res': {
33 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
34 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
35 'default': '600',
36 },
37 }
38
39 extra_css = '''
40 #big-top-caption { font-size:small; text-align:center; }
41 [data-type:"tagline"] { font-style:italic; color:#202020; }
42 .auth, time { font-size:small; }
43 .sub, em, i { color: #202020; }
44 '''
45
46 keep_only_tags = [
47 dict(name=['h1', 'h2']),
48 dict(attrs={'aria-describedby': 'big-top-caption'}),
49 dict(attrs={'id': 'big-top-caption'}),
50 dict(name='article', attrs={'style': lambda x: x and 'article-body' in x}),
51 ]
52
53 remove_tags = [
54 dict(attrs={'data-type': ['inset', 'video']}),
55 dict(attrs={'data-testid': 'ad-container'}),
56 dict(attrs={'data-spotim-app': 'conversation'}),
57 dict(name=['button', 'svg', 'old-script', 'video']),
58 dict(
59 attrs={
60 'aria-label': [
61 'Sponsored Offers',
62 'Listen To Article',
63 'What to Read Next',
64 'Utility Bar',
65 'Conversation',
66 'List of Comments',
67 'Comment',
68 'JR More Articles',
69 ]
70 }
71 ),
72 dict(
73 attrs={
74 'data-spot-im-class': [
75 'message-text',
76 'messages-list',
77 'message-view',
78 'conversation-root',
79 ]
80 }
81 ),
82 dict(
83 attrs={
84 'id': lambda x: (
85 x
86 and x.startswith((
87 'comments_sector',
88 'wrapper-INLINE',
89 'audio-tag-inner-audio-',
90 'article-comments-tool',
91 ))
92 )
93 }
94 ),
95 dict(name='div', attrs={'data-message-depth': True}),
96 ]
97
98 articles_are_obfuscated = True
99
100 def get_obfuscated_article(self, url):
101 from calibre.scraper.simple import read_url
102
103 raw = read_url([], 'https://archive.is/latest/' + url)
104 return {'data': raw, 'url': url}
105
106 def preprocess_html(self, soup):
107 res = '?width=600'
108 w = self.recipe_specific_options.get('res')
109 if w and isinstance(w, str):
110 res = '?width=' + w
111 for img in soup.findAll('img', attrs={'currentsourceurl': True}):
112 img['src'] = img['currentsourceurl'].split('?')[0] + res
113 for p in soup.findAll('div', attrs={'data-type': ['paragraph', 'image']}):
114 p.name = 'p'
115 for a in soup.findAll('a', href=True):
116 a['href'] = 'http' + a['href'].split('http')[-1]
117 for figc in soup.findAll('figcaption'):
118 figc['id'] = 'big-top-caption'
119 if name := soup.find('h2', attrs={'itemprop': 'name'}):
120 name.extract()
121 for h2 in soup.findAll('h2'):
122 if self.tag_to_string(h2).startswith(('What to Read Next', 'Conversation')):
123 h2.extract()
124 h2.name = 'h3'
125 h2['class'] = 'sub'
126 for ph in soup.findAll('a', attrs={'data-type': ['phrase', 'link']}):
127 if div := ph.findParent('div'):
128 div.name = 'span'
129 for auth in soup.findAll('a', attrs={'aria-label': lambda x: x and x.startswith('Author page')}):
130 if div := auth.find_previous_sibling('div'):
131 div.name = 'span'
132 if parent := auth.findParent('div'):
133 parent['class'] = 'auth'
134 for x in soup.findAll('ufc-follow-author-widget'):
135 if y := x.findParent('div'):
136 y.extract()
137 return soup
138
139 def get_browser(self, *args, **kw):
140 br = BasicNewsRecipe.get_browser(self, *args, **kw)
141 br.addheaders += [
142 ('apollographql-client-name', 'wsj-mobile-android-release'),
143 ]
144 return br
145
146 def _download_cover(self):
147 import os
148 from contextlib import closing
149
150 from calibre import browser
151 from calibre.utils.img import save_cover_data_to
152
153 br = browser()
154 raw = br.open('https://frontpages.freedomforum.org/newspapers/wsj-The_Wall_Street_Journal')
155 soup = BeautifulSoup(raw.read())
156 cu = soup.find(
157 'img',
158 attrs={
159 'alt': 'Front Page Image',
160 'src': lambda x: x and x.endswith('front-page-large.jpg'),
161 },
162 )['src'].replace('-large', '-medium')
163 self.report_progress(1, _('Downloading cover from %s') % cu)
164 with closing(br.open(cu, timeout=self.timeout)) as r:
165 cdata = r.read()
166 cpath = os.path.join(self.output_dir, 'cover.jpg')
167 save_cover_data_to(cdata, cpath)
168 self.cover_path = cpath
169
170 def parse_index(self):
171 query = {
172 'operationName': 'IssueQuery',
173 'variables': '{"publication":"WSJ","region":"US","masthead":"ITPNEXTGEN"}',
174 'extensions': '{"persistedQuery":{"version":1,"sha256Hash":"d938226e7d1c1fff050e7d084c72179e2713dcf4736d3a442c618c55b896f847"}}',
175 }
176 url = 'https://shared-data.dowjones.io/gateway/graphql?' + urlencode(query, safe='()!', quote_via=quote)
177 raw = self.index_to_soup(url, raw=True)
178
179 cat_data = json.loads(raw)['data']['mobileIssuesByMasthead']
180 edit = [x['datedLabel'] for x in cat_data][1:]
181 self.log('**Past Editions available : ' + ' | '.join(edit))
182
183 past_edition = self.recipe_specific_options.get('date')
184
185 for itm in cat_data:
186 if past_edition and isinstance(past_edition, str):
187 if past_edition in itm['publishedDateUtc']:
188 self.timefmt = ' [' + itm['datedLabel']
189 sections_ = itm['sections']
190 break
191 self.timefmt = f' [{itm["datedLabel"]}]'
192 sections_ = itm['sections']
193 break
194
195 self.log('Downloading ', self.timefmt)
196
197 feeds = []
198
199 for sec in sections_[:-1]:
200 time.sleep(3)
201 section = sec['label']
202 self.log(section)
203 cont_id = sec['key']
204
205 query = {
206 'operationName': 'SectionQuery',
207 'variables': '{{"id":"{}"}}'.format(cont_id),
208 'extensions': '{"persistedQuery":{"version":1,"sha256Hash":"207fe93376f379bf223ed2734cf9313a28291293366a803db923666fa6b45026"}}',
209 }
210 sec_url = 'https://shared-data.dowjones.io/gateway/graphql?' + urlencode(query, safe='()!', quote_via=quote)
211 sec_raw = self.index_to_soup(sec_url, raw=True)
212
213 sec_data = json.loads(sec_raw)['data']['summaryCollectionContent']['collectionItems']
214
215 articles = []
216
217 for art in sec_data:
218 for arts in art['collectionItems']:
219 mobi = arts['content']['mobileSummary']
220 title = mobi['headline']['text']
221 try:
222 desc = mobi['description']['content']['text']
223 except TypeError:
224 desc = ''
225 url = arts['content']['sourceUrl']
226 self.log(' ', title, '\n\t', desc, '\n\t', url)
227 articles.append({'title': title, 'description': desc, 'url': url})
228 feeds.append((section, articles))
229 return feeds