Debian Codegraph @

recipes/wsj_mag.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 from itertools import zip_longest
5 from urllib.parse import quote, urlencode
6
7 from calibre import browser
8 from calibre.ebooks.BeautifulSoup import BeautifulSoup
9 from calibre.ptempfile import PersistentTemporaryFile
10 from calibre.web.feeds.news import BasicNewsRecipe, classes
11
12
13 def get_article(article_id):
14 from mechanize import Request
15
16 mat_url = 'https://mats.mobile.dowjones.io/translate/' + article_id + '/jpml'
17 headers = {
18 'User-Agent': 'okhttp/4.10.0',
19 'Accept-Encoding': 'gzip',
20 'Cache-Control': 'no-cache',
21 'x-api-key': ('e05995ff442143255eb8381f72d4913bf7503d6c'), # noqa: ISC001
22 }
23 br = browser()
24 req = Request(
25 mat_url,
26 headers=headers,
27 )
28 res = br.open(req)
29 return res.read()
30
31
32 class WSJ(BasicNewsRecipe):
33 title = 'WSJ. Magazine'
34 __author__ = 'unkn0wn'
35 description = (
36 'Eight times a year the print edition of WSJ. Magazine covers contemporary culture '
37 'and the luminaries shaping it—with exclusive features and award-winning photography '
38 'documenting the worlds of entertainment, fashion, design, art, food, travel and more.'
39 )
40 language = 'en_US'
41 encoding = 'utf-8'
42 no_javascript = True
43 no_stylesheets = True
44 remove_attributes = ['style', 'height', 'width']
45 resolve_internal_links = True
46 simultaneous_downloads = 20
47
48 recipe_specific_options = {
49 'res': {
50 'short': 'For hi-res images, select a resolution from the\nfollowing options: 800, 1000, 1200 or 1500',
51 'long': 'This is useful for non e-ink devices, and for a lower file size\nthan the default, use 400 or 300.',
52 'default': '600',
53 },
54 }
55
56 extra_css = '''
57 #subhed, em { font-style:italic; color:#202020; }
58 #byline, #time-to-read, #orig-pubdate-string, .article-byline, time, #flashline { font-size:small; }
59 .figc { font-size:small; text-align:center; }
60 img {display:block; margin:0 auto;}
61 '''
62
63 remove_tags = [
64 dict(name='panel', attrs={'id': 'summary-image'}),
65 dict(name='panel', attrs={'layout': 'inline'}),
66 dict(name='panel', attrs={'embed': 'inner-article-ad'}),
67 dict(name='span', attrs={'embed': 'ticker'}),
68 classes('lamrelated-articles-inset-panel'),
69 dict(
70 name='p',
71 attrs={
72 'id': [
73 'keywords',
74 'orig-pubdate-number',
75 'type',
76 'is-custom-flashline',
77 'grouphed',
78 'author-ids',
79 'article-manifest',
80 'body-extract',
81 'category',
82 'sub-category',
83 'socialhed',
84 'summary',
85 'deckline',
86 'article-flashline',
87 ]
88 },
89 ),
90 ]
91
92 remove_tags_before = [dict(name='p', attrs={'id': 'orig-pubdate-string'})]
93
94 def media_bucket(self, x):
95 res = '?width=600'
96 w = self.recipe_specific_options.get('res')
97 if w and isinstance(w, str):
98 res = '?width=' + w
99 if x.get('type', '') == 'image':
100 if x.get('subtype', '') == 'graphic' or 'images.wsj.net' not in x['manifest-url']:
101 return '<br><img src="{}"><div class="figc">{}</div>\n'.format(x['manifest-url'], x['caption'] + '<i> ' + x['credit'] + '</i>')
102 return '<br><img src="{}"><div class="figc">{}</div>\n'.format(
103 x['manifest-url'].split('?')[0] + res,
104 x['caption'] + '<i> ' + x['credit'] + '</i>',
105 )
106 if x.get('type', '') == 'video':
107 return '<br><a href="{}"><img src="{}"></a><div class="figc">{}</div>\n'.format(
108 x['share_link'],
109 x['thumbnail_url'].split('?')[0] + res,
110 x['caption'] + '<i> ' + x['credit'] + '</i>',
111 )
112 return
113
114 def preprocess_html(self, soup):
115 jpml = soup.find('jpml')
116 if jpml:
117 jpml.name = 'article'
118 h1 = soup.find('p', attrs={'id': 'headline'})
119 if h1:
120 h1.name = 'h1'
121 for h2 in soup.findAll('h2'):
122 h2.name = 'h4'
123 dt = soup.find('p', attrs={'id': 'orig-pubdate-string'})
124 read = soup.find('p', attrs={'id': 'time-to-read'})
125 byl = soup.find('p', attrs={'id': 'byline'})
126 fl = soup.find('p', attrs={'id': 'flashline'})
127 if dt and byl and read and fl:
128 dt.name = read.name = byl.name = fl.name = 'div'
129 byl.insert(0, dt)
130 byl.insert(0, read)
131 url = soup.find('p', attrs={'id': 'share-link'})
132 if url:
133 url.name = 'div'
134 url['title'] = self.tag_to_string(url).strip()
135 url.string = ''
136 panel = soup.find('panel', attrs={'id': 'metadata'})
137 if panel:
138 buck = panel.find('p', attrs={'id': 'media-bucket'})
139 if buck:
140 data = json.loads(buck.string)
141 buck.extract()
142 i_lst = [self.media_bucket(x) for x in data['items']]
143 m_itm = soup.findAll('panel', attrs={'class': 'media-item'})
144 if i_lst and m_itm:
145 for x, y in list(zip_longest(m_itm, i_lst)):
146 x.insert_after(BeautifulSoup(y, 'html.parser'))
147 return soup
148
149 def postprocess_html(self, soup, first_fetch):
150 for pan in soup.findAll('panel'):
151 pan.name = 'div'
152 return soup
153
154 def _download_cover(self):
155 import os
156 from contextlib import closing
157
158 from calibre.utils.img import save_cover_data_to
159
160 br = browser()
161 raw = br.open('https://wsjtodaysedition.cmail19.com/t/d-e-suujjg-thithlkhlr-r/')
162 soup = BeautifulSoup(raw.read())
163 cu = soup.find('img', attrs={'class': 'responsive-img', 'src': lambda x: x and 'WSJMagazine' in x})['src']
164 self.report_progress(1, _('Downloading cover from %s') % cu)
165 with closing(br.open(cu, timeout=self.timeout)) as r:
166 cdata = r.read()
167 cpath = os.path.join(self.output_dir, 'cover.jpg')
168 save_cover_data_to(cdata, cpath)
169 self.cover_path = cpath
170
171 def get_browser(self, *args, **kw):
172 br = BasicNewsRecipe.get_browser(self, *args, **kw)
173 br.addheaders += [
174 ('apollographql-client-name', 'wsj-mobile-android-release'),
175 ]
176 return br
177
178 def parse_index(self):
179 query = {
180 'operationName': 'IssueQuery',
181 'variables': '{"publication":"WSJ","region":"US","masthead":"ITPNEXTGEN"}',
182 'extensions': '{"persistedQuery":{"version":1,"sha256Hash":"d938226e7d1c1fff050e7d084c72179e2713dcf4736d3a442c618c55b896f847"}}',
183 }
184 url = 'https://shared-data.dowjones.io/gateway/graphql?' + urlencode(query, safe='()!', quote_via=quote)
185 raw = self.index_to_soup(url, raw=True)
186
187 cat_data = json.loads(raw)['data']['mobileIssuesByMasthead']
188
189 for itm in cat_data:
190 sections_ = itm['sections']
191 break
192
193 feeds = []
194
195 sec = sections_[-1]
196 section = sec['label']
197 self.log(section)
198 cont_id = sec['key']
199
200 query = {
201 'operationName': 'SectionQuery',
202 'variables': '{{"id":"{}"}}'.format(cont_id),
203 'extensions': '{"persistedQuery":{"version":1,"sha256Hash":"207fe93376f379bf223ed2734cf9313a28291293366a803db923666fa6b45026"}}',
204 }
205 sec_url = 'https://shared-data.dowjones.io/gateway/graphql?' + urlencode(query, safe='()!', quote_via=quote)
206 sec_raw = self.index_to_soup(sec_url, raw=True)
207
208 sec_data = json.loads(sec_raw)['data']['summaryCollectionContent']['collectionItems']
209
210 articles = []
211
212 for art in sec_data:
213 for arts in art['collectionItems']:
214 mobi = arts['content']['mobileSummary']
215 title = mobi['headline']['text']
216 desc = mobi['description']['content']['text']
217 art_id = arts['id']
218 self.log(' ', title, '\n\t', desc)
219 art_cont = get_article(art_id)
220 pt = PersistentTemporaryFile('.html')
221 pt.write(art_cont)
222 pt.close()
223 url = 'file:///' + pt.name
224 articles.append({'title': title, 'description': desc, 'url': url})
225 feeds.append((section, articles))
226 return feeds
227
228 def populate_article_metadata(self, article, soup, first):
229 lnk = soup.find('div', attrs={'id': 'share-link'})
230 if lnk:
231 article.url = lnk['title']