Debian Codegraph @

recipes/reason_magazine.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2015, Kovid Goyal <kovid at kovidgoyal.net>
4 from __future__ import unicode_literals
5
6 import json
7
8 from calibre import prepare_string_for_xml
9 from calibre.web.feeds.news import BasicNewsRecipe, classes
10
11
12 # parse article JSON {{{
13 def process_image_block(lines, block):
14 caption = block.get('captionText')
15 caption_lines = []
16 if caption:
17 if block.get('attributionText', '').strip():
18 caption += ' (' + block['attributionText'] + ')'
19 caption_lines.append('<p style="font-style: italic">' + caption + '</p>')
20 lines.append('<div style="text-align: center"><img src={}/>'.format(prepare_string_for_xml(block['url'], True)))
21 lines.extend(caption_lines)
22 lines.append('</div>')
23
24
25 def json_to_html(raw):
26 data = json.loads(raw)
27 # open('/t/p.json', 'w').write(json.dumps(data, indent=2))
28 data = max((v['data'] for v in data['props']['pageProps']['urqlState'].values()), key=len)
29 article = json.loads(data)['article']
30 lines = []
31 lines.append('<h1 style="align: center">' + prepare_string_for_xml(article['title']) + '</h1>')
32 lines.append('<h2 style="align: center">' + prepare_string_for_xml(article['dek']) + '</h2>')
33 auts = ', '.join(x['displayName'] for x in article['authors'])
34 if auts:
35 lines.append('<p style="align: center">by ' + prepare_string_for_xml(auts) + '</p>')
36 if article.get('leadArt') and 'image' in article['leadArt']:
37 process_image_block(lines, article['leadArt']['image'])
38 for item in article['content']:
39 tn = item.get('__typename', '')
40 if tn.endswith('Image'):
41 process_image_block(lines, item)
42 continue
43 html = item.get('innerHtml')
44 if html is None or '</iframe>' in html:
45 continue
46 if 'innerHtml' not in item:
47 continue
48 tagname = item.get('tagName', 'P').lower()
49 lines.append('<{0}>{1}</{0}>'.format(tagname, html))
50 return '<html><body><div id="from-json-by-calibre">' + '\n'.join(lines) + '</div></body></html>'
51
52
53 class NoJSON(ValueError):
54 pass
55
56
57 def extract_html(soup):
58 script = soup.findAll('script', id='__NEXT_DATA__')
59 if not script:
60 raise NoJSON('No script tag with JSON data found')
61 raw = script[0].contents[0]
62 return json_to_html(raw)
63
64
65 # }}}
66
67
68 class Reason(BasicNewsRecipe):
69 title = 'Reason'
70 description = 'Free minds and free markets'
71 INDEX = 'https://reason.com/magazine/'
72 __author__ = 'Howard Cornett'
73 language = 'en'
74 encoding = 'utf-8'
75 needs_subscription = True
76
77 remove_tags = [
78 classes(
79 'next-post-link the-tags tag rcom-social-tools most-read-container comments-header-show'
80 ' logo-header navbar navbar-expanded-lg primary content-info sidebar magicSidebar advertisement logo entry-subtitle'
81 ),
82 ]
83
84 no_stylesheets = True
85 remove_attributes = ['style']
86 extra_css = '''
87 .credit { text-align: right; font-size: 75%; display: block }
88 .figcaption { font-size: 75% }
89 .caption { font-size: 75% }
90 .lead-img { display: block }
91 p.dropcap:first-letter {
92 float: left; text-transform: uppercase; font-weight: bold; font-size: 5.55em; line-height: 0.83;
93 margin: 0; padding-right: 7px; margin-bottom: -2px; text-align: center;
94 }
95 '''
96
97 def get_browser(self):
98 br = BasicNewsRecipe.get_browser(self)
99 if self.username is not None and self.password is not None:
100 br.open('https://reason.com/login')
101 br.select_form(id='gform_0')
102 br['input_1'] = self.username
103 br['input_2'] = self.password
104 br.submit()
105 return br
106
107 def preprocess_raw_html(self, raw_html, url):
108 try:
109 return extract_html(self.index_to_soup(raw_html))
110 except NoJSON:
111 self.log.warn('No JSON found in: {} falling back to HTML'.format(url))
112 except Exception:
113 self.log.exception('Failed to extract JSON data from: {} falling back to HTML'.format(url))
114 return raw_html
115
116 def preprocess_html(self, soup):
117 for img in soup.findAll('img', attrs={'data-lazy-src': True}):
118 # img['src'] = img['data-lazy-src'].split()[0]
119 data_lazy_src = img['data-lazy-src']
120 if ',' in data_lazy_src:
121 img['src'] = data_lazy_src.split(',')[0]
122 else:
123 img['src'] = data_lazy_src.split()[0]
124 return soup
125
126 def parse_index(self):
127 soup = self.index_to_soup(self.INDEX)
128 cover = soup.find('img', title=lambda value: value and value.startswith('Reason Magazine,'))
129 if cover is not None:
130 self.cover_url = cover['src']
131 current_section, current_articles = 'Cover Story', []
132 feeds = []
133 for div in soup.findAll('div', attrs={'class': lambda x: x and set(x.split()).intersection({'issue-header-right', 'toc-category-list'})}):
134 for h3 in div.findAll('h3', attrs={'class': True}):
135 cls = h3['class']
136 if hasattr(cls, 'split'):
137 cls = cls.split()
138 if 'toc-department' in cls:
139 if current_articles:
140 feeds.append((current_section, current_articles))
141 current_articles = []
142 current_section = self.tag_to_string(h3)
143 self.log('\nFound section:', current_section)
144 title = h3.find_next_sibling().a.text
145 url = h3.find_next_sibling().a['href']
146 desc = h3.find_next_sibling().p.text
147 current_articles.append({'title': title, 'url': url, 'description': desc})
148 for h2 in div.findAll('h2', attrs={'class': True}):
149 cls = h2['class']
150 if hasattr(cls, 'split'):
151 cls = cls.split()
152 if 'toc-department' in cls:
153 if current_articles:
154 feeds.append((current_section, current_articles))
155 current_articles = []
156 current_section = self.tag_to_string(h2)
157 self.log('\nFound section:', current_section)
158 for article in div.findAll('article', attrs={'class': True}):
159 h4 = article.find('h4')
160 if h4.a is not None:
161 title = h4.a.text
162 url = h4.a['href']
163 else:
164 title = ''
165 url = ''
166 desc = h4.find_next_sibling().text
167 current_articles.append({'title': title, 'url': url, 'description': desc})
168
169 if current_articles:
170 feeds.append((current_section, current_articles))
171 return feeds
172
173
174 if __name__ == '__main__':
175 import sys
176
177 from calibre.ebooks.BeautifulSoup import BeautifulSoup
178
179 print(extract_html(BeautifulSoup(open(sys.argv[-1]).read())))