Debian Codegraph @

recipes/hbr.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 import json
4 import re
5 from collections import OrderedDict
6 from urllib.parse import urlparse
7
8 from html5_parser import parse
9 from mechanize import Request
10
11 from calibre import browser, random_user_agent
12 from calibre.web.feeds.news import BasicNewsRecipe, classes
13
14
15 def absurl(url):
16 if url.startswith('/'):
17 url = 'https://www.hbr.org' + url
18 return url
19
20
21 class HBR(BasicNewsRecipe):
22 title = 'Harvard Business Review'
23 __author__ = 'unkn0wn'
24 description = (
25 'Harvard Business Review is the leading destination for smart management thinking. '
26 'Through its flagship magazine, books, and digital content and tools published on HBR.org, '
27 'Harvard Business Review aims to provide professionals around the world with rigorous insights '
28 'and best practices to help lead themselves and their organizations more effectively and to '
29 'make a positive impact.'
30 )
31 language = 'en'
32 masthead_url = 'https://hbr.org/resources/css/images/hbr_logo.svg'
33 publication_type = 'magazine'
34 encoding = 'utf-8'
35 remove_javascript = True
36 no_stylesheets = True
37 compress_news_images = True
38 ignore_duplicate_articles = {'url'}
39 base_url = 'https://hbr.org'
40 remove_attributes = ['height', 'width', 'style']
41 resolve_internal_links = True
42
43 extra_css = '''
44 .article-summary, .article-ideainbrief,
45 .description-text, .link--black, .topic, .auth {font-size:small; color:#202020;}
46 .credits--hero-image, .credits--inline-image, .caption--inline-image,
47 .calibre-nuked-tag-figcaption {font-size:small; text-align:center;}
48 .sub { font-style: italic; }
49 .article-byline-list {font-size:small; font-weight:bold;}
50 .question {font-weight:bold;}
51 .right-rail--container {font-size:small; color:#404040;}
52 .article-callout, .slug-content {color:#404040;}
53 .article-sidebar {color:#202020;}
54 '''
55
56 def preprocess_raw_html(self, raw, url):
57 root = parse(raw)
58 script = root.xpath('//script[@id="__NEXT_DATA__"]')
59 data = json.loads(script[0].text)
60 data = data['props']['pageProps']['article']
61 endpoint_url = 'https://platform.hbr.org/hbr/bff/content/article' + urlparse(url).path
62
63 topic = ''
64 if data.get('primaryTopic'):
65 topic = f'<div class="topic">{data["primaryTopic"]}</div>'
66 title = f'<h1>{data["title"]}</h1>'
67 dek = f'<p class="sub">{data.get("dek", "")}</p>'
68 hero = ''
69 if data.get('hero'):
70 hero = f'<img src={absurl(data["hero"]["image"]["defaultSrc"])}>'
71 auth = ''
72 if data.get('authors'):
73 auth = f'<p class="auth">{"By " + ", ".join(x["name"] for x in data.get("authors", {}))}</p>'
74
75 key_ = {
76 'contentKey': data['contentKey'],
77 }
78 headers = {
79 'User-Agent': random_user_agent(),
80 'Pragma': 'no-cache',
81 'Cache-Control': 'no-cache',
82 'Content-Type': 'application/json',
83 }
84 br = browser()
85 req = Request(
86 endpoint_url,
87 headers=headers,
88 data=json.dumps(key_),
89 method='POST',
90 timeout=self.timeout,
91 )
92 res = br.open(req)
93 body = json.loads(res.read())['content']
94
95 return '<html><body><article>' + topic + title + dek + hero + auth + body + '</article></body></html>'
96
97 recipe_specific_options = {
98 'issue': {
99 'short': 'Enter the Issue Number you want to download ',
100 'long': 'For example, 2403',
101 }
102 }
103
104 def parse_index(self):
105 d = self.recipe_specific_options.get('issue')
106 if not (d and isinstance(d, str)):
107 soup = self.index_to_soup(f'{self.base_url}/magazine')
108 a = soup.find('a', href=lambda x: x and x.startswith('/archive-toc/'))
109 cov_url = a.find('img', attrs={'src': True})['src']
110 self.cover_url = absurl(cov_url)
111 issue_url = absurl(a['href'])
112 else:
113 issue_url = 'https://hbr.org/archive-toc/BR' + d
114 mobj = re.search(r'archive-toc/(?P<issue>(BR)?\d+)\b', issue_url)
115 if mobj:
116 self.cover_url = f'https://hbr.org/resources/images/covers/{mobj.group("issue")}_500.png'
117
118 soup = self.index_to_soup(issue_url)
119 issue_title = soup.find('h1')
120 if issue_title:
121 self.timefmt = f' [{self.tag_to_string(issue_title)}]'
122
123 feeds = OrderedDict()
124
125 for h3 in soup.findAll('h3', attrs={'class': 'hed'}):
126 articles = []
127 a = h3.find('a')
128 title = self.tag_to_string(a)
129 url = absurl(a['href'])
130 auth = ''
131 div = h3.find_next_sibling('div', attrs={'class': 'stream-item-info'})
132 if div:
133 aut = self.tag_to_string(div).replace('Magazine Article ', '')
134 auth = re.sub(r'(?<=\w)([A-Z])', r', \1', aut)
135 des = ''
136 dek = h3.find_next_sibling('div', attrs={'class': 'dek'})
137 if dek:
138 des = self.tag_to_string(dek)
139 desc = des + ' |' + auth.title()
140 section_title = 'Articles'
141 sec = h3.findParent('li').find_previous_sibling('div', **classes('stream-section-label')).find('h4')
142 if sec:
143 section_title = self.tag_to_string(sec).title()
144 self.log(section_title, '\n\t', title, '\n\t', desc, '\n\t\t', url)
145 articles.append({'title': title, 'url': url, 'description': desc})
146 if articles:
147 if section_title not in feeds:
148 feeds[section_title] = []
149 feeds[section_title] += articles
150 ans = [(key, val) for key, val in feeds.items()]
151 return ans
152
153 def preprocess_html(self, soup):
154 for slug in soup.findAll(**classes('slug-content')):
155 del slug['href']
156 for dek in soup.findAll(**classes('article-byline')):
157 for by in dek.findAll('span', attrs={'class': 'by-prefix'}):
158 by.extract()
159 for li in dek.findAll('li'):
160 li.name = 'span'
161 for div in soup.findAll('div', attrs={'class': ['article-summary', 'article-callout']}):
162 div.name = 'blockquote'
163 for sidebar in soup.findAll(('article-sidebar', 'article-ideainbrief')):
164 sidebar.name = 'blockquote'
165 for img in soup.findAll('img', attrs={'srcset': True}):
166 split = img['srcset'].split(',')
167 for x in split:
168 if '700w' in x:
169 img['src'] = absurl(x.split()[0])
170 del img['srcset']
171 return soup
172
173 # HBR changes the content it delivers based on cookies, so the
174 # following ensures that we send no cookies
175 def get_browser(self, *args, **kwargs):
176 return self
177
178 def clone_browser(self, *args, **kwargs):
179 return self.get_browser()
180
181 def open_novisit(self, *args, **kwargs):
182 br = browser()
183 return br.open_novisit(*args, **kwargs)
184
185 open = open_novisit