| 1 |
#!/usr/bin/env python |
| 2 |
# vim:fileencoding=utf-8 |
| 3 |
import json |
| 4 |
import re |
| 5 |
from collections import OrderedDict |
| 6 |
from urllib.parse import urlparse |
| 7 |
|
| 8 |
from html5_parser import parse |
| 9 |
from mechanize import Request |
| 10 |
|
| 11 |
from calibre import browser, random_user_agent |
| 12 |
from calibre.web.feeds.news import BasicNewsRecipe, classes |
| 13 |
|
| 14 |
|
| 15 |
def absurl(url): |
| 16 |
if url.startswith('/'): |
| 17 |
url = 'https://www.hbr.org' + url |
| 18 |
return url |
| 19 |
|
| 20 |
|
| 21 |
class HBR(BasicNewsRecipe): |
| 22 |
title = 'Harvard Business Review' |
| 23 |
__author__ = 'unkn0wn' |
| 24 |
description = ( |
| 25 |
'Harvard Business Review is the leading destination for smart management thinking. ' |
| 26 |
'Through its flagship magazine, books, and digital content and tools published on HBR.org, ' |
| 27 |
'Harvard Business Review aims to provide professionals around the world with rigorous insights ' |
| 28 |
'and best practices to help lead themselves and their organizations more effectively and to ' |
| 29 |
'make a positive impact.' |
| 30 |
) |
| 31 |
language = 'en' |
| 32 |
masthead_url = 'https://hbr.org/resources/css/images/hbr_logo.svg' |
| 33 |
publication_type = 'magazine' |
| 34 |
encoding = 'utf-8' |
| 35 |
remove_javascript = True |
| 36 |
no_stylesheets = True |
| 37 |
compress_news_images = True |
| 38 |
ignore_duplicate_articles = {'url'} |
| 39 |
base_url = 'https://hbr.org' |
| 40 |
remove_attributes = ['height', 'width', 'style'] |
| 41 |
resolve_internal_links = True |
| 42 |
|
| 43 |
extra_css = ''' |
| 44 |
.article-summary, .article-ideainbrief, |
| 45 |
.description-text, .link--black, .topic, .auth {font-size:small; color:#202020;} |
| 46 |
.credits--hero-image, .credits--inline-image, .caption--inline-image, |
| 47 |
.calibre-nuked-tag-figcaption {font-size:small; text-align:center;} |
| 48 |
.sub { font-style: italic; } |
| 49 |
.article-byline-list {font-size:small; font-weight:bold;} |
| 50 |
.question {font-weight:bold;} |
| 51 |
.right-rail--container {font-size:small; color:#404040;} |
| 52 |
.article-callout, .slug-content {color:#404040;} |
| 53 |
.article-sidebar {color:#202020;} |
| 54 |
''' |
| 55 |
|
| 56 |
def preprocess_raw_html(self, raw, url): |
| 57 |
root = parse(raw) |
| 58 |
script = root.xpath('//script[@id="__NEXT_DATA__"]') |
| 59 |
data = json.loads(script[0].text) |
| 60 |
data = data['props']['pageProps']['article'] |
| 61 |
endpoint_url = 'https://platform.hbr.org/hbr/bff/content/article' + urlparse(url).path |
| 62 |
|
| 63 |
topic = '' |
| 64 |
if data.get('primaryTopic'): |
| 65 |
topic = f'<div class="topic">{data["primaryTopic"]}</div>' |
| 66 |
title = f'<h1>{data["title"]}</h1>' |
| 67 |
dek = f'<p class="sub">{data.get("dek", "")}</p>' |
| 68 |
hero = '' |
| 69 |
if data.get('hero'): |
| 70 |
hero = f'<img src={absurl(data["hero"]["image"]["defaultSrc"])}>' |
| 71 |
auth = '' |
| 72 |
if data.get('authors'): |
| 73 |
auth = f'<p class="auth">{"By " + ", ".join(x["name"] for x in data.get("authors", {}))}</p>' |
| 74 |
|
| 75 |
key_ = { |
| 76 |
'contentKey': data['contentKey'], |
| 77 |
} |
| 78 |
headers = { |
| 79 |
'User-Agent': random_user_agent(), |
| 80 |
'Pragma': 'no-cache', |
| 81 |
'Cache-Control': 'no-cache', |
| 82 |
'Content-Type': 'application/json', |
| 83 |
} |
| 84 |
br = browser() |
| 85 |
req = Request( |
| 86 |
endpoint_url, |
| 87 |
headers=headers, |
| 88 |
data=json.dumps(key_), |
| 89 |
method='POST', |
| 90 |
timeout=self.timeout, |
| 91 |
) |
| 92 |
res = br.open(req) |
| 93 |
body = json.loads(res.read())['content'] |
| 94 |
|
| 95 |
return '<html><body><article>' + topic + title + dek + hero + auth + body + '</article></body></html>' |
| 96 |
|
| 97 |
recipe_specific_options = { |
| 98 |
'issue': { |
| 99 |
'short': 'Enter the Issue Number you want to download ', |
| 100 |
'long': 'For example, 2403', |
| 101 |
} |
| 102 |
} |
| 103 |
|
| 104 |
def parse_index(self): |
| 105 |
d = self.recipe_specific_options.get('issue') |
| 106 |
if not (d and isinstance(d, str)): |
| 107 |
soup = self.index_to_soup(f'{self.base_url}/magazine') |
| 108 |
a = soup.find('a', href=lambda x: x and x.startswith('/archive-toc/')) |
| 109 |
cov_url = a.find('img', attrs={'src': True})['src'] |
| 110 |
self.cover_url = absurl(cov_url) |
| 111 |
issue_url = absurl(a['href']) |
| 112 |
else: |
| 113 |
issue_url = 'https://hbr.org/archive-toc/BR' + d |
| 114 |
mobj = re.search(r'archive-toc/(?P<issue>(BR)?\d+)\b', issue_url) |
| 115 |
if mobj: |
| 116 |
self.cover_url = f'https://hbr.org/resources/images/covers/{mobj.group("issue")}_500.png' |
| 117 |
|
| 118 |
soup = self.index_to_soup(issue_url) |
| 119 |
issue_title = soup.find('h1') |
| 120 |
if issue_title: |
| 121 |
self.timefmt = f' [{self.tag_to_string(issue_title)}]' |
| 122 |
|
| 123 |
feeds = OrderedDict() |
| 124 |
|
| 125 |
for h3 in soup.findAll('h3', attrs={'class': 'hed'}): |
| 126 |
articles = [] |
| 127 |
a = h3.find('a') |
| 128 |
title = self.tag_to_string(a) |
| 129 |
url = absurl(a['href']) |
| 130 |
auth = '' |
| 131 |
div = h3.find_next_sibling('div', attrs={'class': 'stream-item-info'}) |
| 132 |
if div: |
| 133 |
aut = self.tag_to_string(div).replace('Magazine Article ', '') |
| 134 |
auth = re.sub(r'(?<=\w)([A-Z])', r', \1', aut) |
| 135 |
des = '' |
| 136 |
dek = h3.find_next_sibling('div', attrs={'class': 'dek'}) |
| 137 |
if dek: |
| 138 |
des = self.tag_to_string(dek) |
| 139 |
desc = des + ' |' + auth.title() |
| 140 |
section_title = 'Articles' |
| 141 |
sec = h3.findParent('li').find_previous_sibling('div', **classes('stream-section-label')).find('h4') |
| 142 |
if sec: |
| 143 |
section_title = self.tag_to_string(sec).title() |
| 144 |
self.log(section_title, '\n\t', title, '\n\t', desc, '\n\t\t', url) |
| 145 |
articles.append({'title': title, 'url': url, 'description': desc}) |
| 146 |
if articles: |
| 147 |
if section_title not in feeds: |
| 148 |
feeds[section_title] = [] |
| 149 |
feeds[section_title] += articles |
| 150 |
ans = [(key, val) for key, val in feeds.items()] |
| 151 |
return ans |
| 152 |
|
| 153 |
def preprocess_html(self, soup): |
| 154 |
for slug in soup.findAll(**classes('slug-content')): |
| 155 |
del slug['href'] |
| 156 |
for dek in soup.findAll(**classes('article-byline')): |
| 157 |
for by in dek.findAll('span', attrs={'class': 'by-prefix'}): |
| 158 |
by.extract() |
| 159 |
for li in dek.findAll('li'): |
| 160 |
li.name = 'span' |
| 161 |
for div in soup.findAll('div', attrs={'class': ['article-summary', 'article-callout']}): |
| 162 |
div.name = 'blockquote' |
| 163 |
for sidebar in soup.findAll(('article-sidebar', 'article-ideainbrief')): |
| 164 |
sidebar.name = 'blockquote' |
| 165 |
for img in soup.findAll('img', attrs={'srcset': True}): |
| 166 |
split = img['srcset'].split(',') |
| 167 |
for x in split: |
| 168 |
if '700w' in x: |
| 169 |
img['src'] = absurl(x.split()[0]) |
| 170 |
del img['srcset'] |
| 171 |
return soup |
| 172 |
|
| 173 |
# HBR changes the content it delivers based on cookies, so the |
| 174 |
# following ensures that we send no cookies |
| 175 |
def get_browser(self, *args, **kwargs): |
| 176 |
return self |
| 177 |
|
| 178 |
def clone_browser(self, *args, **kwargs): |
| 179 |
return self.get_browser() |
| 180 |
|
| 181 |
def open_novisit(self, *args, **kwargs): |
| 182 |
br = browser() |
| 183 |
return br.open_novisit(*args, **kwargs) |
| 184 |
|
| 185 |
open = open_novisit |