| 1 |
#!/usr/bin/env python |
| 2 |
__license__ = 'GPL v3' |
| 3 |
__copyright__ = '2018, Gary Arnold [email protected]' |
| 4 |
__docformat__ = 'restructuredtext en' |
| 5 |
|
| 6 |
''' |
| 7 |
granta.com |
| 8 |
''' |
| 9 |
import re |
| 10 |
|
| 11 |
from calibre.web.feeds.news import BasicNewsRecipe |
| 12 |
|
| 13 |
# Set this variable to the URL of the issue you want to download, if not the current issue |
| 14 |
force_issue_download = None |
| 15 |
|
| 16 |
################################################################## |
| 17 |
# Adapted from https://gist.github.com/FlyingTopHat/7cfdd5434ec704916174 |
| 18 |
|
| 19 |
|
| 20 |
def classes(classes): |
| 21 |
q = frozenset(classes.split(' ')) |
| 22 |
return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)}) |
| 23 |
|
| 24 |
|
| 25 |
def plus_with_unknown_component(first_comp, second_comp, result): |
| 26 |
if result is None: |
| 27 |
return first_comp + second_comp |
| 28 |
|
| 29 |
component = first_comp if second_comp is None else second_comp |
| 30 |
return result - component |
| 31 |
|
| 32 |
|
| 33 |
def subtract_with_unknown_component(first_comp, second_comp, result): |
| 34 |
if result is None: |
| 35 |
return first_comp - second_comp |
| 36 |
|
| 37 |
return (first_comp - result) if second_comp is None else (result + second_comp) |
| 38 |
|
| 39 |
|
| 40 |
def multiply_with_unknown_component(first_comp, second_comp, result): |
| 41 |
if result is None: |
| 42 |
return first_comp * second_comp |
| 43 |
|
| 44 |
component = first_comp if second_comp is None else second_comp |
| 45 |
return result / component |
| 46 |
|
| 47 |
|
| 48 |
def solve_captcha(captcha): |
| 49 |
# Convert from a word problem into a numeric problem |
| 50 |
numeric_problem = '' |
| 51 |
for part in captcha.split(' '): |
| 52 |
numeric_problem = numeric_problem + str(text2num(part)) |
| 53 |
|
| 54 |
# Parse into parts |
| 55 |
pattern = re.compile( |
| 56 |
r'(?P<first_component>[0-9]+)?' |
| 57 |
'\\s*(?P<operator>[+×−])\\s*' # noqa: RUF039 |
| 58 |
r'(?P<second_component>[0-9]+)' |
| 59 |
r'\s*(=)\s*' |
| 60 |
r'(?P<result>[0-9]+)?', |
| 61 |
re.UNICODE, |
| 62 |
) |
| 63 |
|
| 64 |
calculationParts = re.search(pattern, numeric_problem) |
| 65 |
if calculationParts is None: |
| 66 |
return 0 |
| 67 |
|
| 68 |
operator = calculationParts.group('operator') |
| 69 |
|
| 70 |
result = calculationParts.group('result') |
| 71 |
result = int(result) if result is not None else None |
| 72 |
|
| 73 |
component_one = calculationParts.group('first_component') |
| 74 |
component_one = int(component_one) if component_one is not None else None |
| 75 |
|
| 76 |
component_two = calculationParts.group('second_component') |
| 77 |
component_two = int(component_two) if component_two is not None else None |
| 78 |
|
| 79 |
# Calculate answer |
| 80 |
answer = 0 |
| 81 |
if operator == '+': |
| 82 |
answer = plus_with_unknown_component(component_one, component_two, result) |
| 83 |
elif operator == '×': |
| 84 |
answer = multiply_with_unknown_component(component_one, component_two, result) |
| 85 |
elif operator == '−': |
| 86 |
answer = subtract_with_unknown_component(component_one, component_two, result) |
| 87 |
|
| 88 |
return answer |
| 89 |
|
| 90 |
|
| 91 |
################################################################## |
| 92 |
|
| 93 |
|
| 94 |
################################################################## |
| 95 |
# Adapted from https://github.com/ghewgill/text2num/blob/master/text2num.py |
| 96 |
# Removes external dependency on digify library |
| 97 |
Small = { |
| 98 |
'zero': 0, |
| 99 |
'one': 1, |
| 100 |
'two': 2, |
| 101 |
'three': 3, |
| 102 |
'four': 4, |
| 103 |
'five': 5, |
| 104 |
'six': 6, |
| 105 |
'seven': 7, |
| 106 |
'eight': 8, |
| 107 |
'nine': 9, |
| 108 |
'ten': 10, |
| 109 |
'eleven': 11, |
| 110 |
'twelve': 12, |
| 111 |
'thirteen': 13, |
| 112 |
'fourteen': 14, |
| 113 |
'fifteen': 15, |
| 114 |
'sixteen': 16, |
| 115 |
'seventeen': 17, |
| 116 |
'eighteen': 18, |
| 117 |
'nineteen': 19, |
| 118 |
'twenty': 20, |
| 119 |
'thirty': 30, |
| 120 |
'forty': 40, |
| 121 |
'fifty': 50, |
| 122 |
'sixty': 60, |
| 123 |
'seventy': 70, |
| 124 |
'eighty': 80, |
| 125 |
'ninety': 90, |
| 126 |
} |
| 127 |
|
| 128 |
Magnitude = { |
| 129 |
'thousand': 1000, |
| 130 |
'million': 1000000, |
| 131 |
'billion': 1000000000, |
| 132 |
'trillion': 1000000000000, |
| 133 |
'quadrillion': 1000000000000000, |
| 134 |
'quintillion': 1000000000000000000, |
| 135 |
'sextillion': 1000000000000000000000, |
| 136 |
'septillion': 1000000000000000000000000, |
| 137 |
'octillion': 1000000000000000000000000000, |
| 138 |
'nonillion': 1000000000000000000000000000000, |
| 139 |
'decillion': 1000000000000000000000000000000000, |
| 140 |
} |
| 141 |
|
| 142 |
|
| 143 |
def text2num(s): |
| 144 |
a = re.split(r'[\s-]+', s) |
| 145 |
n = 0 |
| 146 |
g = 0 |
| 147 |
for w in a: |
| 148 |
x = Small.get(w, None) |
| 149 |
if x is not None: |
| 150 |
g += x |
| 151 |
elif w == 'hundred' and g != 0: |
| 152 |
g *= 100 |
| 153 |
else: |
| 154 |
x = Magnitude.get(w, None) |
| 155 |
if x is not None: |
| 156 |
n += g * x |
| 157 |
g = 0 |
| 158 |
else: |
| 159 |
return s |
| 160 |
return n + g |
| 161 |
|
| 162 |
|
| 163 |
################################################################## |
| 164 |
|
| 165 |
|
| 166 |
class Granta(BasicNewsRecipe): |
| 167 |
title = 'Granta' |
| 168 |
description = 'The Magazine of New Writing' |
| 169 |
language = 'en' |
| 170 |
|
| 171 |
__author__ = 'Gary Arnold' |
| 172 |
|
| 173 |
needs_subscription = 'optional' |
| 174 |
|
| 175 |
keep_only_tags = [ |
| 176 |
classes('article-header article-content article-feature-image-standard-container article-feature-image-full-width-container'), |
| 177 |
] |
| 178 |
remove_tags = [classes('social-share-container'), classes('article-sidebar__right__social-share-btns')] |
| 179 |
remove_attributes = ['style'] |
| 180 |
|
| 181 |
def get_browser(self): |
| 182 |
br = BasicNewsRecipe.get_browser(self) |
| 183 |
if self.username and self.password: |
| 184 |
# User has a subscription, log in |
| 185 |
response = br.open('https://granta.com/') |
| 186 |
|
| 187 |
# Get captcha solution |
| 188 |
captcha = '0' |
| 189 |
html = response.read() |
| 190 |
soup = self.index_to_soup(html) |
| 191 |
captcha_field = soup.find('input', attrs={'name': 'capcha'}) |
| 192 |
captcha_question = '' |
| 193 |
if captcha_field is not None: |
| 194 |
captcha_question = captcha_field['placeholder'] |
| 195 |
if captcha_question is not None: |
| 196 |
captcha = str(solve_captcha(captcha_question)) |
| 197 |
|
| 198 |
br.select_form(method='post', action='https://granta.com/') |
| 199 |
br['username'] = self.username |
| 200 |
br['password'] = self.password |
| 201 |
br['capcha'] = captcha |
| 202 |
self.log.info('Logging in with captcha ' + str(captcha_question) + ', solution ' + captcha) |
| 203 |
br.submit() |
| 204 |
|
| 205 |
return br |
| 206 |
|
| 207 |
def preprocess_html(self, soup): |
| 208 |
for div in soup.findAll(attrs={'data-background': True}): |
| 209 |
img = soup.new_tag('img') |
| 210 |
img['src'] = div['data-background'] |
| 211 |
div.append(img) |
| 212 |
return soup |
| 213 |
|
| 214 |
def parse_index(self): |
| 215 |
if force_issue_download is None: |
| 216 |
soup = self.index_to_soup('https://granta.com/issues') |
| 217 |
|
| 218 |
# Get latest issue |
| 219 |
issueInfo = soup.find(**classes('explore-issue-result')) |
| 220 |
issueAnchor = issueInfo.findParent('a', href=True) |
| 221 |
issueLink = issueAnchor.get('href') |
| 222 |
else: |
| 223 |
issueLink = force_issue_download |
| 224 |
|
| 225 |
self.log('Fetching issue:', issueLink) |
| 226 |
soup = self.index_to_soup(issueLink) |
| 227 |
# open('/t/raw.html', 'w').write(str(soup)) |
| 228 |
|
| 229 |
# Find cover |
| 230 |
cover = soup.find(**classes('single-issue__cover-image')) |
| 231 |
if cover is not None: |
| 232 |
self.cover_url = cover['data-background'] |
| 233 |
self.log.info('Found cover at:', self.cover_url) |
| 234 |
|
| 235 |
sections = {} |
| 236 |
for item in soup.findAll(**classes('single-contributor_related-row_container')): |
| 237 |
h6 = item.find('h6') |
| 238 |
section = self.tag_to_string(h6.find('a')).strip() |
| 239 |
sections.setdefault(section, []) |
| 240 |
h1 = item.find('h1') |
| 241 |
title = self.tag_to_string(h1).strip() |
| 242 |
url = h1.findParent('a')['href'] |
| 243 |
author = self.tag_to_string(item.findAll('h3')[-1]).strip() |
| 244 |
desc = '' |
| 245 |
for p in item.findAll('p'): |
| 246 |
desc += self.tag_to_string(p) |
| 247 |
sections[section].append({'title': title, 'url': url, 'description': 'by ' + author + '. ' + desc}) |
| 248 |
|
| 249 |
self.log.info('Found article:', title) |
| 250 |
self.log.info('\t', url) |
| 251 |
|
| 252 |
return [(sec, sections[sec]) for sec in sections] |