Debian Codegraph @

recipes/granta.recipe

raw
1 #!/usr/bin/env python
2 __license__ = 'GPL v3'
3 __copyright__ = '2018, Gary Arnold [email protected]'
4 __docformat__ = 'restructuredtext en'
5
6 '''
7 granta.com
8 '''
9 import re
10
11 from calibre.web.feeds.news import BasicNewsRecipe
12
13 # Set this variable to the URL of the issue you want to download, if not the current issue
14 force_issue_download = None
15
16 ##################################################################
17 # Adapted from https://gist.github.com/FlyingTopHat/7cfdd5434ec704916174
18
19
20 def classes(classes):
21 q = frozenset(classes.split(' '))
22 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
23
24
25 def plus_with_unknown_component(first_comp, second_comp, result):
26 if result is None:
27 return first_comp + second_comp
28
29 component = first_comp if second_comp is None else second_comp
30 return result - component
31
32
33 def subtract_with_unknown_component(first_comp, second_comp, result):
34 if result is None:
35 return first_comp - second_comp
36
37 return (first_comp - result) if second_comp is None else (result + second_comp)
38
39
40 def multiply_with_unknown_component(first_comp, second_comp, result):
41 if result is None:
42 return first_comp * second_comp
43
44 component = first_comp if second_comp is None else second_comp
45 return result / component
46
47
48 def solve_captcha(captcha):
49 # Convert from a word problem into a numeric problem
50 numeric_problem = ''
51 for part in captcha.split(' '):
52 numeric_problem = numeric_problem + str(text2num(part))
53
54 # Parse into parts
55 pattern = re.compile(
56 r'(?P<first_component>[0-9]+)?'
57 '\\s*(?P<operator>[+×−])\\s*' # noqa: RUF039
58 r'(?P<second_component>[0-9]+)'
59 r'\s*(=)\s*'
60 r'(?P<result>[0-9]+)?',
61 re.UNICODE,
62 )
63
64 calculationParts = re.search(pattern, numeric_problem)
65 if calculationParts is None:
66 return 0
67
68 operator = calculationParts.group('operator')
69
70 result = calculationParts.group('result')
71 result = int(result) if result is not None else None
72
73 component_one = calculationParts.group('first_component')
74 component_one = int(component_one) if component_one is not None else None
75
76 component_two = calculationParts.group('second_component')
77 component_two = int(component_two) if component_two is not None else None
78
79 # Calculate answer
80 answer = 0
81 if operator == '+':
82 answer = plus_with_unknown_component(component_one, component_two, result)
83 elif operator == '×':
84 answer = multiply_with_unknown_component(component_one, component_two, result)
85 elif operator == '−':
86 answer = subtract_with_unknown_component(component_one, component_two, result)
87
88 return answer
89
90
91 ##################################################################
92
93
94 ##################################################################
95 # Adapted from https://github.com/ghewgill/text2num/blob/master/text2num.py
96 # Removes external dependency on digify library
97 Small = {
98 'zero': 0,
99 'one': 1,
100 'two': 2,
101 'three': 3,
102 'four': 4,
103 'five': 5,
104 'six': 6,
105 'seven': 7,
106 'eight': 8,
107 'nine': 9,
108 'ten': 10,
109 'eleven': 11,
110 'twelve': 12,
111 'thirteen': 13,
112 'fourteen': 14,
113 'fifteen': 15,
114 'sixteen': 16,
115 'seventeen': 17,
116 'eighteen': 18,
117 'nineteen': 19,
118 'twenty': 20,
119 'thirty': 30,
120 'forty': 40,
121 'fifty': 50,
122 'sixty': 60,
123 'seventy': 70,
124 'eighty': 80,
125 'ninety': 90,
126 }
127
128 Magnitude = {
129 'thousand': 1000,
130 'million': 1000000,
131 'billion': 1000000000,
132 'trillion': 1000000000000,
133 'quadrillion': 1000000000000000,
134 'quintillion': 1000000000000000000,
135 'sextillion': 1000000000000000000000,
136 'septillion': 1000000000000000000000000,
137 'octillion': 1000000000000000000000000000,
138 'nonillion': 1000000000000000000000000000000,
139 'decillion': 1000000000000000000000000000000000,
140 }
141
142
143 def text2num(s):
144 a = re.split(r'[\s-]+', s)
145 n = 0
146 g = 0
147 for w in a:
148 x = Small.get(w, None)
149 if x is not None:
150 g += x
151 elif w == 'hundred' and g != 0:
152 g *= 100
153 else:
154 x = Magnitude.get(w, None)
155 if x is not None:
156 n += g * x
157 g = 0
158 else:
159 return s
160 return n + g
161
162
163 ##################################################################
164
165
166 class Granta(BasicNewsRecipe):
167 title = 'Granta'
168 description = 'The Magazine of New Writing'
169 language = 'en'
170
171 __author__ = 'Gary Arnold'
172
173 needs_subscription = 'optional'
174
175 keep_only_tags = [
176 classes('article-header article-content article-feature-image-standard-container article-feature-image-full-width-container'),
177 ]
178 remove_tags = [classes('social-share-container'), classes('article-sidebar__right__social-share-btns')]
179 remove_attributes = ['style']
180
181 def get_browser(self):
182 br = BasicNewsRecipe.get_browser(self)
183 if self.username and self.password:
184 # User has a subscription, log in
185 response = br.open('https://granta.com/')
186
187 # Get captcha solution
188 captcha = '0'
189 html = response.read()
190 soup = self.index_to_soup(html)
191 captcha_field = soup.find('input', attrs={'name': 'capcha'})
192 captcha_question = ''
193 if captcha_field is not None:
194 captcha_question = captcha_field['placeholder']
195 if captcha_question is not None:
196 captcha = str(solve_captcha(captcha_question))
197
198 br.select_form(method='post', action='https://granta.com/')
199 br['username'] = self.username
200 br['password'] = self.password
201 br['capcha'] = captcha
202 self.log.info('Logging in with captcha ' + str(captcha_question) + ', solution ' + captcha)
203 br.submit()
204
205 return br
206
207 def preprocess_html(self, soup):
208 for div in soup.findAll(attrs={'data-background': True}):
209 img = soup.new_tag('img')
210 img['src'] = div['data-background']
211 div.append(img)
212 return soup
213
214 def parse_index(self):
215 if force_issue_download is None:
216 soup = self.index_to_soup('https://granta.com/issues')
217
218 # Get latest issue
219 issueInfo = soup.find(**classes('explore-issue-result'))
220 issueAnchor = issueInfo.findParent('a', href=True)
221 issueLink = issueAnchor.get('href')
222 else:
223 issueLink = force_issue_download
224
225 self.log('Fetching issue:', issueLink)
226 soup = self.index_to_soup(issueLink)
227 # open('/t/raw.html', 'w').write(str(soup))
228
229 # Find cover
230 cover = soup.find(**classes('single-issue__cover-image'))
231 if cover is not None:
232 self.cover_url = cover['data-background']
233 self.log.info('Found cover at:', self.cover_url)
234
235 sections = {}
236 for item in soup.findAll(**classes('single-contributor_related-row_container')):
237 h6 = item.find('h6')
238 section = self.tag_to_string(h6.find('a')).strip()
239 sections.setdefault(section, [])
240 h1 = item.find('h1')
241 title = self.tag_to_string(h1).strip()
242 url = h1.findParent('a')['href']
243 author = self.tag_to_string(item.findAll('h3')[-1]).strip()
244 desc = ''
245 for p in item.findAll('p'):
246 desc += self.tag_to_string(p)
247 sections[section].append({'title': title, 'url': url, 'description': 'by ' + author + '. ' + desc})
248
249 self.log.info('Found article:', title)
250 self.log.info('\t', url)
251
252 return [(sec, sections[sec]) for sec in sections]