Debian Codegraph @

recipes/lrb.recipe

raw
1 #!/usr/bin/env python
2 # vim:fileencoding=utf-8
3 # License: GPLv3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
4 from calibre.web.feeds.news import BasicNewsRecipe
5
6
7 def classes(classes):
8 q = frozenset(classes.split(' '))
9 return dict(attrs={'class': lambda x: x and frozenset(x.split()).intersection(q)})
10
11
12 def absolutize(href):
13 if href.startswith('/'):
14 href = 'https://www.lrb.co.uk' + href
15 return href
16
17
18 class LondonReviewOfBooksPayed(BasicNewsRecipe):
19 title = 'London Review of Books'
20 __author__ = 'Kovid Goyal'
21 description = 'Literary review publishing essay-length book reviews and topical articles on politics, literature, history, philosophy, science and the arts by leading writers and thinkers' # noqa: E501
22 category = 'news, literature, UK'
23 publisher = 'LRB Ltd.'
24 language = 'en_GB'
25 no_stylesheets = True
26 delay = 1
27 encoding = 'utf-8'
28 INDEX = 'https://www.lrb.co.uk/the-paper/'
29 publication_type = 'magazine'
30 needs_subscription = 'optional'
31 requires_version = (3, 0, 0)
32 masthead_url = 'https://www.mylrb.co.uk/out/lrb-2014/img/logo-2x.png'
33 extra_css = '''
34 .article-word-count, #article-tag-holder { font-size:small; color:#202020; }
35 .embedded-image-caption { font-size:small; text-align:center; }
36 blockquote, em { color:#202020; }
37 '''
38 resolve_internal_links = True
39
40 keep_only_tags = [
41 dict(attrs={'id': ['article-tag-holder', 'article-heading-holder']}),
42 classes('article-copy article-word-count'),
43 ]
44
45 remove_attributes = ['style', 'width', 'height']
46
47 def get_browser(self):
48 br = BasicNewsRecipe.get_browser(self)
49 if self.username and self.password:
50 br.open('https://www.lrb.co.uk/login')
51 br.select_form(id='login_form')
52 br['_username'] = self.username
53 br['_password'] = self.password
54 raw = br.submit().read()
55 if b'>My Account<' not in raw:
56 raise ValueError('Failed to login check username and password')
57 return br
58
59 def preprocess_html(self, soup):
60 for h2 in soup.findAll('h2'):
61 h2.name = 'h4'
62 for cap in soup.findAll(**classes('embedded-image-caption')):
63 for p in cap.findAll('p'):
64 p.name = 'div'
65 for img in soup.findAll('img', attrs={'data-srcset': True}):
66 img['src'] = 'https://www.lrb.co.uk/storage/400_filter/images/' + img['data-appsrc'].split('/images/')[-1]
67 return soup
68
69 recipe_specific_options = {
70 'issue_url': {'short': 'The issue URL ', 'long': 'For example, https://www.lrb.co.uk/the-paper/v46/n01', 'default': 'https://www.lrb.co.uk/the-paper/'}
71 }
72
73 def parse_index(self):
74 d = self.recipe_specific_options.get('issue_url')
75 if d and isinstance(d, str):
76 self.INDEX = d
77 soup = self.index_to_soup(self.INDEX)
78 container = soup.find('div', attrs={'class': 'article-issue-cover-image'})
79 if container:
80 self.cover_url = 'https://www.lrb.co.uk/storage/800_filter/images/' + container.img['data-appsrc'].split('/images/')[-1]
81 edition = self.tag_to_string(soup.find('h1', attrs={'class': 'toc-title'}))
82 self.timefmt = ' [{}]'.format(edition)
83 self.log('Downloading: ', edition)
84 grid = soup.find(attrs={'class': 'toc-grid-items'})
85 articles = []
86 for a in grid.findAll(**classes('toc-item')):
87 url = absolutize(a['href'])
88 h3 = a.find('h3')
89 h4 = a.find('h4')
90 title = '{}: {}'.format(self.tag_to_string(h3), self.tag_to_string(h4))
91 self.log(title, url)
92 articles.append({'title': title, 'url': url})
93 return [('Articles', articles)]